diff --git a/data_import/admin.py b/data_import/admin.py index 15fd94015..064ed5afe 100644 --- a/data_import/admin.py +++ b/data_import/admin.py @@ -4,3 +4,4 @@ admin.site.register(models.DataFile) admin.site.register(models.NewDataFileAccessLog) +admin.site.register(models.DataType) diff --git a/data_import/forms.py b/data_import/forms.py new file mode 100644 index 000000000..3a152c04a --- /dev/null +++ b/data_import/forms.py @@ -0,0 +1,60 @@ +from django import forms + +from .models import DataType + + +class DataTypeForm(forms.ModelForm): + """ + A form for creating and editing DataTypes. + """ + + class Meta: # noqa: D101 + model = DataType + fields = ["name", "parent", "description"] + + def __init__(self, *args, **kwargs): + self.editor = kwargs.pop("editor") + return super().__init__(*args, **kwargs) + + def clean_parent(self): + """ + Verify that the parent is not the object itself nor a descendent. + """ + parent = self.cleaned_data.get("parent") + if not parent: + return parent + if self.instance.id == parent.id: + raise forms.ValidationError( + "A DataType cannot be assigned to be its own parent." + ) + elif self.instance in parent.all_parents: + raise forms.ValidationError( + "{0} is not an allowed parent, as it is a descendent of {1}.".format( + parent.name, self.instance.name + ) + ) + return parent + + def clean_name(self): + """ + Verify that the name is case insensitive unique. + """ + name = self.cleaned_data.get("name") + try: + dt = DataType.objects.get(name__iexact=name) + except DataType.DoesNotExist: + dt = self.instance + if not dt == self.instance: + raise forms.ValidationError( + "Please provide a unique name for this datatype" + ) + return name + + def clean(self, *args, **kwargs): + if self.instance: + if not self.instance.editable: + raise forms.ValidationError( + "Not editable: in use by one or more approved projects." + ) + self.instance.editor = self.editor + return super().clean(*args, **kwargs) diff --git a/data_import/migrations/0019_datatype.py b/data_import/migrations/0019_datatype.py new file mode 100644 index 000000000..ec77a4fb2 --- /dev/null +++ b/data_import/migrations/0019_datatype.py @@ -0,0 +1,69 @@ +# Generated by Django 2.2 on 2019-04-12 19:08 + +import django.contrib.postgres.fields.jsonb +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("open_humans", "0014_member_password_reset_redirect"), + ("data_import", "0018_auto_20190402_1947"), + ] + + operations = [ + migrations.CreateModel( + name="DataType", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "name", + models.CharField( + max_length=128, + unique=True, + validators=[ + django.core.validators.RegexValidator( + "^[\\w\\-\\s]+$", + "Only alphanumeric characters, space, dash, and underscore are allowed.", + ) + ], + ), + ), + ("description", models.CharField(max_length=512)), + ("created", models.DateTimeField(auto_now_add=True)), + ("modified", models.DateTimeField(auto_now=True)), + ( + "history", + django.contrib.postgres.fields.jsonb.JSONField(default=dict), + ), + ( + "last_editor", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="open_humans.Member", + ), + ), + ( + "parent", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.PROTECT, + related_name="children", + to="data_import.DataType", + ), + ), + ], + ) + ] diff --git a/data_import/models.py b/data_import/models.py index db2a043ca..f2c81ef3e 100644 --- a/data_import/models.py +++ b/data_import/models.py @@ -1,25 +1,35 @@ +from collections import OrderedDict import datetime import logging import os import uuid +import arrow from botocore.exceptions import ClientError from django.conf import settings from django.contrib.postgres.fields import JSONField +from django.core.validators import RegexValidator from django.urls import reverse from django.db import models from django.db.models import F +from django.utils import timezone from ipware.ip import get_ip from common import fields from common.utils import full_url +from open_humans.models import Member from .utils import get_upload_path logger = logging.getLogger(__name__) +charvalidator = RegexValidator( + r"^[\w\-\s]+$", + "Only alphanumeric characters, space, dash, and underscore are allowed.", +) + def is_public(member, source): """ @@ -267,3 +277,140 @@ class TestUserData(models.Model): related_name="test_user_data", on_delete=models.CASCADE, ) + + +class DataType(models.Model): + """ + Describes the types of data a DataFile can contain. + """ + + name = models.CharField( + max_length=128, blank=False, unique=True, validators=[charvalidator] + ) + parent = models.ForeignKey( + "self", blank=True, null=True, related_name="children", on_delete=models.PROTECT + ) + last_editor = models.ForeignKey(Member, on_delete=models.SET_NULL, null=True) + description = models.CharField(max_length=512, blank=False) + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + history = JSONField(default=dict) + + def __str__(self): + parents = self.all_parents + if parents: + parents.reverse() + parents = [parent.name for parent in parents if parent] + parents = ":".join(parents) + return str("{0}:{1}").format(parents, self.name) + return self.name + + def save(self, *args, **kwargs): + """ + Override save to record edit history and require an associated "editor". + + "editor" is an instance-specific parameter; this avoids accepting an update + that is merely retaining the existing value for the "last_editor" field. + """ + if not self.editor: + raise ValueError("'self.editor' must be set when saving DataType.") + else: + self.last_editor = self.editor + self.history[arrow.get(timezone.now()).isoformat()] = { + "name": self.name, + "parent": self.parent.id if self.parent else None, + "description": self.description, + "editor": self.last_editor.id, + } + return super().save(*args, **kwargs) + + @property + def history_sorted(self): + history_sorted = OrderedDict() + items_sorted = sorted( + self.history.items(), key=lambda item: arrow.get(item[0]), reverse=True + ) + for item in items_sorted: + parent = ( + DataType.objects.get(id=item[1]["parent"]) + if item[1]["parent"] + else None + ) + try: + editor = Member.objects.get(id=item[1]["editor"]) + except Member.DoesNotExist: + editor = None + history_sorted[arrow.get(item[0]).datetime] = { + "name": item[1]["name"], + "parent": parent, + "description": item[1]["description"], + "editor": editor, + } + return history_sorted + + @property + def editable(self): + """ + Return True if no approved projects are registered as using this. + """ + # Always true for a new instance that hasn't yet been saved: + if not self.id: + return True + + approved_registered = self.datarequestproject_set.filter(approved=True) + if approved_registered: + return False + else: + return True + + @property + def all_parents(self): + """ + Return list of parents, from immediate to most ancestral. + """ + parent = self.parent + parents = [] + if parent: + while True: + if not parent: + break + parents.append(parent) + parent = parent.parent + + return parents + + @classmethod + def all_as_tree(cls): + """ + Dict tree of all datatypes. Key = parent & value = array of child dicts. + + This method is intended to make all ancestry relationships available without + having to hit the database more than necessary. + """ + + def _children(parent, all_datatypes): + children = {} + for dt in [dt for dt in all_datatypes if dt.parent == parent]: + children[dt] = _children(dt, all_datatypes) + return children + + all_datatypes = list(DataType.objects.all()) + roots = DataType.objects.filter(parent=None) + tree = {dt: _children(dt, all_datatypes) for dt in roots} + return tree + + @classmethod + def sorted_by_ancestors(cls, queryset=None): + """ + Sort DataTypes by ancestors array of dicts containing 'datatype' and 'depth'. + """ + + def _flatten(node, depth=0): + flattened = [] + for child in sorted(node.keys(), key=lambda obj: obj.name): + flattened.append({"datatype": child, "depth": depth}) + flattened = flattened + _flatten(node[child], depth=depth + 1) + return flattened + + datatypes_tree = cls.all_as_tree() + return _flatten(datatypes_tree) diff --git a/data_import/serializers.py b/data_import/serializers.py index f3767371b..3683ac555 100644 --- a/data_import/serializers.py +++ b/data_import/serializers.py @@ -3,7 +3,9 @@ from rest_framework import serializers -from .models import AWSDataFileAccessLog, DataFile, NewDataFileAccessLog +from private_sharing.models import DataRequestProject + +from .models import AWSDataFileAccessLog, DataFile, DataType, NewDataFileAccessLog def serialize_datafile_to_dict(datafile): @@ -87,3 +89,27 @@ class Meta: # noqa: D101 "host_header", "datafile", ] + + +class DataTypeSerializer(serializers.ModelSerializer): + """ + Serialize DataTypes + """ + + class Meta: # noqa: D101 + model = DataType + + fields = ["id", "name", "parent", "description", "source_projects"] + + source_projects = serializers.SerializerMethodField() + + def get_source_projects(self, obj): + """ + Get approved projects that are registered as potential sources. + """ + projects = ( + DataRequestProject.objects.filter(approved=True) + .filter(registered_datatypes=obj) + .distinct() + ) + return [project.id_label for project in projects] diff --git a/data_import/templates/data_import/datatypes-create.html b/data_import/templates/data_import/datatypes-create.html new file mode 100644 index 000000000..eb1f442f3 --- /dev/null +++ b/data_import/templates/data_import/datatypes-create.html @@ -0,0 +1,69 @@ +{% extends 'base.html' %} + +{% load bootstrap_tags %} +{% load utilities %} + +{% block main %} +

Add a DataType

+

+ Use this form to add a new DataType. Please avoid redundant DataTypes and + assign your DataType to be a subcategory of another DataType when appropriate. +

+
+
+ {% csrf_token %} + + {% if form.errors %} +
+

Form errors

+
    + {% for error in form.errors %} +
  • {{ error }}: {{ form.errors|lookup:error }}
    • + {% endfor %} +
+

+
+ {% endif %} + +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ + +
+
+{% endblock %} diff --git a/data_import/templates/data_import/datatypes-detail.html b/data_import/templates/data_import/datatypes-detail.html new file mode 100644 index 000000000..5658346c0 --- /dev/null +++ b/data_import/templates/data_import/datatypes-detail.html @@ -0,0 +1,68 @@ +{% extends 'base.html' %} + +{% load bootstrap_tags %} + +{% block main %} +

DataType: {{ object.name }}

+ +

Name: {{ object.name }}

+

Description: {{ object.description }}

+

Ancestors: {% if object.parent %} + {% for parent in object.all_parents reversed %} + {{ parent.name }} > + {% endfor %} + {{ object.name }} + {% else %} + None + {% endif %} +

+

Children: {% if not object.children.all %}None{% endif %}

+{% if object.children.all %} + +{% endif %} +

Project sources: {% if not object.datarequestproject_set.all %}None{% endif %}

+{% if object.datarequestproject_set.all %} + +{% endif %} +
+{% if object.editable %} +

+ Edit this entry +

+{% endif %} +

Return to full list of DataTypes.

+{% if object.history_sorted %} +

History

+ +{% endif %} +{% endblock %} diff --git a/data_import/templates/data_import/datatypes-list.html b/data_import/templates/data_import/datatypes-list.html new file mode 100644 index 000000000..750b4416e --- /dev/null +++ b/data_import/templates/data_import/datatypes-list.html @@ -0,0 +1,19 @@ +{% extends 'base.html' %} + +{% block main %} +

DataTypes

+ + + +
+Add datatype + + +{% endblock main %} \ No newline at end of file diff --git a/data_import/templates/data_import/datatypes-update.html b/data_import/templates/data_import/datatypes-update.html new file mode 100644 index 000000000..9f978423c --- /dev/null +++ b/data_import/templates/data_import/datatypes-update.html @@ -0,0 +1,105 @@ +{% extends 'base.html' %} + +{% load bootstrap_tags %} +{% load utilities %} + +{% block main %} +

Update DataType: {{ object.name }}

+

+ Use this form to edit the following DataType: +

+ + + + + + + + + + + + + + + + + +
Name:{{ object.name }}
Description:{{ object.description }}
Parent:{% if object.parent %} + + {{ object.parent.name }}{% else %} + None + {% endif %}
Children: + {% if object.children.all %} + {% for child in object.children.all %} + + {{ child.name }}{% if not forloop.last %},{% endif %} + {% endfor %} + {% else %}None{% endif %} +
+
+
+ {% csrf_token %} + + {% if form.errors %} +
+

Form errors

+
    + {% for error in form.errors %} +
  • {{ error }}: {{ form.errors|lookup:error }}
    • + {% endfor %} +
+

+
+ {% endif %} + +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ + +
+
+{% endblock %} diff --git a/data_import/urls.py b/data_import/urls.py index 18c0ad6b1..6ff014a71 100644 --- a/data_import/urls.py +++ b/data_import/urls.py @@ -3,6 +3,10 @@ from .views import ( AWSDataFileAccessLogView, DataFileDownloadView, + DataTypesCreateView, + DataTypesDetailView, + DataTypesListView, + DataTypesUpdateView, NewDataFileAccessLogView, ) @@ -14,6 +18,21 @@ DataFileDownloadView.as_view(), name="datafile-download", ), + # DataType endpoints + re_path( + r"^datatypes/create/", DataTypesCreateView.as_view(), name="datatypes-create" + ), + re_path( + r"^datatypes/update/(?P[\w-]+)$", + DataTypesUpdateView.as_view(), + name="datatypes-update", + ), + re_path( + r"^datatypes/(?P[\w-]+)$", + DataTypesDetailView.as_view(), + name="datatypes-detail", + ), + re_path(r"^datatypes/", DataTypesListView.as_view(), name="datatypes-list"), # Custom API endpoints for OHLOG_PROJECT_ID path( "awsdatafileaccesslog/", diff --git a/data_import/views.py b/data_import/views.py index b50619c92..207e47611 100644 --- a/data_import/views.py +++ b/data_import/views.py @@ -2,14 +2,24 @@ from django.contrib.auth import get_user_model from django.http import HttpResponseForbidden, HttpResponseRedirect -from django.views.generic import View +from django.urls import reverse +from django.views.generic import CreateView, DetailView, TemplateView, UpdateView, View from django_filters.rest_framework import DjangoFilterBackend from ipware.ip import get_ip from rest_framework.generics import ListAPIView +from common.mixins import NeverCacheMixin, PrivateMixin + from .filters import AccessLogFilter -from .models import AWSDataFileAccessLog, DataFile, DataFileKey, NewDataFileAccessLog +from .forms import DataTypeForm +from .models import ( + AWSDataFileAccessLog, + DataFile, + DataFileKey, + DataType, + NewDataFileAccessLog, +) from .permissions import LogAPIAccessAllowed from common.mixins import NeverCacheMixin from data_import.serializers import ( @@ -122,3 +132,84 @@ def get_queryset(self): serialized_data_file__user_id=self.request.user.id ) return queryset + + +class DataTypesSortedMixin(object): + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + datatypes_sorted = DataType.sorted_by_ancestors() + try: + max_depth = max([i["depth"] for i in datatypes_sorted]) + except ValueError: + max_depth = 0 + context.update({"datatypes_sorted": datatypes_sorted, "max_depth": max_depth}) + return context + + +class DataTypesListView(NeverCacheMixin, TemplateView): + """ + List all DataTypes. + """ + + template_name = "data_import/datatypes-list.html" + + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + datatypes_sorted = DataType.sorted_by_ancestors() + try: + max_depth = max([i["depth"] for i in datatypes_sorted]) + except ValueError: + max_depth = 0 + context.update({"datatypes_sorted": datatypes_sorted, "max_depth": max_depth}) + return context + + +class DataTypesDetailView(NeverCacheMixin, DetailView): + """ + Information about a DataType. + """ + + model = DataType + template_name = "data_import/datatypes-detail.html" + + +class FormEditorMixin(object): + """ + Override get_form_kwargs to pass request user as 'editor' kwarg to a form. + """ + + def get_form_kwargs(self, *args, **kwargs): + kwargs = super().get_form_kwargs(*args, **kwargs) + kwargs["editor"] = self.request.user.member + return kwargs + + +class DataTypesCreateView( + PrivateMixin, DataTypesSortedMixin, FormEditorMixin, CreateView +): + """ + Create a new DataType. + """ + + form_class = DataTypeForm + template_name = "data_import/datatypes-create.html" + + def get_success_url(self): + return reverse("data-management:datatypes-list") + + +class DataTypesUpdateView( + PrivateMixin, DataTypesSortedMixin, FormEditorMixin, UpdateView +): + """ + Edit a DataType. + """ + + model = DataType + form_class = DataTypeForm + template_name = "data_import/datatypes-update.html" + + def get_success_url(self): + return reverse( + "data-management:datatypes-detail", kwargs={"pk": self.object.id} + ) diff --git a/open_humans/api_urls.py b/open_humans/api_urls.py index 609dae7ee..9aaf51f87 100644 --- a/open_humans/api_urls.py +++ b/open_humans/api_urls.py @@ -18,6 +18,16 @@ api_views.PublicDataUsersBySourceAPIView.as_view(), name="members-by-source", ), + path( + "public-data/datatypes/", + api_views.PublicDataTypesListAPIView.as_view(), + name="datatypes", + ), + path( + "public-data/projects/", + api_views.PublicProjectsListAPIView.as_view(), + name="projects", + ), ] diff --git a/open_humans/api_views.py b/open_humans/api_views.py index dc20d7ecb..684319399 100644 --- a/open_humans/api_views.py +++ b/open_humans/api_views.py @@ -4,11 +4,13 @@ from rest_framework.filters import SearchFilter from rest_framework.generics import ListAPIView -from data_import.models import DataFile -from private_sharing.models import DataRequestProject +from common.mixins import NeverCacheMixin +from data_import.models import DataFile, DataType +from data_import.serializers import DataTypeSerializer +from private_sharing.models import id_label_to_project, DataRequestProject +from private_sharing.serializers import ProjectDataSerializer from public_data.serializers import PublicDataFileSerializer -from common.mixins import NeverCacheMixin from .filters import PublicDataFileFilter from .serializers import ( DataUsersBySourceSerializer, @@ -89,3 +91,45 @@ class PublicDataUsersBySourceAPIView(NeverCacheMixin, ListAPIView): active=True, approved=True, no_public_data=False ) serializer_class = DataUsersBySourceSerializer + + +class PublicDataTypesListAPIView(ListAPIView): + """ + Return list of DataTypes and source projects that have registered them. + """ + + serializer_class = DataTypeSerializer + + def get_queryset(self): + """ + Get the queryset and filter on project if provided. + """ + source_project_label = self.request.GET.get("source_project_label", None) + if source_project_label: + source_project = id_label_to_project(source_project_label) + queryset = source_project.registered_datatypes.all() + else: + queryset = DataType.objects.all() + return queryset + + +class PublicProjectsListAPIView(ListAPIView): + """ + Return list of DataTypes and source projects that have registered them. + """ + + serializer_class = ProjectDataSerializer + + def get_queryset(self): + """ + Get the queryset and filter on project if provided. + """ + qs = DataRequestProject.objects.filter(approved=True) + project_label = self.request.GET.get("id_label", None) + if project_label: + project = id_label_to_project(project_label) + qs = qs.filter(id=project.id) + proj_id = self.request.GET.get("id", None) + if proj_id: + qs = qs.filter(id=proj_id) + return qs diff --git a/open_humans/fixtures/test-data.json b/open_humans/fixtures/test-data.json index e96da8223..4dac3bac2 100644 --- a/open_humans/fixtures/test-data.json +++ b/open_humans/fixtures/test-data.json @@ -759,5 +759,65 @@ "updated": "2017-06-04T04:42:21.812Z", "access_token": 772 } + }, +{ + "model": "data_import.datatype", + "pk": 1, + "fields": { + "name": "survey", + "description": "survey", + "created": "2016-02-24T22:11:38.721Z", + "modified": "2016-02-24T22:11:38.721Z", + "parent_id": null + } +}, +{ + "model": "data_import.datatype", + "pk": 2, + "fields": { + "name": "all your base", + "description": "are belong to us", + "created": "2016-03-24T22:11:38.721Z", + "modified": "2016-03-24T22:11:38.721Z", + "parent_id": 1 + } +}, +{ + "model": "data_import.datatype", + "pk": 3, + "fields": { + "name": "genetics", + "description": "genetics", + "created": "2016-02-24T22:11:38.721Z", + "modified": "2016-02-24T22:11:38.721Z", + "parent_id": null + } +}, +{ + "model": "data_import.datatype", + "pk": 4, + "fields": { + "name": "fourth one", + "description": "because I decided to have four", + "created": "2016-02-24T22:11:38.721Z", + "modified": "2016-02-24T22:11:38.721Z", + "parent_id": null + } +}, +{ + "model": "private_sharing.datarequestproject_registered_datatypes", + "pk": 1, + "fields": { + "datarequestproject_id": 2, + "datatype_id": 1 + } +}, +{ + "model": "private_sharing.datarequestproject_registered_datatypes", + "pk": 2, + "fields": { + "datarequestproject_id": 2, + "datatype_id": 4 } +} ] diff --git a/open_humans/templates/pages/public-data-api.html b/open_humans/templates/pages/public-data-api.html index 318af4f4a..20f316731 100644 --- a/open_humans/templates/pages/public-data-api.html +++ b/open_humans/templates/pages/public-data-api.html @@ -1,6 +1,7 @@ {% extends 'base.html' %} {% load static %} +{% load utilities %} {% block head_title %}The Public Data API{% endblock %} @@ -34,8 +35,8 @@

Instructions

The base URL for the API is - https://www.openhumans.org/api/public-data/. + href="{% domain %}/api/public-data/"> + {% domain %}/api/public-data/.

@@ -74,8 +75,8 @@

Examples

- https://www.openhumans.org/api/public-data/?username=madprime&limit=5 + href="{% domain %}/api/public-data/?username=madprime&limit=5"> + {% domain %}/api/public-data/?username=madprime&limit=5

@@ -86,8 +87,8 @@

Examples

- https://www.openhumans.org/api/public-data/?created_start=2/14/2016&created_end=2/14/2016&source=direct-sharing-139 + href="{% domain %}/api/public-data/?created_start=2/14/2016&created_end=2/14/2016&source=direct-sharing-139"> + {% domain %}/api/public-data/?created_start=2/14/2016&created_end=2/14/2016&source=direct-sharing-139

@@ -99,12 +100,12 @@

Which users have which sources?

Query for a specific data type

@@ -128,6 +129,48 @@

Query for a specific data type

URL:

+ +

Information about projects

+

+ You can query the projects endpoint to get information about projects: +

+

+ + {% domain %}/api/public-data/projects/ + +

+

+ You can filter this list by a project's `id_label` or `id`: +

+ + +

DataTypes

+

+ We've introduced DataTypes as a method for managing data sharing permissions. + Projects that add data to Open Humans accounts should register DataTypes for the data + they may upload. You can see a list of all DataTypes here: +

+

+ + {% domain %}/api/public-data/datatypes/ + +

+

+ You can also filter on a specific project label to see what DataTypes it has registered. +

+

+ + {% domain %}/api/public-data/datatypes/?source_project_label=direct-sharing-176 + +

{% endblock %} diff --git a/open_humans/templatetags/utilities.py b/open_humans/templatetags/utilities.py index b85f286b0..faf1959d0 100644 --- a/open_humans/templatetags/utilities.py +++ b/open_humans/templatetags/utilities.py @@ -370,3 +370,11 @@ def get_download_url(context, data_file): Returns the download url """ return data_file.download_url(context.request) + + +@register.simple_tag() +def domain(): + """ + For setting the currently set domain in html templates. + """ + return settings.DEFAULT_HTTP_PROTOCOL + "://" + settings.DOMAIN diff --git a/open_humans/tests.py b/open_humans/tests.py index 9eb6cc4b4..d9492be3c 100644 --- a/open_humans/tests.py +++ b/open_humans/tests.py @@ -38,6 +38,8 @@ class SmokeTests(SmokeTestCase): "/api/public-data/?created_start=2/14/2016&created_end=2/14/2016", "/api/public-data/sources-by-member/", "/api/public-data/members-by-source/", + "/api/public-data/datatypes/", + "/api/public-data/projects/", "/beau/", "/community-guidelines/", "/contact-us/", diff --git a/private_sharing/api_views.py b/private_sharing/api_views.py index 12e71dc9b..c8dbad733 100644 --- a/private_sharing/api_views.py +++ b/private_sharing/api_views.py @@ -210,7 +210,10 @@ def post(self, request): if project_member: req_data["project_member_id"] = project_member.project_member_id - form = self.form_class(req_data, request.FILES) + try: + form = self.form_class(req_data, request.FILES, project=project) + except TypeError: + form = self.form_class(req_data, request.FILES) if not form.is_valid(): raise serializers.ValidationError(form.errors) @@ -308,18 +311,19 @@ class ProjectFileDirectUploadView(ProjectFormBaseView): form_class = DirectUploadDataFileForm def post(self, request): - super(ProjectFileDirectUploadView, self).post(request) + super().post(request) key = get_upload_path(self.project.id_label, self.form.cleaned_data["filename"]) - data_file = ProjectDataFile( + datafile = ProjectDataFile( user=self.project_member.member.user, file=key, metadata=self.form.cleaned_data["metadata"], direct_sharing_project=self.project, ) - data_file.save() + datafile.save() + datafile.datatypes.set(self.form.cleaned_data["datatypes"]) s3 = S3Connection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY) @@ -330,9 +334,7 @@ def post(self, request): key=key, ) - return Response( - {"id": data_file.id, "url": url}, status=status.HTTP_201_CREATED - ) + return Response({"id": datafile.id, "url": url}, status=status.HTTP_201_CREATED) class ProjectFileDirectUploadCompletionView(ProjectFormBaseView): @@ -343,11 +345,14 @@ class ProjectFileDirectUploadCompletionView(ProjectFormBaseView): form_class = DirectUploadDataFileCompletionForm def post(self, request): - super(ProjectFileDirectUploadCompletionView, self).post(request) - - data_file = ProjectDataFile.all_objects.get( - pk=self.form.cleaned_data["file_id"] - ) + super().post(request) + # If upload failed, file_id would be empty. Let's fail gracefully. + file_id = self.form.cleaned_data.get("file_id", None) + if not file_id: + return Response( + {"detail": "file does not exist"}, status=status.HTTP_400_BAD_REQUEST + ) + data_file = ProjectDataFile.all_objects.get(pk=file_id) data_file.completed = True data_file.save() @@ -374,17 +379,17 @@ class ProjectFileUploadView(ProjectFormBaseView): def post(self, request): super().post(request) - data_file = ProjectDataFile( + datafile = ProjectDataFile( user=self.project_member.member.user, file=self.form.cleaned_data["data_file"], metadata=self.form.cleaned_data["metadata"], direct_sharing_project=self.project, completed=True, ) + datafile.save() + datafile.datatypes.set(self.form.cleaned_data["datatypes"]) - data_file.save() - - return Response({"id": data_file.id}, status=status.HTTP_201_CREATED) + return Response({"id": datafile.id}, status=status.HTTP_201_CREATED) class ProjectFileDeleteView(ProjectFormBaseView): diff --git a/private_sharing/fixtures/test-data.json b/private_sharing/fixtures/test-data.json index 0b12a9186..df29061f5 100644 --- a/private_sharing/fixtures/test-data.json +++ b/private_sharing/fixtures/test-data.json @@ -168,5 +168,65 @@ "beau" ] } +}, +{ + "model": "data_import.datatype", + "pk": 1, + "fields": { + "name": "survey", + "description": "survey", + "created": "2016-02-24T22:11:38.721Z", + "modified": "2016-02-24T22:11:38.721Z", + "parent_id": null + } +}, +{ + "model": "data_import.datatype", + "pk": 2, + "fields": { + "name": "all your base", + "description": "are belong to us", + "created": "2016-03-24T22:11:38.721Z", + "modified": "2016-03-24T22:11:38.721Z", + "parent_id": 1 + } +}, +{ + "model": "data_import.datatype", + "pk": 3, + "fields": { + "name": "genetics", + "description": "genetics", + "created": "2016-02-24T22:11:38.721Z", + "modified": "2016-02-24T22:11:38.721Z", + "parent_id": null + } +}, +{ + "model": "data_import.datatype", + "pk": 4, + "fields": { + "name": "fourth one", + "description": "because I decided to have four", + "created": "2016-02-24T22:11:38.721Z", + "modified": "2016-02-24T22:11:38.721Z", + "parent_id": null + } +}, +{ + "model": "private_sharing.datarequestproject_registered_datatypes", + "pk": 1, + "fields": { + "datarequestproject_id": 3, + "datatype_id": 1 + } +}, +{ + "model": "private_sharing.datarequestproject_registered_datatypes", + "pk": 2, + "fields": { + "datarequestproject_id": 3, + "datatype_id": 4 + } } ] diff --git a/private_sharing/forms.py b/private_sharing/forms.py index df5d8073b..185ac2b15 100644 --- a/private_sharing/forms.py +++ b/private_sharing/forms.py @@ -6,6 +6,7 @@ import arrow from common import tasks +from data_import.models import DataType from .models import ( DataRequestProject, @@ -313,6 +314,79 @@ class UploadDataFileBaseForm(forms.Form): metadata = forms.CharField(label="Metadata", required=True) + # TODO: When updated to require datatypes - should set required to true. + datatypes = forms.CharField(label="Data type", required=False) + + def __init__(self, *args, **kwargs): + self.project = kwargs.pop("project") + super().__init__(*args, **kwargs) + + def _get_datatypes(self, dt_list, method): + """ + Get DataType objects from IDs or names, report if unmatched and unregistered. + """ + assert method in ["id", "name"] + datatypes, unregistered, unmatched = [], [], [] + registered_datatypes = self.project.registered_datatypes.all() + for item in dt_list: + try: + if method == "id": + dt = DataType.objects.get(id=int(item)) + elif method == "name": + dt = DataType.objects.get(name__iexact=item) + if dt in registered_datatypes: + datatypes.append(dt) + else: + unregistered.append(item) + except DataType.DoesNotExist: + unmatched.append(item) + return datatypes, unregistered, unmatched + + def clean_datatypes(self): + """ + Cleans and returns DataType objects specified by JSON list of IDs or names. + """ + # TODO: When updated to require datatypes - no longer handle missing field. + if not self.cleaned_data["datatypes"]: + if self.project.auto_add_datatypes: + return self.project.registered_datatypes.all() + else: + return [] + + # Load as JSON-fomatted list + try: + dt_set = set(json.loads(self.cleaned_data["datatypes"])) + except ValueError: + raise forms.ValidationError( + "Could not parse the uploaded datatypes from: '{}'".format( + self.cleaned_data["datatypes"] + ) + ) + + # Try loading as IDs first, then names. + try: + datatypes, notreg, notmatch = self._get_datatypes(dt_set, method="id") + except ValueError: + datatypes, notreg, notmatch = self._get_datatypes(dt_set, method="name") + + # Error fon unmatched datatypes. + if notmatch: + raise forms.ValidationError( + "The following datatypes don't match items in our database: {}".format( + ", ".join(notmatch) + ) + ) + + # Error for unregistered datatypes. + if notreg: + raise forms.ValidationError( + "The following datatypes aren't registered for this project: {}".format( + ", ".join(notreg) + ) + ) + + return datatypes + def clean_metadata(self): try: metadata = json.loads(self.cleaned_data["metadata"]) @@ -393,3 +467,18 @@ class DeleteDataFileForm(forms.Form): file_basename = forms.CharField(required=False, label="File basename") all_files = forms.BooleanField(required=False, label="All files") + + +class SelectDatatypesForm(forms.ModelForm): + """ + Select registered datatypes for a project. + """ + + class Meta: # noqa: D101 + model = DataRequestProject + fields = ["registered_datatypes"] + widgets = {"registered_datatypes": forms.CheckboxSelectMultiple} + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields["registered_datatypes"].required = False diff --git a/private_sharing/migrations/0020_auto_20190222_0036.py b/private_sharing/migrations/0020_auto_20190222_0036.py index 0cb4e6e3e..1be5f30cb 100644 --- a/private_sharing/migrations/0020_auto_20190222_0036.py +++ b/private_sharing/migrations/0020_auto_20190222_0036.py @@ -5,17 +5,13 @@ class Migration(migrations.Migration): - dependencies = [ - ('private_sharing', '0019_auto_20190214_1915'), - ] + dependencies = [("private_sharing", "0019_auto_20190214_1915")] operations = [ migrations.RemoveField( - model_name='datarequestproject', - name='request_sources_access', + model_name="datarequestproject", name="request_sources_access" ), migrations.RemoveField( - model_name='datarequestprojectmember', - name='sources_shared', + model_name="datarequestprojectmember", name="sources_shared" ), ] diff --git a/private_sharing/migrations/0021_auto_20190412_1908.py b/private_sharing/migrations/0021_auto_20190412_1908.py new file mode 100644 index 000000000..1d34271fd --- /dev/null +++ b/private_sharing/migrations/0021_auto_20190412_1908.py @@ -0,0 +1,29 @@ +# Generated by Django 2.2 on 2019-04-12 19:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("data_import", "0019_datatype"), + ("private_sharing", "0020_auto_20190222_0036"), + ] + + operations = [ + migrations.AddField( + model_name="datarequestproject", + name="auto_add_datatypes", + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name="datarequestproject", + name="registered_datatypes", + field=models.ManyToManyField(to="data_import.DataType"), + ), + migrations.AddField( + model_name="projectdatafile", + name="datatypes", + field=models.ManyToManyField(to="data_import.DataType"), + ), + ] diff --git a/private_sharing/models.py b/private_sharing/models.py index c245dd9e7..11ba734a6 100644 --- a/private_sharing/models.py +++ b/private_sharing/models.py @@ -23,7 +23,7 @@ from oauth2_provider.models import AccessToken, Application, RefreshToken from common.utils import app_label_to_verbose_name, generate_id -from data_import.models import DataFile +from data_import.models import DataFile, DataType from open_humans.models import Member from open_humans.storage import PublicStorage @@ -228,6 +228,7 @@ class DataRequestProject(models.Model): ), verbose_name="Are you requesting Open Humans usernames?", ) + registered_datatypes = models.ManyToManyField(DataType) class Meta: ordering = ["name"] @@ -247,6 +248,7 @@ class Meta: token_expiration_date = models.DateTimeField(default=now_plus_24_hours) token_expiration_disabled = models.BooleanField(default=False) no_public_data = models.BooleanField(default=False) + auto_add_datatypes = models.BooleanField(default=False) def __init__(self, *args, **kwargs): # Adds self.old_approved so that we can detect when the field changes @@ -646,7 +648,7 @@ class ProjectDataFile(DataFile): related_name="parent_project_data_file", on_delete=models.CASCADE, ) - + datatypes = models.ManyToManyField(DataType) completed = models.BooleanField(default=False) direct_sharing_project = models.ForeignKey( DataRequestProject, on_delete=models.CASCADE diff --git a/private_sharing/serializers.py b/private_sharing/serializers.py index 4b20c58d3..1348abbe6 100644 --- a/private_sharing/serializers.py +++ b/private_sharing/serializers.py @@ -3,7 +3,7 @@ from rest_framework import serializers from common.utils import full_url -from data_import.models import DataFile +from data_import.models import DataFile, DataType from data_import.serializers import DataFileSerializer from .models import DataRequestProject, DataRequestProjectMember diff --git a/private_sharing/templates/private_sharing/project-detail.html b/private_sharing/templates/private_sharing/project-detail.html index 942c86d0d..3e05afd28 100644 --- a/private_sharing/templates/private_sharing/project-detail.html +++ b/private_sharing/templates/private_sharing/project-detail.html @@ -23,6 +23,9 @@

Project details

Remove members + + Configure uploaded datatypes

@@ -119,6 +122,12 @@

"{{ object.name }}"

{% endfor %} {% endif %} +
Registered datatypes
+
+ {% for datatype in object.registered_datatypes.all %} + {{ datatype.name }}{% if not forloop.last %}, {% endif %} + {% endfor %} +
Username permission
{{ object.request_username_access }}
diff --git a/private_sharing/templates/private_sharing/select-datatypes.html b/private_sharing/templates/private_sharing/select-datatypes.html new file mode 100644 index 000000000..24e65bbaa --- /dev/null +++ b/private_sharing/templates/private_sharing/select-datatypes.html @@ -0,0 +1,49 @@ +{% extends 'panel.html' %} + +{% load bootstrap_tags %} +{% load static %} + +{% block head_title %}{{ object.name }}: Registered DataTypes{% endblock %} + +{% block panel_content %} +

+ Is your project potentially adding data to Open Humans? Please use this form + to register DataTypes. Files uploaded to member accounts must specify one + or more DataTypes registered here. +

+
+
+ {% csrf_token %} + + {% if form.non_field_errors %} +
+

+ {% for error in form.non_field_errors%} + Error: {{ error }} + {% endfor %} +

+
+ {% endif %} + {% for item in datatypes_sorted %} + {# checkbox for each datatype, lefthand padding according to datatype depth #} +
+ - {{ item.datatype.description }} +
+ {% endfor %} +
+ +
+
+
+Don't see the DataType you need? +Go to the DataTypes section +to explore, update, and create DataTypes. +{% endblock %} diff --git a/private_sharing/testing.py b/private_sharing/testing.py index 7e11f22cb..882d68fab 100644 --- a/private_sharing/testing.py +++ b/private_sharing/testing.py @@ -2,8 +2,11 @@ from unittest import skipIf from django.conf import settings +from django.db.models import Count from common.testing import SmokeTestCase +from data_import.models import DataType +from open_humans.models import Member from .models import DataRequestProjectMember, ProjectDataFile @@ -22,6 +25,24 @@ def setUp(): """ DataRequestProjectMember.objects.all().delete() + def insert_datatypes(self): + editor = Member.objects.get(user__username="chickens") + while DataType.objects.all(): + DataType.objects.annotate(num_children=Count("children")).filter( + num_children=0 + ).delete() + for dt in range(1, 5): + new_datatype = DataType(name=str(dt)) + new_datatype.editor = editor + new_datatype.save() + new_datatype = DataType(name="all your base") + new_datatype.editor = editor + new_datatype.save() + new_datatype = DataType(name="are belong to us") + new_datatype.editor = editor + new_datatype.save() + return DataType.objects.all() + def update_member(self, joined, authorized, revoked=False): # first delete the ProjectMember try: @@ -55,6 +76,14 @@ class DirectSharingTestsMixin(object): @skipIf((not settings.AWS_STORAGE_BUCKET_NAME), "AWS bucket not set up.") def test_file_upload(self): member = self.update_member(joined=True, authorized=True) + datatypes = self.insert_datatypes() + self.member1_project.registered_datatypes.clear() + self.member1_project.registered_datatypes.add( + datatypes.get(name="all your base") + ) + self.member1_project.registered_datatypes.add( + datatypes.get(name="are belong to us") + ) response = self.client.post( "/api/direct-sharing/project/files/upload/?access_token={}".format( @@ -62,6 +91,7 @@ def test_file_upload(self): ), data={ "project_member_id": member.project_member_id, + "datatypes": '["all your base", "are belong to us"]', "metadata": ( '{"description": "Test description...", ' '"tags": ["tag 1", "tag 2", "tag 3"]}' @@ -88,6 +118,58 @@ def test_file_upload(self): self.assertEqual(data_file.file.readlines(), [b"just testing..."]) + def test_file_upload_bad_datatypes(self): + member = self.update_member(joined=True, authorized=True) + datatypes = self.insert_datatypes() + self.member1_project.registered_datatypes.clear() + self.member1_project.registered_datatypes.add( + datatypes.get(name="all your base") + ) + + # Test unregistered datatype. + response = self.client.post( + "/api/direct-sharing/project/files/upload/?access_token={}".format( + self.member1_project.master_access_token + ), + data={ + "project_member_id": member.project_member_id, + "datatypes": '["are belong to us"]', + "metadata": ( + '{"description": "Test description...", ' + '"tags": ["tag 1", "tag 2", "tag 3"]}' + ), + "data_file": StringIO("just testing..."), + }, + ) + + response_json = response.json() + + self.assertIn("datatypes", response_json) + self.assertRegexpMatches(response_json["datatypes"][0], "aren't registered") + self.assertEqual(response.status_code, 400) + + # Test datatype not found. + response = self.client.post( + "/api/direct-sharing/project/files/upload/?access_token={}".format( + self.member1_project.master_access_token + ), + data={ + "project_member_id": member.project_member_id, + "datatypes": '["you have no chance"]', + "metadata": ( + '{"description": "Test description...", ' + '"tags": ["tag 1", "tag 2", "tag 3"]}' + ), + "data_file": StringIO("just testing..."), + }, + ) + + response_json = response.json() + + self.assertIn("datatypes", response_json) + self.assertRegexpMatches(response_json["datatypes"][0], "don't match", msg=None) + self.assertEqual(response.status_code, 400) + def test_file_upload_bad_metadata(self): member = self.update_member(joined=True, authorized=True) @@ -247,6 +329,14 @@ def test_file_delete_bad_request(self): @skipIf((not settings.AWS_STORAGE_BUCKET_NAME), "AWS bucket not set up.") def test_direct_upload(self): member = self.update_member(joined=True, authorized=True) + datatypes = self.insert_datatypes() + self.member1_project.registered_datatypes.clear() + self.member1_project.registered_datatypes.add( + datatypes.get(name="all your base") + ) + self.member1_project.registered_datatypes.add( + datatypes.get(name="are belong to us") + ) response = self.client.post( "/api/direct-sharing/project/files/upload/direct/?access_token={}".format( @@ -255,6 +345,7 @@ def test_direct_upload(self): data={ "project_member_id": member.project_member_id, "filename": "test-file.json", + "registered_datatypes": '["all your base", "are belong to us"]', "metadata": ( '{"description": "Test description...", ' '"tags": ["tag 1", "tag 2", "tag 3"]}' @@ -269,3 +360,4 @@ def test_direct_upload(self): self.assertIn("/member-files/direct-sharing-", json["url"]) self.assertEqual(response.status_code, 201) + self.member1_project.registered_datatypes.clear() diff --git a/private_sharing/tests.py b/private_sharing/tests.py index e96a274b5..029cbbb1e 100644 --- a/private_sharing/tests.py +++ b/private_sharing/tests.py @@ -13,6 +13,7 @@ from oauth2_provider.models import AccessToken from common.testing import BrowserTestCase, get_or_create_user, SmokeTestCase +from data_import.models import DataType from open_humans.models import Member from .models import ( @@ -332,6 +333,14 @@ def test_oauth2_authorize(self): @unittest.skipIf((not settings.AWS_STORAGE_BUCKET_NAME), "AWS not set up.") def test_member_access_token(self): member = self.update_member(joined=True, authorized=True) + datatypes = self.insert_datatypes() + self.member1_project.registered_datatypes.clear() + self.member1_project.registered_datatypes.add( + datatypes.get(name="all your base") + ) + self.member1_project.registered_datatypes.add( + datatypes.get(name="are belong to us") + ) response = self.client.post( "/api/direct-sharing/project/files/upload/?access_token={}".format( @@ -339,6 +348,7 @@ def test_member_access_token(self): ), data={ "project_member_id": member.project_member_id, + "datatypes": '["all your base", "are belong to us"]', "metadata": ( '{"description": "Test description...", ' '"tags": ["tag 1", "tag 2", "tag 3"]}' diff --git a/private_sharing/urls.py b/private_sharing/urls.py index cacc72632..1d75f6c94 100644 --- a/private_sharing/urls.py +++ b/private_sharing/urls.py @@ -16,6 +16,11 @@ views.CreateOnSiteDataRequestProjectView.as_view(), name="create-on-site", ), + re_path( + r"^projects/registered_datatypes/(?P[a-z0-9_-]+)/$", + views.SelectDatatypesView.as_view(), + name="select-datatypes", + ), re_path( r"^projects/on-site/join/(?P[a-z0-9_-]+)/$", views.JoinOnSiteDataRequestProjectView.as_view(), diff --git a/private_sharing/views.py b/private_sharing/views.py index 9057f9652..366075287 100644 --- a/private_sharing/views.py +++ b/private_sharing/views.py @@ -3,6 +3,7 @@ from django.core.exceptions import ObjectDoesNotExist from django.http import Http404, HttpResponseRedirect from django.urls import reverse, reverse_lazy +from django.utils.safestring import SafeString from django.views.generic import ( CreateView, DetailView, @@ -12,21 +13,24 @@ UpdateView, View, ) +from django.views.generic.detail import SingleObjectMixin from common.mixins import LargePanelMixin, PrivateMixin from common.views import BaseOAuth2AuthorizationView +from data_import.models import DataType # TODO: move this to common from open_humans.mixins import SourcesContextMixin -from private_sharing.models import ActivityFeed from .forms import ( MessageProjectMembersForm, OAuth2DataRequestProjectForm, OnSiteDataRequestProjectForm, RemoveProjectMembersForm, + SelectDatatypesForm, ) from .models import ( + ActivityFeed, DataRequestProject, DataRequestProjectMember, OAuth2DataRequestProject, @@ -676,3 +680,52 @@ def get_object(self, queryset=None): self.object = queryset.get(slug=slug) return self.object + + +class SelectDatatypesView(PrivateMixin, CoordinatorOnlyView, UpdateView): + """ + Select the datatypes for a project. + """ + + form_class = SelectDatatypesForm + model = DataRequestProject + success_url = reverse_lazy("direct-sharing:manage-projects") + template_name = "private_sharing/select-datatypes.html" + + def dispatch(self, *args, **kwargs): + """ + Override dispatch to redirect if project is approved + """ + self.object = self.get_object() + if self.object.approved: + django_messages.error( + self.request, + ( + "Sorry, {0} has been approved and the project's datatypes cannot be changed " + "without re-approval.".format(self.object.name) + ), + ) + + return HttpResponseRedirect( + reverse( + "direct-sharing:detail-{0}".format(self.object.type), + kwargs={"slug": self.object.slug}, + ) + ) + return super().dispatch(*args, **kwargs) + + def get_context_data(self, *args, **kwargs): + context = super().get_context_data(*args, **kwargs) + datatypes_sorted = DataType.sorted_by_ancestors() + try: + max_depth = max([i["depth"] for i in datatypes_sorted]) + except ValueError: + max_depth = 0 + context.update({"datatypes_sorted": datatypes_sorted, "max_depth": max_depth}) + return context + + def get_success_url(self): + return reverse_lazy( + "direct-sharing:detail-{0}".format(self.object.type), + args=[self.object.slug], + )