Test adding and removing annotator from projects part way & check utility for repeat annotation

backend/management/commands/check_util.py

@@ -0,0 +1,63 @@
+from django.core.management.base import BaseCommand, CommandError
+from django.db.models import Count, Q
+
+from backend.models import Project, Annotation, Document
+
+
+def check_repeat_annotation(cmd_object: BaseCommand):
+    """
+    Check the database to highlight any documents that has more than one annotation by a single annotator. This
+    only includes annotations with status COMPLETED,PENDING and REJECTED. ABORTED and TIMED out annotations are ignored.
+    """
+
+    completed_pending_rejected_filter = Q(status=Annotation.COMPLETED) | Q(status=Annotation.PENDING) | Q(status=Annotation.REJECTED)
+    repeat_count = Count("annotations")
+    docs = Document.objects.annotate(num_annotations=repeat_count).filter(num_annotations__gt=1)
+    docs_count = 0
+    repeat_count = 0
+    for doc in docs:
+        annotator_id_set = set()
+        has_repeat = False
+        for annotation in doc.annotations.filter(completed_pending_rejected_filter):
+            if annotation.user.pk not in annotator_id_set:
+                annotator_id_set.add(annotation.user.pk)
+            else:
+                has_repeat = True
+
+        if has_repeat:
+            repeat_count += 1
+            cmd_object.stdout.write(f"Document {doc.pk} has multiple annotations from the same annotator (ID {annotation.user.pk})")
+            for annotation in doc.annotations.all():
+                print(
+                    f"\tAnnotation ID {annotation.pk} Annotator ID {annotation.user.pk}, status: {annotation.status} data: {annotation.data}")
+
+        docs_count += 1
+
+    cmd_object.stdout.write(f"Check completed\n\tNumber of documents checked: {docs_count}.\tNumber of documents with repeats {repeat_count}.")
+
+
+
+class Command(BaseCommand):
+
+    help = "Utility for performing various checks for diagnosing issues"
+
+    def add_arguments(self, parser):
+        parser.add_argument("check_type", type=str, help="Type of check to perform: repeat_annotation - Check for documents that have repeated annotations")
+
+
+    def handle(self, *args, **options):
+        if "check_type" in options:
+            if "repeat_annotation" == options["check_type"]:
+                check_repeat_annotation(self)
+                return
+            else:
+                raise CommandError(f"Unknown check type: {options['check_type']}")
+
+
+
+
+
+
+
+
+

backend/models.py

@@ -408,10 +408,10 @@ def get_annotator_document_score(self, user, doc_type):
 
         for document in test_docs:
             # Checks answers for all test documents
-            user_annotations = document.annotations.filter(user_id=user.pk)
+            user_annotations = document.annotations.filter(user_id=user.pk, status=Annotation.COMPLETED)
             if user_annotations.count() > 1:
                 # User should not have more than 1 annotation per document
-                raise Exception(f"User {user.username} has more than 1 annotation in document")
+                raise Exception(f"User {user.username} has more than 1 completed annotation in document")
 
             annotation = user_annotations.first()
 

backend/tests/test_models.py

@@ -678,7 +678,7 @@ def test_get_annotator_document_score(self):
 
         # All incorrect
         for doc in self.test_docs:
-            anno = Annotation.objects.create(user=annotator, document=doc)
+            anno = Annotation.objects.create(user=annotator, document=doc, status=Annotation.COMPLETED)
             anno.data = incorrect_data
 
         self.assertEqual(0, self.project.get_annotator_document_score(annotator, DocumentType.TEST))
@@ -690,7 +690,7 @@ def test_get_annotator_document_score(self):
                 "label1": doc.data["gold"]["label1"]["value"],
                 "label2": doc.data["gold"]["label2"]["value"],
             }
-            anno = Annotation.objects.create(user=annotator2, document=doc)
+            anno = Annotation.objects.create(user=annotator2, document=doc, status=Annotation.COMPLETED)
             anno.data = correct_annotation_data
 
         self.assertEqual(self.num_test_docs, self.project.get_annotator_document_score(annotator2, DocumentType.TEST))
@@ -705,7 +705,7 @@ def test_get_annotator_document_score(self):
                 "label2": doc.data["gold"]["label2"]["value"],
             }
             data = correct_annotation_data if counter < num_correct else incorrect_data
-            anno = Annotation.objects.create(user=annotator3, document=doc)
+            anno = Annotation.objects.create(user=annotator3, document=doc, status=Annotation.COMPLETED)
             anno.data = data
             counter += 1
 

backend/tests/test_rpc_endpoints.py

@@ -20,7 +20,7 @@
     get_project_training_documents, get_project_test_documents, project_annotator_allow_annotation, \
     annotator_leave_project, login, change_annotation, delete_annotation_change_history, get_annotation_task_with_id, \
     set_user_document_format_preference, initialise, is_authenticated, user_delete_personal_information, \
-    user_delete_account, admin_delete_user_personal_information, admin_delete_user
+    user_delete_account, admin_delete_user_personal_information, admin_delete_user, reject_project_annotator
 from backend.rpcserver import rpc_method
 from backend.errors import AuthError
 from backend.tests.test_models import create_each_annotation_status_for_user, TestUserModelDeleteUser
@@ -1761,6 +1761,107 @@ def test_delete_annotation_change_history(self):
 
 
 
+class TestMovingUsersToDifferentProjects(TestEndpoint):
+
+    def setUp(self) -> None:
+
+        # Create initial projects (x5) with documents (x10)
+        self.projects = []
+        for i in range(5):
+            project = Project.objects.create(name="Test1", owner=self.get_default_user())
+            self.projects.append(project)
+            for j in range(10):
+                Document.objects.create(project=project)
+
+        # Create annotator
+        self.annotator = get_user_model().objects.create(username="annotator")
+
+    def test_moving_annotator_to_different_project(self):
+        projects = self.projects
+        annotator = self.annotator
+
+        annotator_request = self.get_request()
+        annotator_request.user = annotator
+        add_project_annotator(self.get_loggedin_request(), projects[0].pk, annotator.username)
+
+        # Check annotator has been added to 0th project
+        project_annotators = get_project_annotators(self.get_loggedin_request(), projects[0].pk)
+        self.assertTrue(project_annotators[0]["username"] == annotator.username)
+
+        # Make two annotations
+        self.get_and_complete_annotation_task(annotator_request, projects[0])
+        self.get_and_complete_annotation_task(annotator_request, projects[0])
+
+        # Make a pending annotation
+        annotation_task = get_annotation_task(annotator_request)
+
+        # Remove annotator from 0th project, add to 1st project
+        remove_project_annotator(self.get_loggedin_request(), projects[0].pk, annotator.username)
+        add_project_annotator(self.get_loggedin_request(), projects[1].pk, annotator.username)
+
+        # Make two more annotations
+        self.get_and_complete_annotation_task(annotator_request, projects[1])
+        self.get_and_complete_annotation_task(annotator_request, projects[1])
+
+        # Remove annotator from 1st project, add to 2nd project
+        remove_project_annotator(self.get_loggedin_request(), projects[1].pk, annotator.username)
+        add_project_annotator(self.get_loggedin_request(), projects[2].pk, annotator.username)
+
+        # Make two more annotations
+        self.get_and_complete_annotation_task(annotator_request, projects[2])
+        self.get_and_complete_annotation_task(annotator_request, projects[2])
+
+        # Check no. of associated annotations
+        self.assertEqual(7, Annotation.objects.filter(user=annotator).count(), "Annotator should have 7 associated annotations")
+
+    def get_and_complete_annotation_task(self, annotator_request, project):
+        annotation_task = get_annotation_task(annotator_request)
+        self.assertEqual(project.pk, annotation_task["project_id"], f"Id of the project should be {project.pk}")
+        complete_annotation_task(annotator_request, annotation_task["annotation_id"], {"test": "result"})
+
+
+    def test_moving_rejected_annotator_to_different_project(self):
+        projects = self.projects
+        annotator = self.annotator
+
+        annotator_request = self.get_request()
+        annotator_request.user = annotator
+        add_project_annotator(self.get_loggedin_request(), projects[0].pk, annotator.username)
+
+        # Check annotator has been added to 0th project
+        project_annotators = get_project_annotators(self.get_loggedin_request(), projects[0].pk)
+        self.assertTrue(project_annotators[0]["username"] == annotator.username)
+
+        # Make two annotations
+        self.get_and_complete_annotation_task(annotator_request, projects[0])
+        self.get_and_complete_annotation_task(annotator_request, projects[0])
+
+        # Make a pending annotation
+        annotation_task = get_annotation_task(annotator_request)
+
+        # Remove annotator from 0th project, add to 1st project
+        reject_project_annotator(self.get_loggedin_request(), projects[0].pk, annotator.username)
+        add_project_annotator(self.get_loggedin_request(), projects[1].pk, annotator.username)
+
+        # Make two more annotations
+        self.get_and_complete_annotation_task(annotator_request, projects[1])
+        self.get_and_complete_annotation_task(annotator_request, projects[1])
+
+        # Remove annotator from 1st project, add to 2nd project
+        reject_project_annotator(self.get_loggedin_request(), projects[1].pk, annotator.username)
+        add_project_annotator(self.get_loggedin_request(), projects[2].pk, annotator.username)
+
+        # Make two more annotations
+        self.get_and_complete_annotation_task(annotator_request, projects[2])
+        self.get_and_complete_annotation_task(annotator_request, projects[2])
+
+        # Check no. of associated annotations
+        self.assertEqual(7, Annotation.objects.filter(user=annotator).count(),
+                         "Annotator should have 7 associated annotations")
+
+
+
+