Priority fix (#914)

* done, untested

* Prepared for testing, untested

* Tested

Makefile

@@ -27,6 +27,7 @@ tests:
 	pytest -c compscicenter_ru/pytest.ini --ds=compscicenter_ru.settings.test
 	pytest -c compsciclub_ru/pytest.ini --ds=compsciclub_ru.settings.test
 	pytest -c lk_yandexdataschool_ru/pytest.ini --ds=lk_yandexdataschool_ru.settings.test
+	python manage.py clear_scheduled_jobs
 
 migrate:
 	python manage.py migrate $(DJANGO_POSTFIX)

apps/admission/reports.py

@@ -6,7 +6,7 @@
 from pandas import DataFrame
 
 from django.db.models import Prefetch
-from django.utils import formats
+from django.utils import formats, timezone
 from django.utils.encoding import force_str
 
 from admission.constants import ApplicantStatuses, InterviewSections
@@ -107,7 +107,7 @@ def get_queryset(self):
         return Applicant.objects.filter(campaign=self.campaign.pk).order_by("pk")
 
     def get_filename(self):
-        today = datetime.datetime.now()
+        today = timezone.now()
         return "admission_{}_{}_report_{}".format(
             self.campaign.branch.code,
             self.campaign.year,
@@ -149,7 +149,7 @@ def get_queryset(self):
         return Applicant.objects.filter(campaign__year=self.year).exclude(campaign__branch__name='Тест').order_by("pk")
 
     def get_filename(self):
-        today = datetime.datetime.now()
+        today = timezone.now()
         return f"admission_{self.year}_report_{formats.date_format(today, 'SHORT_DATE_FORMAT')}"
 
 class AdmissionExamReport:
@@ -210,7 +210,7 @@ def get_queryset(self):
         )
 
     def get_filename(self):
-        today = datetime.datetime.now()
+        today = timezone.now()
         return "admission_{}_{}_exam_report_{}".format(
             self.campaign.year,
             self.campaign.branch.code,

apps/core/management/commands/clear_scheduled_jobs.py

@@ -0,0 +1,14 @@
+import django_rq
+
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument("-q", "--queue", type=str, default="default")
+
+    def handle(self, *args, **options):
+        queue = django_rq.get_queue(options.get("queue"))
+        registry = queue.scheduled_job_registry
+        for job_id in registry.get_job_ids():
+            registry.remove(job_id, delete_job=True)

apps/courses/models.py

@@ -151,6 +151,19 @@ def get_current(cls, tz: Timezone = settings.DEFAULT_TIMEZONE):
                                                  type=term_pair.type)
         return obj
 
+    def get_next(self) -> "Semester":
+        term_pair = self.term_pair.get_next()
+        obj, created = self.__class__.objects.get_or_create(year=term_pair.year,
+                                                 type=term_pair.type)
+        return obj
+
+    def get_prev(self) -> "Semester":
+        term_pair = self.term_pair.get_prev()
+        obj, created = self.__class__.objects.get_or_create(year=term_pair.year,
+                                                 type=term_pair.type)
+        return obj
+
+    @property
     def is_current(self, tz: Timezone = settings.DEFAULT_TIMEZONE):
         term_pair = get_current_term_pair(tz)
         return term_pair.year == self.year and term_pair.type == self.type

apps/courses/signals.py

@@ -1,9 +1,13 @@
+import datetime
 from django.conf import settings
 from django.db.models.signals import post_save
 from django.dispatch import receiver
+from django_rq.queues import get_queue
 
 from courses.models import Assignment, CourseTeacher, Semester
+from courses.tasks import recalculate_invited_priority
 from learning.models import EnrollmentPeriod
+from django.utils import timezone
 
 
 @receiver(post_save, sender=Semester)
@@ -16,3 +20,24 @@ def create_enrollment_period_for_compsciclub_ru(sender, instance: Semester,
     EnrollmentPeriod.objects.get_or_create(site_id=settings.CLUB_SITE_ID,
                                            semester=instance,
                                            defaults={"ends_on": ends_on})
+
+@receiver(post_save, sender=Semester)
+def schedule_invited_stundent_priority_recalculation(sender, instance: Semester,
+                                                created, *args, **kwargs):
+    """Schedule job that recalculates priority of every invited student profiles of this semester"""
+    queue = get_queue('default')
+    recalculation_day = instance.ends_at.date() + datetime.timedelta(days=1)
+    # Tests trigger job scheduling. Scheduled ones are deleted after using clear_scheduled_jobs command
+    # This is needed to prevent jobs to be queued without scheduling while the tests are running
+    if recalculation_day <= timezone.now().date():
+        return
+    scheduled_registry = queue.scheduled_job_registry
+    for job_id in scheduled_registry.get_job_ids():
+        job = queue.fetch_job(job_id)
+        if job and job.kwargs.get('semester_id', None) == instance.id:
+            return
+    job = queue.enqueue_at(
+        datetime.datetime(recalculation_day.year, recalculation_day.month, recalculation_day.day),
+        recalculate_invited_priority,
+        semester_id=instance.id
+    )

apps/courses/tasks.py

@@ -1,3 +1,4 @@
+import logging
 import posixpath
 
 import webdav3.client as wc
@@ -6,8 +7,12 @@
 from django.apps import apps
 from django.conf import settings
 
+from courses.models import Semester
+from users.models import StudentProfile, StudentTypes
+
 from .slides import upload_file
 
+logger = logging.getLogger(__name__)
 
 @job('default')
 def maybe_upload_slides_yandex(class_pk):
@@ -29,3 +34,20 @@ def maybe_upload_slides_yandex(class_pk):
     client = wc.Client(options)
     upload_file(webdav_client=client, local_path=instance.slides.file.name,
                 remote_path=remote_path)
+
+@job('default')
+def recalculate_invited_priority(semester_id = None):
+    try:
+        semester = Semester.objects.get(id=semester_id)
+        profiles = StudentProfile.objects.filter(type=StudentTypes.INVITED,
+                                                 invitation__semester=semester)
+    except Semester.DoesNotExist:
+        current_semester = Semester.get_current()
+        previos_semester = current_semester.get_prev()
+        preprevios_semester = previos_semester.get_prev()
+        logger.warning(f"Semester with ID {semester_id} is not found. Updating 3 last semesters: "
+                       f"{preprevios_semester}, {previos_semester} and {current_semester}")
+        profiles = StudentProfile.objects.filter(type=StudentTypes.INVITED,
+                                                 invitation__semester__in=[preprevios_semester, previos_semester, current_semester])
+    for profile in profiles:
+        profile.save()

apps/courses/tests/test_tasks.py

@@ -0,0 +1,76 @@
+from unittest import mock
+from django.conf import settings
+import pytest
+
+from courses.models import Semester
+from courses.tasks import recalculate_invited_priority
+from courses.tests.factories import SemesterFactory
+from learning.tests.factories import InvitationFactory
+from users.tests.factories import InvitedStudentFactory
+
+
+@pytest.mark.django_db
+def test_recalculate_invited_priority(mocker):
+    """Email has been generated after registration in yandex contest"""
+    current_semester = SemesterFactory.create_current()
+    previos_semester = SemesterFactory.create_prev(current_semester)
+    preprevios_semester = SemesterFactory.create_prev(previos_semester)
+    current_invitation = InvitationFactory(semester=current_semester)
+    previos_invitation = InvitationFactory(semester=previos_semester)
+    preprevios_invitation = InvitationFactory(semester=preprevios_semester)
+    assert Semester.get_current() == current_semester
+    previos_user = InvitedStudentFactory()
+    previos_profile = previos_user.get_student_profile()
+    assert previos_profile.priority == 1000
+    with mock.patch('django.utils.timezone.now', return_value=previos_semester.term_pair.starts_at(settings.DEFAULT_TIMEZONE)):
+        assert Semester.get_current() == previos_semester
+        previos_profile.invitation = preprevios_invitation
+        previos_profile.save()
+        assert previos_profile.priority == 1300
+        previos_profile.invitation = previos_invitation
+        previos_profile.save()
+        assert previos_profile.priority == 1000
+
+    assert previos_profile.priority == 1000
+    recalculate_invited_priority(preprevios_semester.id)
+    previos_profile.refresh_from_db()
+    assert previos_profile.priority == 1000
+    recalculate_invited_priority(previos_semester.id)
+    previos_profile.refresh_from_db()
+    assert previos_profile.priority == 1300
+
+    preprevios_user = InvitedStudentFactory()
+    preprevios_profile = preprevios_user.get_student_profile()
+    preprevios_profile.invitation = preprevios_invitation
+    preprevios_profile.save()
+    assert preprevios_profile.priority == 1300
+
+    with mock.patch('django.utils.timezone.now', return_value=previos_semester.term_pair.starts_at(settings.DEFAULT_TIMEZONE)):
+        recalculate_invited_priority(previos_semester.id)
+        previos_profile.refresh_from_db()
+        assert previos_profile.priority == 1000
+
+    current_user = InvitedStudentFactory()
+    current_profile = current_user.get_student_profile()
+    assert current_profile.priority == 1000
+    current_profile.invitation = current_invitation
+    current_profile.save()
+    assert current_profile.priority == 1000
+
+    with mock.patch('django.utils.timezone.now', return_value=preprevios_semester.term_pair.starts_at(settings.DEFAULT_TIMEZONE)):
+        recalculate_invited_priority(preprevios_semester.id)
+        preprevios_profile.refresh_from_db()
+        assert preprevios_profile.priority == 1000
+        recalculate_invited_priority(current_semester.id)
+        current_profile.refresh_from_db()
+        assert current_profile.priority == 1300
+
+    recalculate_invited_priority()
+    previos_profile.refresh_from_db()
+    preprevios_profile.refresh_from_db()
+    current_profile.refresh_from_db()
+    assert previos_profile.priority == 1300
+    assert preprevios_profile.priority == 1300
+    assert current_profile.priority == 1000
+
+

apps/grading/tasks.py

@@ -11,6 +11,7 @@
     ContestAPIError, SubmissionVerdict, Unavailable, YandexContestAPI
 )
 from grading.constants import CheckingSystemTypes
+from grading.models import Submission
 from grading.utils import YandexContestScoreSource
 
 logger = logging.getLogger(__name__)

apps/users/models.py

@@ -1010,15 +1010,6 @@ def get_comment_changed_at_display(self, default=''):
             return self.comment_changed_at.strftime(DATETIME_FORMAT_RU)
         return default
 
-    @property
-    def is_invited_student_active(self):
-        """
-        Checks if INVITED StudentProfile has invitation and it is relevant
-        """
-        if self.type != StudentTypes.INVITED:
-            raise ValueError("Works only with invited students. Use is_active for others")
-        return self.invitation is not None and self.invitation.is_active
-
 
 class StudentFieldLog(TimestampedModel):
     changed_at = models.DateField(

apps/users/services.py

@@ -204,7 +204,7 @@ def get_student_profile_priority(student_profile: StudentProfile) -> int:
         priority = min_priority + 100
     elif student_profile.status in StudentStatuses.inactive_statuses:
         priority = min_priority + 200
-    if student_profile.type == StudentTypes.INVITED and not student_profile.is_invited_student_active:
+    if student_profile.invitation is not None and not student_profile.invitation.semester.is_current:
         priority = min_priority + 300
     return priority
 

apps/users/tests/test_models.py

@@ -191,44 +191,22 @@ def test_student_profile_year_of_admission():
 @pytest.mark.django_db
 def test_student_profile_is_invited_student_active():
     student_profile = StudentProfileFactory()
-    with pytest.raises(ValueError):
-        student_profile.is_invited_student_active()
+    assert student_profile.priority == 200
 
     student_profile.type = StudentTypes.INVITED
-    assert not student_profile.is_invited_student_active
+    student_profile.save()
+    assert student_profile.priority == 1000
 
     course = CourseFactory(semester=Semester.get_current())
     invitation = InvitationFactory()
     invitation.courses.add(course)
     student_profile.invitation = invitation
-    assert not student_profile.is_invited_student_active
+    student_profile.save()
+    assert student_profile.priority == 1300
 
     invitation.semester = Semester.get_current()
-    assert not student_profile.is_invited_student_active
-
-    today = now_local(student_profile.branch.get_timezone()).date()
-    enrollmentperiod = EnrollmentPeriodFactory()
-    assert not student_profile.is_invited_student_active
-
-    enrollmentperiod.semester = Semester.get_current()
-    enrollmentperiod.save()
-    assert not student_profile.is_invited_student_active
-
-    enrollmentperiod.site_id = settings.SITE_ID
-    enrollmentperiod.save()
-    assert not student_profile.is_invited_student_active
-
-    enrollmentperiod.starts_on = today
-    enrollmentperiod.save()
-    assert not student_profile.is_invited_student_active
-
-    enrollmentperiod.ends_on = today
-    enrollmentperiod.save()
-    assert student_profile.is_invited_student_active
-
-    course.ends_on = today - datetime.timedelta(days=1)
-    course.save()
-    assert not student_profile.is_invited_student_active
+    student_profile.save()
+    assert student_profile.priority == 1000
 
 
 def test_get_abbreviated_short_name():

apps/users/tests/test_services.py

@@ -221,13 +221,9 @@ def test_get_student_profile_priority():
     student_profile1 = StudentProfileFactory(type=StudentTypes.REGULAR,
                                              status=StudentStatuses.REINSTATED)
     current_semester = Semester.get_current()
+    previos_semester = current_semester.get_prev()
     invitation = InvitationFactory(semester=current_semester)
-    CourseInvitationFactory(invitation=invitation, course__semester=current_semester)
-    today = now_local(student_profile1.branch.get_timezone()).date()
-    EnrollmentPeriodFactory(semester=current_semester,
-                            site_id=settings.SITE_ID,
-                            starts_on=today,
-                            ends_on=today)
+    invitation2 = InvitationFactory(semester=previos_semester)
     student_profile2 = StudentProfileFactory(type=StudentTypes.INVITED,
                                              status=StudentStatuses.REINSTATED,
                                              invitation=invitation)
@@ -244,7 +240,8 @@ def test_get_student_profile_priority():
     student_profile6 = StudentProfileFactory(type=StudentTypes.VOLUNTEER,
                                              status=StudentStatuses.EXPELLED)
     assert get_student_profile_priority(student_profile5) == get_student_profile_priority(student_profile6)
-    student_profile7 = StudentProfileFactory(type=StudentTypes.INVITED)
+    student_profile7 = StudentProfileFactory(type=StudentTypes.INVITED,
+                                             invitation=invitation2)
     assert get_student_profile_priority(student_profile3) < get_student_profile_priority(student_profile7)
     assert get_student_profile_priority(student_profile4) < get_student_profile_priority(student_profile7)
     assert get_student_profile_priority(student_profile5) < get_student_profile_priority(student_profile7)

lk_yandexdataschool_ru/k8s/templates/queue/deployment.yaml

@@ -41,7 +41,7 @@ spec:
             - mountPath: /etc/ldap/certs/
               name: ldap-certs
           command: ["/bin/sh"]
-          args: ["-c", "python manage.py rqworker high default"]
+          args: ["-c", "python manage.py rqworker high default --with-scheduler"]
           resources:
             requests:
               cpu: 100m

lms/tests.py

@@ -178,8 +178,8 @@ def test_view_course_offerings_invited_restriction(client):
         future = now() + datetime.timedelta(days=3)
         autumn_term = SemesterFactory.create_current(enrollment_period__ends_on=future.date())
         site = SiteFactory(id=settings.SITE_ID)
-        course_invitation = CourseInvitationFactory(course__semester=autumn_term)
-        student_profile = StudentProfileFactory(type=StudentTypes.INVITED)
+        course_invitation = CourseInvitationFactory(course__semester=autumn_term, invitation__semester=autumn_term)
+        student_profile = StudentProfileFactory(type=StudentTypes.INVITED, invitation=course_invitation.invitation)
         student = student_profile.user
         student.branch = student_profile.branch
         student.save()