Yd dump cron (#919)

* cron done

* add yadisk lib

* update cron

* Halfway here

* Done

* update pipfile

Pipfile

@@ -89,6 +89,7 @@ cryptography = "*"
 # API
 djangorestframework = "==3.13.1"
 rest-pandas = "==1.1.0"
+yadisk = "==3.1.0"
 # Misc
 django-admin-env-notice = "==1.0"  # https://github.com/dizballanze/django-admin-env-notice
 python-json-logger = "*"

apps/admission/locale/ru/LC_MESSAGES/django.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-11-30 10:17+0000\n"
+"POT-Creation-Date: 2024-12-12 14:45+0000\n"
 "PO-Revision-Date: 2024-07-23 17:04+0000\n"
 "Last-Translator: Дмитрий Чернушевич <[email protected]>\n"
 "Language-Team: LANGUAGE <[email protected]>\n"

apps/api/admin.py

@@ -3,7 +3,7 @@
 
 from api.services import TokenService
 
-from .models import Token
+from .models import ExternalServiceToken, Token
 
 
 @admin.register(Token)
@@ -24,3 +24,12 @@ def save_model(self, request, obj, form, change):
             msg = _("New secret token %s has been created. Save it somewhere "
                     "since you see it here for the last time.")
             messages.add_message(request, messages.WARNING, msg % secret_token)
+
+@admin.register(ExternalServiceToken)
+class ExternalServiceTokenAdmin(admin.ModelAdmin):
+    list_display = ('service_tag', 'created')
+    fields = ('service_tag', 'access_key')
+    ordering = ('-created',)
+
+    def get_readonly_fields(self, request, obj=None):
+        return ['service_tag'] if obj else []

apps/api/management/commands/dump_to_yandex_disk.py

@@ -0,0 +1,60 @@
+import csv
+import datetime
+import io
+import yadisk
+from django.core.management import BaseCommand
+from django.db.models import Max
+from django.utils.translation import gettext_lazy as _
+
+from courses.models import Assignment
+from learning.models import StudentAssignment
+from users.models import StudentProfile, StudentTypes
+from api.models import ExternalServiceToken
+
+def get_max_assignment_grade(assignment: Assignment):
+    assert isinstance(assignment, Assignment), f"Assignment object expected, {type(assignment)} object found"
+    return assignment.studentassignment_set.all().aggregate(Max('score'))['score__max']
+
+
+class Command(BaseCommand):
+    help = "Dump enrollments csv and upload to yandex disk"
+
+    def handle(self, *args, **options):
+        with io.StringIO() as csv_file:
+            csv_writer = csv.writer(csv_file)
+            csv_writer.writerow([_('Student ID'), _('Curriculum year'), _('Semester'), _('Course'), _('Branch'), _('Student type'), 
+                                 _('Student Group'), _('Teacher'), _('Assignment'), _('Assignment status'), _('Assignment Grade'), 
+                                 _('Maximum score'), _('Assignment count'), _('Maximum student score'), _('Grade'), _('Grade re-credited')])
+
+            current_year = datetime.datetime.now().year
+            student_profiles = (StudentProfile.objects.filter(type__in=[StudentTypes.REGULAR, StudentTypes.PARTNER],
+                                                             year_of_curriculum__in=[current_year - 1, current_year])
+                                                        .select_related('user')
+                                                        .prefetch_related('enrollment_set__course__assignment_set__studentassignment_set'))
+            max_assignment_grades: dict[Assignment, int] = dict()
+            for student_profile in student_profiles:
+                user = student_profile.user
+                for enrollment in student_profile.enrollment_set.all():
+                    course = enrollment.course
+                    assignments = course.assignment_set.all()
+                    for assignment in assignments:
+                        try:
+                            student_assignment = StudentAssignment.objects.get(assignment=assignment, student=user)
+                            if assignment not in max_assignment_grades:
+                                max_assignment_grades[assignment] = get_max_assignment_grade(assignment)
+                            max_assignment_grade = max_assignment_grades[assignment]
+                            teacher = student_assignment.assignee.teacher if student_assignment.assignee is not None else ""
+                            csv_writer.writerow([user.id, student_profile.year_of_curriculum, course.semester, course.meta_course, student_profile.branch.name, student_profile.get_type_display(), 
+                                        enrollment.student_group, teacher, assignment.title, student_assignment.get_status_display(), student_assignment.score, 
+                                        assignment.maximum_score, len(assignments), max_assignment_grade, enrollment.grade_honest, enrollment.is_grade_recredited])
+                        except StudentAssignment.DoesNotExist:
+                            # No student assignment for assignment {assignment} in course {course} and user {user}
+                            pass
+
+            csv_file.seek(0)
+            client = yadisk.Client(token=ExternalServiceToken.objects.get(service_tag="syrop_yandex_disk").access_key)
+            with client:
+                if not client.check_token():
+                    raise AssertionError("Token seems to ne invalid. Is it expired?")
+                client.upload(io.BytesIO(csv_file.getvalue().encode()), 
+                              f"/ysda_weekly_dump/dump_{datetime.datetime.now().strftime('%d_%m_%Y')}.csv")

apps/api/migrations/0002_externalservicetoken.py

@@ -0,0 +1,29 @@
+# Generated by Django 3.2.18 on 2024-12-10 13:55
+
+from django.db import migrations, models
+import django.utils.timezone
+import model_utils.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ExternalServiceToken',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')),
+                ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')),
+                ('service_tag', models.CharField(max_length=100, verbose_name='External service tag')),
+                ('access_key', models.CharField(db_index=True, help_text='Plain external service token', max_length=100, verbose_name='External token')),
+            ],
+            options={
+                'verbose_name': 'External token',
+                'verbose_name_plural': 'External tokens',
+            },
+        ),
+    ]

apps/api/models.py

@@ -34,3 +34,21 @@ class Meta:
 
     def __str__(self):
         return '%s : %s' % (self.access_key, self.user)
+
+class ExternalServiceToken(TimeStampedModel):
+    """
+    This model stores plain external service token
+    """
+    service_tag = models.CharField(_("External service tag"), max_length=100)
+    access_key = models.CharField(
+        verbose_name=_("External token"),
+        max_length=100,
+        db_index=True,
+        help_text=_("Plain external service token"))
+
+    class Meta:
+        verbose_name = _("External token")
+        verbose_name_plural = _("External tokens")
+
+    def __str__(self):
+        return f"External token for {self.service_tag}"

apps/htmlpages/locale/ru/LC_MESSAGES/django.po

@@ -9,7 +9,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: django\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-11-30 10:17+0000\n"
+"POT-Creation-Date: 2024-12-12 14:45+0000\n"
 "PO-Revision-Date: 2015-03-18 08:34+0000\n"
 "Last-Translator: Jannis Leidel <[email protected]>\n"
 "Language-Team: Russian (http://www.transifex.com/projects/p/django/language/"

apps/projects/locale/ru/LC_MESSAGES/django.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-11-30 10:17+0000\n"
+"POT-Creation-Date: 2024-12-12 14:45+0000\n"
 "PO-Revision-Date: 2022-02-21 15:24+0000\n"
 "Last-Translator: Сергей Жеревчук <[email protected]>\n"
 "Language-Team: LANGUAGE <[email protected]>\n"

apps/surveys/locale/ru/LC_MESSAGES/django.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-11-30 10:17+0000\n"
+"POT-Creation-Date: 2024-12-12 14:45+0000\n"
 "PO-Revision-Date: 2019-10-31 16:30+0000\n"
 "Last-Translator: b'  <[email protected]>'\n"
 "Language-Team: LANGUAGE <[email protected]>\n"

compscicenter_ru/locale/ru/LC_MESSAGES/django.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-11-30 10:17+0000\n"
+"POT-Creation-Date: 2024-12-12 14:45+0000\n"
 "PO-Revision-Date: 2020-02-03 16:52+0000\n"
 "Last-Translator: b'  <[email protected]>'\n"
 "Language-Team: LANGUAGE <[email protected]>\n"

compsciclub_ru/locale/ru/LC_MESSAGES/django.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-11-30 10:17+0000\n"
+"POT-Creation-Date: 2024-12-12 14:45+0000\n"
 "PO-Revision-Date: 2020-09-09 04:43+0000\n"
 "Last-Translator: b'  <[email protected]>'\n"
 "Language-Team: LANGUAGE <[email protected]>\n"

lk_yandexdataschool_ru/k8s/templates/periodic-tasks/dump_to_yandex_disk.yaml

@@ -0,0 +1,28 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: dump_to_yandex_disk
+  namespace: "{{ k8s_namespace}}"
+spec:
+  # https://crontab.guru/#0_10_*_*_0
+  schedule: "0 10 * * 0"
+  concurrencyPolicy: Replace
+  suspend: false
+  successfulJobsHistoryLimit: 0
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:
+            name: dump_to_yandex_disk
+        spec:
+          containers:
+            - name: dump_to_yandex_disk
+              image: "{{ docker_registry }}/{{ backend_django_image_name }}:{{ backend_django_image_tag }}"
+              imagePullPolicy: IfNotPresent
+              command: [ "/bin/sh" ]
+              args: [ "-c", "python manage.py dump_to_yandex_disk" ]
+              env:
+                {% filter indent(width=16) %}{% include 'app-env.yaml' %}{% endfilter %}
+          restartPolicy: Never