Reapplied last fixes before transition to public repo

pipeline/core/migrations/0015_tenant_audescribe_active.py

@@ -0,0 +1,16 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0014_auto_20230213_0505'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='tenant',
+            name='audescribe_active',
+            field=models.BooleanField(default=False),
+        ),
+    ]

pipeline/core/models/tenant.py

@@ -21,6 +21,7 @@ class Tenant(models.Model):
     aws_active = models.BooleanField(default=True)
     mllp_active = models.BooleanField(default=True)
     deepl_active = models.BooleanField(default=False)
+    audescribe_active = models.BooleanField(default=False)
 
     def get_secret(self, key, raise_exception=True, default=None):
         if key in self.secrets:
@@ -57,6 +58,9 @@ def active_cloud_services(self):
         if self.deepl_active:
             l.append(Subtitle.Origin.DEEPL)
 
+        if self.audescribe_active:
+            l.append(Subtitle.Origin.AUDESCRIBE)
+
         l.append(TranslationService.MANUAL)
 
         return l

pipeline/mooclink/services/aws_translation_service.py

@@ -176,7 +176,7 @@ def fetch_translation_jobs(self, video_id):
         if len(video.workflow_data['waiting_job_ids']) == 0:
             video.workflow_status = None
             if periodic_task_id := video.workflow_data.get('periodic_task_id'):
-                PeriodicTask.objects.get(pk=periodic_task_id).delete()
+                PeriodicTask.objects.filter(pk=periodic_task_id).delete()
                 video.workflow_data['cleared'] = True
 
         video.save()

pipeline/mooclink/services/periodic_task_service.py

@@ -0,0 +1,30 @@
+import json
+from datetime import datetime, timedelta
+
+from django_celery_beat.models import IntervalSchedule, PeriodicTask
+
+
+class PeriodicTaskService:
+
+    @classmethod
+    def create_periodic_task(cls, service_type, task, video, start_time=None, end_time=None):
+        schedule, created = IntervalSchedule.objects.get_or_create(
+            every=10,
+            period=IntervalSchedule.MINUTES,
+        )
+
+        if PeriodicTask.objects.filter(name=f"Check AWS StandaloneTranslation for video={video.pk}").count():
+            return
+
+        periodic = PeriodicTask.objects.create(
+            interval=schedule,
+            name=f'Check AWS StandaloneTranslation for video={video.pk}',
+            task=task,
+            start_time=datetime.now() + timedelta(minutes=15),
+            kwargs=json.dumps({
+                'tenant_id': video.tenant_id,
+                'video_id': video.pk,
+            })
+        )
+
+        return periodic

pipeline/mooclink/templates/mooclink/admin/job_view.html

@@ -0,0 +1,72 @@
+{% extends "subtitles/base.html" %}
+{% block content %}
+    {% load static %}
+    {% load utils %}
+    <link rel="stylesheet" href="{% static 'subtitles/css/mooclink.css' %}">
+
+    <div class="container-fluid">
+        <h1>Pending Videos</h1>
+
+        <table class="table table-sm">
+            <thead>
+            <tr>
+                <th>Video</th>
+                <th>Service</th>
+                <th>Started</th>
+                <th>Action</th>
+            </tr>
+            </thead>
+            <tbody>
+            {% for video in pending_videos %}
+                <tr>
+                    <td>
+                        <a href="{% tenant_url 'mooclink.video.index' course_id=video.course_section.course.ext_id video_id=video.ext_id tenant=video.tenant %}" target="_blank">
+                            {{ video }}
+                        </a>
+                    </td>
+                    <td>{{ video.workflow_status }}</td>
+                    <td>{{ video.job_initiated }}</td>
+                    <td>
+                        <button form="cancellation-form" type="submit" name="video_id" value="{{ video.id }}"
+                                class="btn btn-danger btn-sm"
+                                onclick="confirm('Please confirm the cancellation of job for video `{{ video }}`')">
+                            Cancel
+                        </button>
+                    </td>
+                </tr>
+            {% endfor %}
+            </tbody>
+        </table>
+
+        <h2>Orphaned Periodic Tasks</h2>
+        <table class="table table-sm">
+            <thead>
+            <tr>
+                <th>Video</th>
+                <th>Started</th>
+                <th>Action</th>
+            </tr>
+            </thead>
+            <tbody>
+            {% for orphaned in orphaned_tasks %}
+                <tr>
+                    <td>
+                        <a href="{% tenant_url 'mooclink.video.index' course_id=orphaned.video.course_section.course.ext_id video_id=orphaned.video.ext_id tenant=orphaned.video.tenant %}" target="_blank">
+                            {{ orphaned.video }}
+                        </a>
+                    </td>
+                    <td>{{ orphaned.task.start_time }}</td>
+                    <td>
+                        <button type="submit" class="btn btn-sm btn-danger" name="periodic_task_id"
+                                value="{{ orphaned.task.pk }}" form="cancellation-form">Cancel
+                        </button>
+                    </td>
+                </tr>
+            {% endfor %}
+            </tbody>
+        </table>
+    </div>
+    <form action="{% url "mooclink.jobs.reset" %}" method="post" id="cancellation-form">
+        {% csrf_token %}
+    </form>
+{% endblock %}

pipeline/mooclink/templates/mooclink/course/overview.html

@@ -67,6 +67,7 @@ <h1 class="mt-2">{{ course.title }}</h1>
                             {% if perms.subtitles.change_settings %}
                                 <option value="remove-assignment">Remove Assignments</option>
                             {% endif %}
+                            <option value="download-vtt-files">Download VTT-Files</option>
                         </select>
                         <div class="input-group-append">
                             <button class="btn btn-outline-primary" type="submit">GO</button>

pipeline/mooclink/templates/mooclink/course/settings.html

@@ -7,14 +7,16 @@
     <div class="container-fluid">
         <h1>{{ course.title }}</h1>
 
-        <form action="{% tenant_url 'mooclink.course.settings' course_id=course.ext_id tenant=course.tenant %}" method="post"
+        <form action="{% tenant_url 'mooclink.course.settings' course_id=course.ext_id tenant=course.tenant %}"
+              method="post"
               id="delete-assigned-language-form">
             {% csrf_token %}
             <input type="hidden" name="action" value="delete_assigned_language">
         </form>
 
         <h2 class="mt-3">Settings</h2>
-        <form action="{% tenant_url 'mooclink.course.settings' course_id=course.ext_id tenant=course.tenant %}" method="post">
+        <form action="{% tenant_url 'mooclink.course.settings' course_id=course.ext_id tenant=course.tenant %}"
+              method="post">
             {% csrf_token %}
             <h4>Transcription</h4>
             <table>
@@ -26,7 +28,8 @@ <h4>Transcription</h4>
                     <td>
                         <select name="course-language" class="form-control">
                             {% for lang in available_languages %}
-                            <option value="{{ lang.iso_code }}" {% if lang == course.language %}selected{% endif %}>{{ lang }}</option>
+                                <option value="{{ lang.iso_code }}"
+                                        {% if lang == course.language %}selected{% endif %}>{{ lang }}</option>
                             {% endfor %}
                         </select>
                     </td>
@@ -36,7 +39,13 @@ <h4>Transcription</h4>
                             {% for transcription_service in supported_translation_services %}
                                 <option value="{{ transcription_service }}"
                                         {% if course.transcription_service == transcription_service %}
-                                        selected{% endif %}>{{ transcription_service }}</option>
+                                        selected{% endif %}>
+                                    {% if transcription_service == "AUDESCR" %}
+                                        AUDESCRIBE
+                                    {% else %}
+                                        {{ transcription_service }}
+                                    {% endif %}
+                                </option>
                             {% endfor %}
                         </select>
                     </td>
@@ -55,7 +64,8 @@ <h4 class="mt-3">Translation</h4>
                     <tr>
                         <td>{{ lang.iso_language }}</td>
                         <td>
-                            <select name="service-provider-{{ lang.pk }}" class="form-control" id="service-provider" required>
+                            <select name="service-provider-{{ lang.pk }}" class="form-control" id="service-provider"
+                                    required>
                                 {{ lang.translation_service }}
                                 <option value="">-- Please Select --</option>
                                 {% for translation_service in supported_translation_services %}
@@ -65,7 +75,8 @@ <h4 class="mt-3">Translation</h4>
                                 {% endfor %}
                             </select>
                         </td>
-                        <td><input type="checkbox" class="form-control" name="mandatory-{{ lang.pk }}"{% if lang.required %}
+                        <td><input type="checkbox" class="form-control" name="mandatory-{{ lang.pk }}"
+                                {% if lang.required %}
                                    checked{% endif %}></td>
                         <td>
                             <button type="submit" name="delete_asid" value="{{ lang.pk }}" class="btn btn-danger btn-sm"
@@ -87,7 +98,8 @@ <h4 class="mt-3">Translation</h4>
             </table>
             <div class="mt-3">
                 <button type="submit" class="btn btn-primary">Save</button>
-                <a href="{% tenant_url 'mooclink.course.overview' course_id=course.ext_id tenant=course.tenant %}" class="btn btn-secondary">Back</a>
+                <a href="{% tenant_url 'mooclink.course.overview' course_id=course.ext_id tenant=course.tenant %}"
+                   class="btn btn-secondary">Back</a>
             </div>
         </form>
     </div>

pipeline/mooclink/templates/mooclink/partials/subtitle_actions.html

@@ -24,3 +24,4 @@
         Publish
     </button>
 {% endif %}
+<button class="btn btn-outline-primary mr-1" type="submit" name="action" value="download_vtt">Download VTT</button>

pipeline/mooclink/templates/mooclink/partials/subtitle_legend.html

@@ -1,7 +1,3 @@
-<div class="row">
-    <div class="col-2">
-        <h4>Legend</h4>
-
 <div class="row">
     <div class="col-2">
         <h4>Legend</h4>

pipeline/mooclink/urls.py

@@ -2,7 +2,7 @@
 
 from mooclink.views.course import CourseOverView, CourseSettingsView, CourseAddLanguageView, RemoveUserAssignment
 from mooclink.views.couse_video_bulkaction import CourseBulkActionConfirmation, CourseDoBulkAction, CourseSubscribe
-from mooclink.views.main import MainView, MainViewCourse, RedirectByItemId
+from mooclink.views.main import MainView, MainViewCourse, RedirectByItemId, JobAdminView, JobAdminResetView
 from mooclink.views.service_provider_usage import ServiceProviderUsage, TodoView, ServiceProviderUsageCSV, \
     ServiceProviderUsageQuarter
 from mooclink.views.subtitle_action import SubtitleToAction
@@ -37,4 +37,7 @@
 
     # Endpoint for redirecting to course / video by primary-key. Allowed only for admins
     path('debug/get_item/', RedirectByItemId.as_view()),
+
+    path('jobs/', JobAdminView.as_view(), name="mooclink.jobs.index"),
+    path('jobs/reset/', JobAdminResetView.as_view(), name="mooclink.jobs.reset"),
 ]

pipeline/mooclink/views/couse_video_bulkaction.py

@@ -1,24 +1,28 @@
+import io
 import itertools
-import json
+import zipfile
 from datetime import datetime, timedelta
 from operator import itemgetter
+from zipfile import ZipFile
 
 from django import views
 from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth.mixins import PermissionRequiredMixin, LoginRequiredMixin
 from django.core.exceptions import PermissionDenied
 from django.db import transaction
+from django.http import HttpResponse
 from django.shortcuts import render, redirect
 from django.template.loader import render_to_string
 from django.urls import reverse
 from django.utils import timezone
-from django_celery_beat.models import IntervalSchedule, PeriodicTask
+from django.utils.text import slugify
+from django_celery_beat.models import IntervalSchedule
 
-from core.exceptions import SecretNotFound
 from core.models import Tenant, TranspipeUser
 from mooclink.services.aws_translation_service import AwsTranslationService
 from mooclink.services.deepl_translation_service import DeeplTranslationService
+from mooclink.services.periodic_task_service import PeriodicTaskService
 from subtitles.api.xikolo_api import publish_subtitle_to_xikolo
 from subtitles.models import Course, Video, IsoLanguage, Subtitle, SubtitleAssignment, AssignedLanguage
 from subtitles.models.translation_service import TranslationService
@@ -274,19 +278,27 @@ def post(self, request, course_id, tenant_slug=None):
                             period=IntervalSchedule.MINUTES,
                         )
 
-                        periodic = PeriodicTask.objects.create(
-                            interval=schedule,
-                            name=f'Check AWS StandaloneTranslation for video={video.pk}',
-                            task='core.tasks.task_update_aws_standalone_translation_status',
+                        # periodic = PeriodicTask.objects.create(
+                        #     interval=schedule,
+                        #     name=f'Check AWS StandaloneTranslation for video={video.pk}',
+                        #     task='core.tasks.task_update_aws_standalone_translation_status',
+                        #     start_time=datetime.now() + timedelta(minutes=15),
+                        #     kwargs=json.dumps({
+                        #         'tenant_id': video.tenant_id,
+                        #         'video_id': video.pk,
+                        #     })
+                        # )
+
+                        periodic = PeriodicTaskService.create_periodic_task(
+                            service_type="core.tasks.task_update_aws_standalone_translation_status",
+                            task="core.tasks.task_update_aws_standalone_translation_status",
+                            video=video,
                             start_time=datetime.now() + timedelta(minutes=15),
-                            kwargs=json.dumps({
-                                'tenant_id': video.tenant_id,
-                                'video_id': video.pk,
-                            })
                         )
 
                         video.workflow_data['type'] = 'AWS_TRANSLATION_S_v1'
-                        video.workflow_data['periodic_task_id'] = periodic.pk
+                        if periodic:
+                            video.workflow_data['periodic_task_id'] = periodic.pk
                         video.workflow_data['initiated'] = str(datetime.utcnow())
                         video.save()
 
@@ -385,6 +397,32 @@ def post(self, request, course_id, tenant_slug=None):
                     SubtitleAssignment.objects.filter(subtitle=subtitle).update(deleted=timezone.now())
 
                     number_of_affected += 1
+        elif action == "download-vtt-files":
+            buffer = io.BytesIO()
+            zip_file = ZipFile(buffer, mode="w", compression=zipfile.ZIP_LZMA)
+
+            for video in videos_to_transcript:
+                with zip_file.open(f"{video.index:02d}_{slugify(video.title)}_{video.original_language.iso_code}.vtt", "w") as f:
+                    if video.current_transcript:
+                        f.write(video.current_transcript.latest_content.encode("utf-8"))
+
+            for (language, video) in videos_to_translate:
+                subtitle = video.subtitle_set.filter(language=language).first()
+
+                if not subtitle:
+                    continue
+
+                with zip_file.open(f"{video.index:02d}_{slugify(video.title)}_{subtitle.language.iso_code}.vtt", "w") as f:
+                    if video.current_transcript:
+                        f.write(subtitle.latest_content.encode("utf-8"))
+
+            zip_file.close()
+
+            response = HttpResponse(buffer.getvalue())
+            response['Content-Type'] = 'application/x-zip-compressed'
+            response['Content-Disposition'] = f'attachment; filename=subtitles_{slugify(course.title)}.zip'
+
+            return response
 
         else:
             messages.error(request, f"Action {action} not available")