issue #1171 - progress indicator

analysis/tasks/analysis_grid_export_tasks.py

@@ -39,13 +39,16 @@ def get_grid_downloadable_file_params_hash(pk, export_type):
     return sha256sum_str(f"{pk}-{export_type}")
 
 
-def _write_cached_generated_file(cgf: CachedGeneratedFile, filename, file_iterator):
+def _write_cached_generated_file(cgf: CachedGeneratedFile, total_records, filename, file_iterator):
+    update_size = max(1000, total_records / 100)  # 1% or every 1k records
+    update_progress_iterator = update_cgf_progress_iterator(file_iterator(), cgf.pk, total_records, update_size)
+
     logging.info("Starting to write %s", filename)
     media_root_filename = os.path.join(settings.MEDIA_ROOT, str(uuid.uuid4()), filename)
     try:
         mk_path_for_file(media_root_filename)
         with open(media_root_filename, "w") as f:
-            for line in file_iterator():
+            for line in update_progress_iterator:
                 f.write(line)  # Already has newline
         cgf.filename = media_root_filename
         cgf.task_status = "SUCCESS"
@@ -58,6 +61,19 @@ def _write_cached_generated_file(cgf: CachedGeneratedFile, filename, file_iterat
     cgf.save()
 
 
+def update_cgf_progress_iterator(iterator, cgf_id, total_records, update_size):
+    update_size = int(update_size)  # make sure int so modulus below will hit
+    cgf_qs = CachedGeneratedFile.objects.filter(id=cgf_id)
+    cgf_qs.update(progress=0)
+
+    for i, record in enumerate(iterator):
+        if i % update_size == 0:
+            progress = i / total_records
+            cgf_qs.update(progress=progress)
+        yield record
+    cgf_qs.update(progress=1, task_status='SUCCESS')
+
+
 @celery.shared_task
 def export_cohort_to_downloadable_file(cohort_id, export_type):
     # This should have been created in analysis.views.views_grid.cohort_grid_export
@@ -74,7 +90,7 @@ def export_cohort_to_downloadable_file(cohort_id, export_type):
 
     request = FakeRequest(user=admin_bot())
     filename, file_iterator = node_grid_get_export_iterator(request, node, export_type, basename=basename)
-    _write_cached_generated_file(cgf, filename, file_iterator)
+    _write_cached_generated_file(cgf, node.count, filename, file_iterator)
 
 
 @celery.shared_task
@@ -92,4 +108,4 @@ def export_sample_to_downloadable_file(sample_id, export_type):
                          str(sample.genome_build)])
     request = FakeRequest(user=admin_bot())
     filename, file_iterator = node_grid_get_export_iterator(request, node, export_type, basename=basename)
-    _write_cached_generated_file(cgf, filename, file_iterator)
+    _write_cached_generated_file(cgf, node.count, filename, file_iterator)

snpdb/migrations/0146_cachedgeneratedfile_progress.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.11 on 2024-09-19 02:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("snpdb", "0145_one_off_handle_renamed_table_inherited_index"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="cachedgeneratedfile",
+            name="progress",
+            field=models.FloatField(null=True),
+        ),
+    ]

snpdb/models/models.py

@@ -63,6 +63,7 @@ class CachedGeneratedFile(models.Model):
     task_status = models.TextField(null=True)
     generate_start = models.DateTimeField(null=True)
     generate_end = models.DateTimeField(null=True)
+    progress = models.FloatField(null=True)
 
     class Meta:
         unique_together = ("generator", "params_hash")
@@ -92,6 +93,7 @@ def get_or_create_and_launch(generator, params_hash, task: signature) -> 'Cached
             async_result = task.apply_async()
             cgf.task_id = async_result.id
             cgf.generate_start = timezone.now()
+            cgf.progress = 0.0
             cgf.save()
         else:
             async_result = AsyncResult(cgf.task_id)

snpdb/templates/snpdb/data/sample_graphs_tab.html

@@ -1,5 +1,4 @@
 {% load static %}
-<script src="{% static 'js/cached_generated_files.js' %}"></script>
 <style>
     .generated-graph, .generated-graph img {
         width: 640px;

snpdb/views/views_json.py

@@ -1,13 +1,12 @@
 import json
-import os
 
 from celery.result import AsyncResult
 from django.conf import settings
 from django.http.response import JsonResponse
 from django.shortcuts import get_object_or_404
 from django.views.decorators.http import require_POST
 
-from library.django_utils import require_superuser, get_url_from_media_root_filename
+from library.django_utils import require_superuser
 from snpdb.models import CachedGeneratedFile, Cohort, Sample, VCF, CustomColumnsCollection, TagColorsCollection
 from snpdb.tasks.clingen_tasks import populate_clingen_alleles_from_vcf
 from snpdb.tasks.cohort_genotype_tasks import create_cohort_genotype_and_launch_task
@@ -28,8 +27,11 @@ def job_status(request, job_id):
 def cached_generated_file_check(request, cgf_id):
     cgf = get_object_or_404(CachedGeneratedFile, pk=cgf_id)
     cgf.check_ready()
-    data = {"status": cgf.task_status,
-            "cgf_id": cgf.id}
+    data = {
+        "status": cgf.task_status,
+        "cgf_id": cgf.id,
+        "progress": cgf.progress
+    }
 
     if cgf.exception:
         data["exception"] = str(cgf.exception)

variantgrid/static_files/default_static/js/cached_generated_files.js

@@ -47,15 +47,19 @@ function poll_graph_status(graph_selector, poll_url, delete_url) {
 }
 
 
-function poll_cached_generated_file(poll_url, success_func, failure_func) {
+function poll_cached_generated_file(poll_url, success_func, failure_func, update_func) {
 	$.getJSON(poll_url, function (data) {
 		if (data.status == "SUCCESS") {
 			success_func(data);
 		} else if (data.status == 'FAILURE') {
 			failure_func(data);
 		} else {
+			if (update_func) {
+				update_func(data.progress);
+			}
+
 			const retry_func = function () {
-				poll_cached_generated_file(poll_url, success_func, failure_func);
+				poll_cached_generated_file(poll_url, success_func, failure_func, update_func);
 			};
 			window.setTimeout(retry_func, freq);
 		}
@@ -103,7 +107,18 @@ class AnnotatedFileDownload {
 				window.location.href = data.url;
 			}
 		}
-		$(this.selector).html(`<span><i class="fas fa-spinner fa-spin"></i> Preparing download...</span>`);
-		poll_cached_generated_file(this.pollUrl, downloadFile, this.setError)
+		let spinner = $(`<span><i class="fas fa-spinner fa-spin"></i> Preparing download... </span>`);
+		let progressIndicator = $("<span></span>");
+		spinner.append(progressIndicator);
+
+		function updateProgress(progress) {
+			console.log("updateProgress");
+			let percent = Math.floor(100 * progress);
+			progressIndicator.empty();
+			progressIndicator.append(`${percent}% complete`);
+		}
+
+		$(this.selector).html(spinner);
+		poll_cached_generated_file(this.pollUrl, downloadFile, this.setError, updateProgress)
 	}
 }