From da84b8b7f05e6a8d26abda87f6115379837ec3dc Mon Sep 17 00:00:00 2001 From: Dave Lawrence Date: Wed, 31 Jul 2024 14:05:51 +0930 Subject: [PATCH] Break up annotation page by genome build so it can load in parallel --- .../templates/annotation/annotation.html | 33 ++- .../annotation/annotation_build_detail.html | 203 ++++++++++++++++ .../annotation/annotation_detail.html | 217 +----------------- annotation/urls.py | 1 + annotation/views.py | 38 +-- 5 files changed, 263 insertions(+), 229 deletions(-) create mode 100644 annotation/templates/annotation/annotation_build_detail.html diff --git a/annotation/templates/annotation/annotation.html b/annotation/templates/annotation/annotation.html index 9c081c03a..9fd69bac8 100644 --- a/annotation/templates/annotation/annotation.html +++ b/annotation/templates/annotation/annotation.html @@ -19,8 +19,39 @@ {% load ui_help %} {% page_help page_id='annotation/annotation_help' title='Annotations' %} + + + + {% for genome_build in genome_build_list %} +
+
Loading {{ genome_build }} annotation...
+
+ {% endfor %} +
-
Loading Annotations, this may take a while.
+
Loading non-build annotations...
{% endblock content %} diff --git a/annotation/templates/annotation/annotation_build_detail.html b/annotation/templates/annotation/annotation_build_detail.html new file mode 100644 index 000000000..e78041a29 --- /dev/null +++ b/annotation/templates/annotation/annotation_build_detail.html @@ -0,0 +1,203 @@ +{% load static %} +{% load js_tags %} +{% load ui_menu_bars %} +{% load ui_utils %} +{% load humanize %} +{% load tz %} +{% load settings_tags %} + +{% settings_value 'PYTHON_COMMAND' as python_command %} + +
+
{{ build_name }}
+
+ {% labelled label="Latest" %}{{ details.latest }}{% endlabelled %} + {% labelled label="Sync with current VEP" %} + {% if details.sync_with_current_vep %} + {{ details.sync_with_current_vep }} + {% endif %} + {% if details.out_of_sync_with_current_vep %} + + {{ details.out_of_sync_with_current_vep }} + + {% endif %} + {% install-instructions "Sync with current VEP" installed=details.sync_with_current_vep %} + {{ python_command }} manage.py create_new_variant_annotation_version --genome_build={{ build_name }} + {% endinstall-instructions %} + {% endlabelled %} + {% labelled label="Genome Fasta" %} + {% if details.reference_fasta %} + {{ details.reference_fasta }} + {% else %} +
Not found
+ {% endif %} + {% install-instructions "Reference Fasta" installed=details.reference_fasta %} +
{{ details.reference_fasta_error }}
+ You need to download the annotations, or change settings.GENOME_BUILD_REFERENCE_FASTA + {% endinstall-instructions %} + {% endlabelled %} + {% labelled label="Contigs" %} +
+ Contigs: {{ details.contigs.num_contigs }} Toggle +
{{ details.contigs.contigs }}
+
+ {% endlabelled %} + {% labelled label="Gene/Transcript Information" %} + {% if details.genes_and_transcripts %} +
+ Lastest cdot {{ details.genes_and_transcripts.latest_cdot }} +
+
+ Annotation: {{ details.annotation_consortium }} + Genes: {{ details.genes_and_transcripts.genes | intcomma }} + Transcripts: {{ details.genes_and_transcripts.transcripts | intcomma }} + Toggle import sources +
+
+ Import Sources + {% for import_source, import_stats in details.genes_and_transcripts.import_sources.items %} +
+ {{ import_source }}: {{ import_stats.transcripts }} transcripts ({{ import_stats.created|localtime }}) +
+ {% endfor %} +
+ {% endif %} + + {% for cons_name, other_consortia_data in details.other_consortia.items %} +
+ Annotation: {{ cons_name }} + Genes: {{ other_consortia_data.genes | intcomma }} + Transcripts: {{ other_consortia_data.transcripts | intcomma }} + Toggle import sources +
+
+ Import Sources + {% for file_name, import_stats in other_consortia_data.import_sources.items %} +
+ {{ file_name }}: {{ import_stats.transcripts }} transcripts ({{ import_stats.created|localtime }}) +
+ {% endfor %} +
+ {% endfor %} + + {% install-instructions build_name installed=details.genes_and_transcripts.transcripts %} +

+ You can install both Ensembl and RefSeq transcripts (for HGVS) but you NEED the ones used by your variant annotation pipeline which is: '{{ details.annotation_consortium }}' +

+ +

+ Download data from cdot. See Gene and Transcript version data for details and how to generate this from scratch +

+ + +export gene_annotation_dir=${VARIANTGRID_SETUP_DATA}/gene_annotation +mkdir -p ${gene_annotation_dir} +cd ${gene_annotation_dir} + + {% if build_name == 'GRCh37' %} +
Ensembl GRCh37
+ + +wget https://cdot.cc/download/{{ cdot_version }}/cdot-{{ cdot_version }}.ensembl.grch37.json.gz +{{ python_command }} manage.py import_gene_annotation --annotation-consortium=Ensembl --genome-build=GRCh37 --json-file ${gene_annotation_dir}/cdot-{{ cdot_version }}.ensembl.grch37.json.gz + + {% elif build_name == 'GRCh38' %} +
Ensembl GRCh38
+ +wget https://cdot.cc/download/{{ cdot_version }}/cdot-{{ cdot_version }}.ensembl.grch38.json.gz +{{ python_command }} manage.py import_gene_annotation --annotation-consortium=Ensembl --genome-build=GRCh38 --json-file ${gene_annotation_dir}/cdot-{{ cdot_version }}.ensembl.grch38.json.gz + + {% endif %} + + {% if build_name == 'GRCh37' %} +
RefSeq GRCh37
+ +wget https://cdot.cc/download/{{ cdot_version }}/cdot-{{ cdot_version }}.refseq.grch37.json.gz +{{ python_command }} manage.py import_gene_annotation --annotation-consortium=RefSeq --genome-build=GRCh37 --json-file ${gene_annotation_dir}/cdot-{{ cdot_version }}.refseq.grch37.json.gz + + {% elif build_name == 'GRCh38' %} +
RefSeq GRCh38
+ +wget https://cdot.cc/download/{{ cdot_version }}/cdot-{{ cdot_version }}.refseq.grch38.json.gz +{{ python_command }} manage.py import_gene_annotation --annotation-consortium=RefSeq --genome-build=GRCh38 --json-file ${gene_annotation_dir}/cdot-{{ cdot_version }}.refseq.grch38.json.gz + + {% endif %} + {% endinstall-instructions %} + {% endlabelled %} + + {% labelled label="Gene Annotation Release" %} + {% if details.gene_annotation_release %} + {{ details.gene_annotation_release.name }} + {% else %} +
Not Installed
+ {% endif %} + {% install-instructions "Gene Annotation Release" installed=details.gene_annotation_release %} + You need a Gene Annotation Release that matches the exact symbol/gene/transcript versions used by VEP. + See Wiki for details + + {% with vav=details.latest.variant_annotation_version %} + {% if vav.cdot_gene_release_filename %} + +wget http://cdot.cc/download/{{ vav.get_annotation_consortium_display }}/{{ build_name }}/{{ vav.cdot_gene_release_filename }} +{{ python_command }} manage.py import_gene_annotation --annotation-consortium={{ vav.get_annotation_consortium_display }} --genome-build={{ build_name }} --json-file {{ vav.cdot_gene_release_filename }} --release={{ vav.suggested_gene_annotation_release_name }} + + If it doesn't auto-link, go to Admin then set the gene annotation release. + {% endif %} + {% endwith %} + + {% endinstall-instructions %} + {% endlabelled %} + + {% settings_value 'ANNOTATION_GENE_ANNOTATION_VERSION_ENABLED' as ANNOTATION_GENE_ANNOTATION_VERSION_ENABLED %} + {% if ANNOTATION_GENE_ANNOTATION_VERSION_ENABLED %} + {% labelled label="Gene Annotation" %} + {% if details.gene_level_annotation %} + {{ details.gene_level_annotation }} + {% else %} +
Not Installed
+ {% endif %} + {% install-instructions "Gene Annotation" installed=details.gene_level_annotation %} + {{ python_command }} manage.py gene_annotation --gene-annotation-release={{ details.latest.variant_annotation_version.gene_annotation_release_id|default_if_none:'(Gene Annotation Release ID)' }} + {% endinstall-instructions %} + {% endlabelled %} + {% endif %} + + {% labelled label="ClinVar" %} + {% if details.clinvar %} + {{ details.clinvar }} + {% else %} +
Not installed
+ {% endif %} + {% install-instructions "Clinvar" installed=details.clinvar %} + Download VCF (with date in file name) from NCBI ClinVar site, then run: + + {{ python_command }} manage.py clinvar_import ${VARIANTGRID_SETUP_DATA}/clinvar_20180401.vcf.gz + {% endinstall-instructions %} + + {% endlabelled %} + {% if somalier_enabled %} + {% labelled label="Somalier" %} + {% if details.somalier %} + {{ details.somalier }} + {% else %} + Not installed + {% endif %} + {% install-instructions "Somalier" installed=details.somalier %} + Upload the Somalier sites VCF for {{ build_name }}. + {% endinstall-instructions %} + {% endlabelled %} + {% endif %} +
+
+ +
+ Details.ok = {{ details.ok }} +
+ + {% if not details.ok %} + + {% endif %} diff --git a/annotation/templates/annotation/annotation_detail.html b/annotation/templates/annotation/annotation_detail.html index 50840da21..055677fd0 100644 --- a/annotation/templates/annotation/annotation_detail.html +++ b/annotation/templates/annotation/annotation_detail.html @@ -8,216 +8,6 @@ {% settings_value 'PYTHON_COMMAND' as python_command %} -{% if not annotations_all_imported %} -
-
Installation Instructions
-
-

- See instructions at Annotation Setup (VariantGrid Wiki) - -

Command line examples should be executed in the variantgrid dir (${VG_INSTALL_DIR}) running as the (${VARIANTGRID_USER}) - -

The variables should be set as per wiki link (same as above): - -

    -
  • ${ANNOTATION_BASEDIR} where you downloaded reference annotation -
- -

- Follow the instruction steps below in order (as some have dependencies) -

- -

- Details instructions contain the way to download the latest copy of the annotations, or you can just use the out of date data from ${VARIANTGRID_SETUP_DATA} dir (which came from downloading annotations) -

-
-
-{% endif %} - -{% for build_name, details in genome_build_annotations.items %} -
-
{{ build_name }}
-
- {% labelled label="Latest" %}{{ details.latest }}{% endlabelled %} - {% labelled label="Sync with current VEP" %} - {% if details.sync_with_current_vep %} - {{ details.sync_with_current_vep }} - {% endif %} - {% if details.out_of_sync_with_current_vep %} - - {{ details.out_of_sync_with_current_vep }} - - {% endif %} - {% install-instructions "Sync with current VEP" installed=details.sync_with_current_vep %} - {{ python_command }} manage.py create_new_variant_annotation_version --genome_build={{ build_name }} - {% endinstall-instructions %} - {% endlabelled %} - {% labelled label="Genome Fasta" %} - {% if details.reference_fasta %} - {{ details.reference_fasta }} - {% else %} -
Not found
- {% endif %} - {% install-instructions "Reference Fasta" installed=details.reference_fasta %} -
{{ details.reference_fasta_error }}
- You need to download the annotations, or change settings.GENOME_BUILD_REFERENCE_FASTA - {% endinstall-instructions %} - {% endlabelled %} - {% labelled label="Contigs" %} -
- Contigs: {{ details.contigs.num_contigs }} Toggle -
{{ details.contigs.contigs }}
-
- {% endlabelled %} - {% labelled label="Gene/Transcript Information" %} - {% if details.genes_and_transcripts %} -
- Lastest cdot {{ details.genes_and_transcripts.latest_cdot }} -
-
- Annotation: {{ details.annotation_consortium }} - Genes: {{ details.genes_and_transcripts.genes | intcomma }} - Transcripts: {{ details.genes_and_transcripts.transcripts | intcomma }} - Toggle import sources -
-
- Import Sources - {% for import_source, import_stats in details.genes_and_transcripts.import_sources.items %} -
- {{ import_source }}: {{ import_stats.transcripts }} transcripts ({{ import_stats.created|localtime }}) -
- {% endfor %} -
- {% endif %} - - {% for cons_name, other_consortia_data in details.other_consortia.items %} -
- Annotation: {{ cons_name }} - Genes: {{ other_consortia_data.genes | intcomma }} - Transcripts: {{ other_consortia_data.transcripts | intcomma }} - Toggle import sources -
-
- Import Sources - {% for file_name, import_stats in other_consortia_data.import_sources.items %} -
- {{ file_name }}: {{ import_stats.transcripts }} transcripts ({{ import_stats.created|localtime }}) -
- {% endfor %} -
- {% endfor %} - - {% install-instructions build_name installed=details.genes_and_transcripts.transcripts %} -

- You can install both Ensembl and RefSeq transcripts (for HGVS) but you NEED the ones used by your variant annotation pipeline which is: '{{ details.annotation_consortium }}' -

- -

- Download data from cdot. See Gene and Transcript version data for details and how to generate this from scratch -

- - -export gene_annotation_dir=${VARIANTGRID_SETUP_DATA}/gene_annotation -mkdir -p ${gene_annotation_dir} -cd ${gene_annotation_dir} - - {% if build_name == 'GRCh37' %} -
Ensembl GRCh37
- - -wget https://cdot.cc/download/{{ cdot_version }}/cdot-{{ cdot_version }}.ensembl.grch37.json.gz -{{ python_command }} manage.py import_gene_annotation --annotation-consortium=Ensembl --genome-build=GRCh37 --json-file ${gene_annotation_dir}/cdot-{{ cdot_version }}.ensembl.grch37.json.gz - - {% elif build_name == 'GRCh38' %} -
Ensembl GRCh38
- -wget https://cdot.cc/download/{{ cdot_version }}/cdot-{{ cdot_version }}.ensembl.grch38.json.gz -{{ python_command }} manage.py import_gene_annotation --annotation-consortium=Ensembl --genome-build=GRCh38 --json-file ${gene_annotation_dir}/cdot-{{ cdot_version }}.ensembl.grch38.json.gz - - {% endif %} - - {% if build_name == 'GRCh37' %} -
RefSeq GRCh37
- -wget https://cdot.cc/download/{{ cdot_version }}/cdot-{{ cdot_version }}.refseq.grch37.json.gz -{{ python_command }} manage.py import_gene_annotation --annotation-consortium=RefSeq --genome-build=GRCh37 --json-file ${gene_annotation_dir}/cdot-{{ cdot_version }}.refseq.grch37.json.gz - - {% elif build_name == 'GRCh38' %} -
RefSeq GRCh38
- -wget https://cdot.cc/download/{{ cdot_version }}/cdot-{{ cdot_version }}.refseq.grch38.json.gz -{{ python_command }} manage.py import_gene_annotation --annotation-consortium=RefSeq --genome-build=GRCh38 --json-file ${gene_annotation_dir}/cdot-{{ cdot_version }}.refseq.grch38.json.gz - - {% endif %} - {% endinstall-instructions %} - {% endlabelled %} - - {% labelled label="Gene Annotation Release" %} - {% if details.gene_annotation_release %} - {{ details.gene_annotation_release.name }} - {% else %} -
Not Installed
- {% endif %} - {% install-instructions "Gene Annotation Release" installed=details.gene_annotation_release %} - You need a Gene Annotation Release that matches the exact symbol/gene/transcript versions used by VEP. - See Wiki for details - - {% with vav=details.latest.variant_annotation_version %} - {% if vav.cdot_gene_release_filename %} - -wget http://cdot.cc/download/{{ vav.get_annotation_consortium_display }}/{{ build_name }}/{{ vav.cdot_gene_release_filename }} -{{ python_command }} manage.py import_gene_annotation --annotation-consortium={{ vav.get_annotation_consortium_display }} --genome-build={{ build_name }} --json-file {{ vav.cdot_gene_release_filename }} --release={{ vav.suggested_gene_annotation_release_name }} - - If it doesn't auto-link, go to Admin then set the gene annotation release. - {% endif %} - {% endwith %} - - {% endinstall-instructions %} - {% endlabelled %} - - {% settings_value 'ANNOTATION_GENE_ANNOTATION_VERSION_ENABLED' as ANNOTATION_GENE_ANNOTATION_VERSION_ENABLED %} - {% if ANNOTATION_GENE_ANNOTATION_VERSION_ENABLED %} - {% labelled label="Gene Annotation" %} - {% if details.gene_level_annotation %} - {{ details.gene_level_annotation }} - {% else %} -
Not Installed
- {% endif %} - {% install-instructions "Gene Annotation" installed=details.gene_level_annotation %} - {{ python_command }} manage.py gene_annotation --gene-annotation-release={{ details.latest.variant_annotation_version.gene_annotation_release_id|default_if_none:'(Gene Annotation Release ID)' }} - {% endinstall-instructions %} - {% endlabelled %} - {% endif %} - - {% labelled label="ClinVar" %} - {% if details.clinvar %} - {{ details.clinvar }} - {% else %} -
Not installed
- {% endif %} - {% install-instructions "Clinvar" installed=details.clinvar %} - Download VCF (with date in file name) from NCBI ClinVar site, then run: - - {{ python_command }} manage.py clinvar_import ${VARIANTGRID_SETUP_DATA}/clinvar_20180401.vcf.gz - {% endinstall-instructions %} - - {% endlabelled %} - {% if somalier_enabled %} - {% labelled label="Somalier" %} - {% if details.somalier %} - {{ details.somalier }} - {% else %} - Not installed - {% endif %} - {% install-instructions "Somalier" installed=details.somalier %} - Upload the Somalier sites VCF for {{ build_name }}. - {% endinstall-instructions %} - {% endlabelled %} - {% endif %} -
-
-{% endfor %} -
Information
@@ -410,4 +200,11 @@
RefSeq GRCh38
} }); }); + + {% if not annotations_all_imported %} + $(document).ready(function() { + $("#installation-instructions").show(); + }); + {% endif %} + \ No newline at end of file diff --git a/annotation/urls.py b/annotation/urls.py index f1f7ee2b9..520ca74f3 100644 --- a/annotation/urls.py +++ b/annotation/urls.py @@ -9,6 +9,7 @@ urlpatterns = [ perm_path('', views.annotation, name='annotation'), + perm_path('annotation_build_detail//', views.annotation_build_detail, name='annotation_build_detail'), perm_path('annotation_detail', views.annotation_detail, name='annotation_detail'), perm_path('annotation_versions', views.annotation_versions, name='annotation_versions'), perm_path('version_diffs', views.version_diffs, name='version_diffs'), diff --git a/annotation/views.py b/annotation/views.py index 7fa0ec355..7072a99ec 100644 --- a/annotation/views.py +++ b/annotation/views.py @@ -42,6 +42,7 @@ def get_build_contigs(): + # TODO: We know what build we have when we call this - so could only retrieve for that build_contigs = {} for genome_build in GenomeBuild.objects.all(): contigs_list = list(genome_build.contigs.values_list("name", flat=True)) @@ -76,7 +77,11 @@ def _get_gene_and_transcript_stats(genome_build: GenomeBuild, annotation_consort return genes_and_transcripts -def _get_build_annotation_details(build_contigs, genome_build): +@not_minified_response +def annotation_build_detail(request, genome_build_name): + genome_build = GenomeBuild.get_name_or_alias(genome_build_name) + build_contigs = get_build_contigs() + annotation_details = { "contigs": build_contigs.get(genome_build.name), "annotation_consortium": genome_build.settings["annotation_consortium"], @@ -140,7 +145,8 @@ def _get_build_annotation_details(build_contigs, genome_build): annotation_details["clinvar"] = f"{clinvar_counts} ClinVar records" clinvar_counts = av.get_clinvar().count() - annotation_sub_components = [reference_ok, genes_and_transcripts, gene_annotation_release, clinvar_counts] + annotation_sub_components = [sync_with_current_vep, reference_ok, + genes_and_transcripts, gene_annotation_release, clinvar_counts] if settings.ANNOTATION_GENE_ANNOTATION_VERSION_ENABLED: annotation_sub_components.append(gene_annotation_counts) @@ -155,11 +161,20 @@ def _get_build_annotation_details(build_contigs, genome_build): annotation_sub_components.append(somalier) annotation_details["ok"] = all(annotation_sub_components) - return annotation_details + + context = { + "build_name": genome_build.name, + "details": annotation_details, + "cdot_version": cdot.__version__, + } + return render(request, "annotation/annotation_build_detail.html", context) def annotation(request): - return render(request, "annotation/annotation.html", {}) + context = { + "genome_build_list": GenomeBuild.builds_with_annotation(), + } + return render(request, "annotation/annotation.html", context) @not_minified_response @@ -168,16 +183,6 @@ def annotation_detail(request): ensembl_biomart_transcript_genes = None diagnostic_gene_list = None - build_contigs = get_build_contigs() - genome_build_annotations = {} - - builds_ok = [] - for genome_build in GenomeBuild.builds_with_annotation(): - annotation_details = _get_build_annotation_details(build_contigs, genome_build) - genome_build_annotations[genome_build.name] = annotation_details - - builds_ok.append(annotation_details.get("ok", False)) - gene_symbol_alias_counts = get_field_counts(GeneSymbolAlias.objects.all(), "source") if gene_symbol_alias_counts: gene_symbol_alias_counts = {GeneSymbolAliasSource(k).label: v for k, v in gene_symbol_alias_counts.items()} @@ -228,8 +233,7 @@ def annotation_detail(request): somalier = _verify_somalier_config() # These are empty/None if not set. - annotations_ok = [all(builds_ok), - all_ontologies_accounted_for, + annotations_ok = [all_ontologies_accounted_for, hpa_counts > 0] if somalier_enabled: annotations_ok.append(somalier) @@ -243,7 +247,6 @@ def annotation_detail(request): context = { "annotations_all_imported": annotations_all_imported, "all_ontologies_accounted_for": all_ontologies_accounted_for, - "genome_build_annotations": genome_build_annotations, "ensembl_biomart_transcript_genes": ensembl_biomart_transcript_genes, "ontology_services": ontology_services, "ontology_counts": ontology_counts, @@ -257,7 +260,6 @@ def annotation_detail(request): "num_annotation_columns": VariantGridColumn.objects.count(), "cached_web_resources": cached_web_resources, "somalier": somalier, - "cdot_version": cdot.__version__, } return render(request, "annotation/annotation_detail.html", context)