From f5096bbb0d2ac38805565afcde2df168fd9d126e Mon Sep 17 00:00:00 2001 From: Dave Lawrence Date: Mon, 26 Aug 2024 15:12:30 +0930 Subject: [PATCH] issue #76 seqauto API step 1 - serializers/initial api --- genes/serializers.py | 31 +- seqauto/models/models_seqauto.py | 12 +- seqauto/serializers.py | 69 ---- seqauto/serializers/__init__.py | 3 + .../serializers/enrichment_kit_serializers.py | 39 +++ seqauto/serializers/seqauto_qc_serializers.py | 69 ++++ seqauto/serializers/seqauto_serializers.py | 34 ++ seqauto/serializers/sequencing_serializers.py | 323 ++++++++++++++++++ .../SampleSheet.csv | 8 +- seqauto/urls.py | 30 +- seqauto/views_rest.py | 85 ++++- upload/urls.py | 7 + upload/views/views.py | 37 +- upload/views/views_rest.py | 21 ++ .../settings/components/default_settings.py | 11 +- 15 files changed, 675 insertions(+), 104 deletions(-) delete mode 100644 seqauto/serializers.py create mode 100644 seqauto/serializers/__init__.py create mode 100644 seqauto/serializers/enrichment_kit_serializers.py create mode 100644 seqauto/serializers/seqauto_qc_serializers.py create mode 100644 seqauto/serializers/seqauto_serializers.py create mode 100644 seqauto/serializers/sequencing_serializers.py create mode 100644 upload/views/views_rest.py diff --git a/genes/serializers.py b/genes/serializers.py index 6026e90a7..d7e4f31f9 100644 --- a/genes/serializers.py +++ b/genes/serializers.py @@ -1,9 +1,10 @@ from rest_framework import serializers from genes.models import GeneInfo, GeneListCategory, GeneList, Gene, Transcript, GeneListGeneSymbol, \ - GeneAnnotationRelease, SampleGeneList, ActiveSampleGeneList, GeneSymbol, TranscriptVersion, GeneVersion, HGNC + GeneAnnotationRelease, SampleGeneList, ActiveSampleGeneList, GeneSymbol, TranscriptVersion, GeneVersion, HGNC, \ + GeneCoverageCollection, GeneCoverageCanonicalTranscript from snpdb.models import Company -from snpdb.serializers import UserSerializer, GenomeBuildSerializer +from snpdb.serializers import GenomeBuildSerializer class GeneSymbolSerializer(serializers.ModelSerializer): @@ -74,7 +75,7 @@ class Meta: class GeneListSerializer(serializers.ModelSerializer): category = GeneListCategorySerializer() - user = UserSerializer() + user = serializers.StringRelatedField() genelistgenesymbol_set = GeneListGeneSymbolSerializer(many=True) can_write = serializers.SerializerMethodField() absolute_url = serializers.URLField(source='get_absolute_url', read_only=True) @@ -131,14 +132,34 @@ class Meta: class SampleGeneListSerializer(serializers.ModelSerializer): - active = serializers.SerializerMethodField() + active = serializers.SerializerMethodField(read_only=True) + gene_list = GeneListSerializer() class Meta: model = SampleGeneList - fields = ('pk', 'visible', 'active') + fields = ('pk', 'visible', 'gene_list', 'active') def get_active(self, obj): try: return obj.sample.activesamplegenelist.sample_gene_list == obj except ActiveSampleGeneList.DoesNotExist: return False + + +class GeneCoverageCanonicalTranscriptSerializer(serializers.ModelSerializer): + transcript_version = TranscriptVersionSerializer() + + class Meta: + model = GeneCoverageCanonicalTranscript + exclude = ("gene_coverage_collection", ) + #fields = "__all__" + + +class GeneCoverageCollectionSerializer(serializers.ModelSerializer): + genome_build = GenomeBuildSerializer() + genecoveragecanonicaltranscript_set = GeneCoverageCanonicalTranscriptSerializer(many=True) + + class Meta: + model = GeneCoverageCollection + # TODO: Check if "__all__" also does related automatically? + fields = ["path", "data_state", "genome_build", "genecoveragecanonicaltranscript_set"] diff --git a/seqauto/models/models_seqauto.py b/seqauto/models/models_seqauto.py index 77a346457..ff9adcd81 100644 --- a/seqauto/models/models_seqauto.py +++ b/seqauto/models/models_seqauto.py @@ -389,7 +389,10 @@ def __str__(self): class SequencingSample(models.Model): - """ Represents a row in a SampleSheet.csv """ + """ Represents a row in a SampleSheet.csv + + As it's not a file, not a SeqAutoRecord + """ sample_sheet = models.ForeignKey(SampleSheet, on_delete=CASCADE) sample_id = models.TextField() # sample_name is used to name files. In MiSeq/NextSeq samplesheet you can add names. @@ -973,7 +976,12 @@ def __str__(self): class QCGeneList(SeqAutoRecord): - """ This represents a text file containing genes which will be used for initial pass and QC filters """ + """ This represents a text file containing genes which will be used for initial pass and QC filters + + The reason we have both a sample_gene_list and a custom_text_gene_list is because we wanted to + represent the text from a file on disk. I think we probably could have gotten around that as + SeqAutoRecord contains a hash - could maybe remove that and just use gene list + """ qc = models.ForeignKey(QC, on_delete=CASCADE) custom_text_gene_list = models.OneToOneField(CustomTextGeneList, null=True, on_delete=SET_NULL) sample_gene_list = models.ForeignKey(SampleGeneList, null=True, on_delete=SET_NULL) diff --git a/seqauto/serializers.py b/seqauto/serializers.py deleted file mode 100644 index 221a33cf3..000000000 --- a/seqauto/serializers.py +++ /dev/null @@ -1,69 +0,0 @@ - - -import numpy as np -from rest_framework import serializers - -from genes.serializers import GeneListSerializer, TranscriptSerializer, GeneSymbolSerializer, \ - TranscriptVersionSerializer -from seqauto.models import GoldCoverageSummary, GoldReference, EnrichmentKit - - -class EnrichmentKitSerializer(serializers.ModelSerializer): - gene_list = GeneListSerializer() - enrichment_kit_type = serializers.SerializerMethodField() - manufacturer = serializers.StringRelatedField() - __str__ = serializers.SerializerMethodField() - - class Meta: - model = EnrichmentKit - fields = ('pk', 'name', 'version', 'enrichment_kit_type', 'manufacturer', 'gene_list', '__str__') - - def get_enrichment_kit_type(self, obj): - return obj.get_enrichment_kit_type_display() - - def get___str__(self, obj): - return str(obj) - - -class EnrichmentKitSummarySerializer(serializers.ModelSerializer): - """ Doesn't return genes (much faster) """ - enrichment_kit_type = serializers.SerializerMethodField() - manufacturer = serializers.StringRelatedField() - __str__ = serializers.SerializerMethodField() - - class Meta: - model = EnrichmentKit - fields = ('pk', 'name', 'version', 'enrichment_kit_type', 'manufacturer', '__str__') - - def get_enrichment_kit_type(self, obj): - return obj.get_enrichment_kit_type_display() - - def get___str__(self, obj): - return str(obj) - - -class GoldReferenceSerializer(serializers.ModelSerializer): - enrichment_kit = EnrichmentKitSummarySerializer() - - class Meta: - model = GoldReference - fields = ('enrichment_kit', 'created') - - -class GoldCoverageSummarySerializer(serializers.ModelSerializer): - gold_reference = GoldReferenceSerializer() - gene_symbol = GeneSymbolSerializer() - transcript = TranscriptSerializer() - transcript_version = TranscriptVersionSerializer() - standard_error = serializers.SerializerMethodField() - - class Meta: - model = GoldCoverageSummary - fields = '__all__' - - def get_standard_error(self, obj): - """ This can occasionally be NaN which isn't valid JSON """ - standard_error = obj.standard_error - if np.isnan(standard_error): - standard_error = -1 - return standard_error diff --git a/seqauto/serializers/__init__.py b/seqauto/serializers/__init__.py new file mode 100644 index 000000000..0b1ed73f9 --- /dev/null +++ b/seqauto/serializers/__init__.py @@ -0,0 +1,3 @@ +from .enrichment_kit_serializers import * +from .seqauto_serializers import * +from .seqauto_serializers import * diff --git a/seqauto/serializers/enrichment_kit_serializers.py b/seqauto/serializers/enrichment_kit_serializers.py new file mode 100644 index 000000000..dd555aaba --- /dev/null +++ b/seqauto/serializers/enrichment_kit_serializers.py @@ -0,0 +1,39 @@ +from rest_framework import serializers + +from genes.serializers import GeneListSerializer +from seqauto.models import EnrichmentKit + + + +class EnrichmentKitSerializer(serializers.ModelSerializer): + gene_list = GeneListSerializer() + enrichment_kit_type = serializers.SerializerMethodField() + manufacturer = serializers.StringRelatedField() + __str__ = serializers.SerializerMethodField(read_only=True) + + class Meta: + model = EnrichmentKit + fields = ('pk', 'name', 'version', 'enrichment_kit_type', 'manufacturer', 'gene_list', '__str__') + + def get_enrichment_kit_type(self, obj): + return obj.get_enrichment_kit_type_display() + + def get___str__(self, obj): + return str(obj) + + +class EnrichmentKitSummarySerializer(serializers.ModelSerializer): + """ Doesn't return genes (much faster) """ + enrichment_kit_type = serializers.SerializerMethodField() + manufacturer = serializers.StringRelatedField() + __str__ = serializers.SerializerMethodField(read_only=True) + + class Meta: + model = EnrichmentKit + fields = ('pk', 'name', 'version', 'enrichment_kit_type', 'manufacturer', '__str__') + + def get_enrichment_kit_type(self, obj): + return obj.get_enrichment_kit_type_display() + + def get___str__(self, obj): + return str(obj) \ No newline at end of file diff --git a/seqauto/serializers/seqauto_qc_serializers.py b/seqauto/serializers/seqauto_qc_serializers.py new file mode 100644 index 000000000..1b44fea83 --- /dev/null +++ b/seqauto/serializers/seqauto_qc_serializers.py @@ -0,0 +1,69 @@ +from rest_framework import serializers + +from genes.serializers import SampleGeneListSerializer, GeneCoverageCollectionSerializer +from seqauto.models import IlluminaFlowcellQC, QCGeneList, QC, QCGeneCoverage, QCExecSummary, FastQC, SequencingSample +from seqauto.serializers.sequencing_serializers import SampleSheetLookupSerializer, FastqSerializer, SeqAutoViewMixin, \ + BamFilePathSerializer, VCFFilePathSerializer, SequencingSampleLookupSerializer + + +class FastQCSerializer(SeqAutoViewMixin, serializers.ModelSerializer): + fastq = FastqSerializer() + + class Meta: + model = FastQC + fields = "__all__" + + +class IlluminaFlowcellQCSerializer(SeqAutoViewMixin, serializers.ModelSerializer): + sample_sheet = SampleSheetLookupSerializer() + + class Meta: + model = IlluminaFlowcellQC + #fields = "__all__" + exclude = ("sequencing_run", ) # Already part of sample_sheet + + +class QCSerializer(SeqAutoViewMixin, serializers.ModelSerializer): + # Instead of dealing with all the bam/vcf etc - we'll just deal with sequencing_sample and + # assume we're using the latest ones associated with that + sequencing_sample = serializers.SerializerMethodField(read_only=True) + bam_file = BamFilePathSerializer() # These end up really big, think we just want to pass PKs + vcf_file = VCFFilePathSerializer() + + class Meta: + model = QC + fields = ("sequencing_sample", "bam_file", "vcf_file") + + def get_sequencing_sample(self, instance): + try: + ss = instance.bam_file.unaligned_reads.sequencing_sample + serializer = SequencingSampleLookupSerializer(ss) + return serializer.data + except SequencingSample.DoesNotExist: + return None + + +class QCGeneListSerializer(SeqAutoViewMixin, serializers.ModelSerializer): + qc = QCSerializer() + sample_gene_list = SampleGeneListSerializer() + + class Meta: + model = QCGeneList + fields = ("path", "qc", "sample_gene_list") + + +class QCGeneCoverageSerializer(SeqAutoViewMixin, serializers.ModelSerializer): + qc = QCSerializer() + gene_coverage_collection = GeneCoverageCollectionSerializer() + + class Meta: + model = QCGeneCoverage + fields = ("path", "qc", "gene_coverage_collection") + + +class QCExecSummarySerializer(SeqAutoViewMixin, serializers.ModelSerializer): + qc = QCSerializer() + + class Meta: + model = QCExecSummary + exclude = ('gene_list', ) diff --git a/seqauto/serializers/seqauto_serializers.py b/seqauto/serializers/seqauto_serializers.py new file mode 100644 index 000000000..c50df8859 --- /dev/null +++ b/seqauto/serializers/seqauto_serializers.py @@ -0,0 +1,34 @@ +import numpy as np +from rest_framework import serializers + +from genes.serializers import TranscriptSerializer, GeneSymbolSerializer, \ + TranscriptVersionSerializer +from seqauto.models import GoldCoverageSummary, GoldReference +from seqauto.serializers.enrichment_kit_serializers import EnrichmentKitSummarySerializer + + +class GoldReferenceSerializer(serializers.ModelSerializer): + enrichment_kit = EnrichmentKitSummarySerializer() + + class Meta: + model = GoldReference + fields = ('enrichment_kit', 'created') + + +class GoldCoverageSummarySerializer(serializers.ModelSerializer): + gold_reference = GoldReferenceSerializer() + gene_symbol = GeneSymbolSerializer() + transcript = TranscriptSerializer() + transcript_version = TranscriptVersionSerializer() + standard_error = serializers.SerializerMethodField() + + class Meta: + model = GoldCoverageSummary + fields = '__all__' + + def get_standard_error(self, obj): + """ This can occasionally be NaN which isn't valid JSON """ + standard_error = obj.standard_error + if np.isnan(standard_error): + standard_error = -1 + return standard_error diff --git a/seqauto/serializers/sequencing_serializers.py b/seqauto/serializers/sequencing_serializers.py new file mode 100644 index 000000000..3447d1593 --- /dev/null +++ b/seqauto/serializers/sequencing_serializers.py @@ -0,0 +1,323 @@ +import logging + +from rest_framework import serializers + +from seqauto.models import Sequencer, Experiment, VariantCaller, SequencingRun, SequencerModel, SampleSheet, \ + SequencingSampleData, SequencingSample, UnalignedReads, Flagstats, FastQC, SampleSheetCombinedVCFFile, VCFFile, \ + BamFile, Fastq, Aligner +from seqauto.serializers import EnrichmentKitSerializer +from snpdb.models import Manufacturer, DataState + + +class SequencerModelSerializer(serializers.ModelSerializer): + manufacturer = serializers.StringRelatedField() + data_naming_convention = serializers.SerializerMethodField() + + class Meta: + model = SequencerModel + fields = "__all__" + + def create(self, validated_data): + model = validated_data.get('model') + manufacturer = validated_data.get('manufacturer') + manufacturer, _ = Manufacturer.objects.get_or_create(name=manufacturer) + + # Check if the object already exists + instance, _created = SequencerModel.objects.get_or_create( + model=model, + defaults={ + "manufacturer": manufacturer, + "data_naming_convention": model.data_naming_convention, + } + ) + return instance # Return the existing or new instance + + def get_data_naming_convention(self, obj): + return obj.get_data_naming_convention_display() + + +class SequencerSerializer(serializers.ModelSerializer): + sequencer_model = SequencerModelSerializer() + + class Meta: + model = Sequencer + fields = "__all__" + + def create(self, validated_data): + name = validated_data.get('name') + sequencer_model = validated_data.get('sequencer_model') + logging.info("sequencer_model=%s", sequencer_model) + + instance, _created = Sequencer.objects.get_or_create( + name=name, + defaults={ + "sequencer_model": sequencer_model, + } + ) + return instance + + +class ExperimentSerializer(serializers.ModelSerializer): + class Meta: + model = Experiment + fields = ["name"] + + def create(self, validated_data): + name = validated_data.get('name') + instance, _created = Experiment.objects.get_or_create( + name=name + ) + return instance + + +class AlignerSerializer(serializers.ModelSerializer): + class Meta: + model = Aligner + fields = "__all__" + + def create(self, validated_data): + name = validated_data.get('name') + version = validated_data.get('version') + + instance, _created = Aligner.objects.get_or_create( + name=name, + version=version + ) + return instance + + +class VariantCallerSerializer(serializers.ModelSerializer): + class Meta: + model = VariantCaller + fields = "__all__" + + def create(self, validated_data): + name = validated_data.get('name') + version = validated_data.get('version') + run_params = validated_data.get('run_params') + + instance, _created = VariantCaller.objects.get_or_create( + name=name, + version=version, + defaults={"run_params": run_params} + ) + return instance + + +class SeqAutoViewMixin: + """ + This sets SeqAutoRecord.data_state to COMPLETED for anything created via API + + SeqAutoRecord.data_state represents eg whether the file exists on disk or has been deleted + or we expect it, and it's not available yet. + + Now we're moving to an API, I think we should just have the SeqAuto records match the disk + and be updated via clients, or just be added and then if they are deleted we don't care + + TODO: We should consider removing the data_state field + """ + def set_data_state_complete(self, validated_data): + validated_data['data_state'] = DataState.COMPLETE + + def create(self, validated_data): + self.set_data_state_complete(validated_data) + return super().create(validated_data) + + def update(self, instance, validated_data): + self.set_data_state_complete(validated_data) + return super().update(instance, validated_data) + + +class SequencingRunSerializer(SeqAutoViewMixin, serializers.ModelSerializer): + sequencer = SequencerSerializer() + experiment = ExperimentSerializer() + enrichment_kit = EnrichmentKitSerializer() + + class Meta: + model = SequencingRun + fields = ("name", "date", "sequencer", "gold_standard", "bad", "hidden", "experiment", "enrichment_kit", "has_basecalls", "has_interop") + + def create(self, validated_data): + name = validated_data.get('name') + instance, _created = SequencingRun.objects.get_or_create( + name=name, + defaults=validated_data + ) + return instance + + +class SequencingSampleDataSerializer(serializers.ModelSerializer): + # sequencing_run = SequencingRunSerializer() + + class Meta: + model = SequencingSampleData + fields = ("column", "value") + + +class SampleSheetLookupSerializer(serializers.Serializer): + """ This is when we want to refer to it in related objects in a minimal way + The samplesheet MUST have already been created """ + sequencing_run = serializers.CharField() + hash = serializers.CharField() + + def validate(self, attrs): + sequencing_run = attrs.get('sequencing_run') + hash = attrs.get('hash') + + try: + return SampleSheet.objects.get( + sequencing_run__name=sequencing_run, + hash=hash, + ) + except SampleSheet.DoesNotExist: + raise serializers.ValidationError("SampleSheet not found.") + + +class SequencingSampleLookupSerializer(serializers.Serializer): + """ This is when we want to refer to it in related objects in a minimal way """ + sample_sheet = SampleSheetLookupSerializer() + sample_name = serializers.CharField() + + +class SequencingSampleSerializer(serializers.ModelSerializer): + """ This is when we want the whole object as a dict """ + sequencingsampledata_set = SequencingSampleDataSerializer(many=True, required=False) + enrichment_kit = EnrichmentKitSerializer() + + class Meta: + model = SequencingSample + fields = ['sample_id', 'sample_name', 'sample_project', 'sample_number', 'lane', 'barcode', 'enrichment_kit', 'is_control', 'failed', 'sequencingsampledata_set'] + + +class SampleSheetSerializer(SeqAutoViewMixin, serializers.ModelSerializer): + sequencing_run = SequencingRunSerializer() + sequencingsample_set = SequencingSampleSerializer(many=True) + + class Meta: + model = SampleSheet + fields = ("path", "sequencing_run", "file_last_modified", "hash", "sequencingsample_set") + + @staticmethod + def _create_sequencing_samples(sample_sheet, sequencing_samples_data): + for sample_data in sequencing_samples_data: + sample_data_data = sample_data.pop('sample_data', []) + sequencing_sample = SequencingSample.objects.create(sample_sheet=sample_sheet, **sample_data) + for data in sample_data_data: + SequencingSampleData.objects.create(sequencing_sample=sequencing_sample, **data) + + def create(self, validated_data): + sequencing_samples_data = validated_data.pop('sequencing_samples') + sample_sheet, _created = SampleSheet.objects.update_or_create( + sequencing_run=validated_data["sequencing_run"], + hash=validated_data["hash"], + defaults=validated_data, + ) + self._create_sequencing_samples(sample_sheet, sequencing_samples_data) + return sample_sheet + + def update(self, instance, validated_data): + sequencing_samples_data = validated_data.pop('sequencing_samples') + + instance = super().update(instance, validated_data) + # Clear existing samples and add the new ones + instance.sequencing_samples.all().delete() + self._create_sequencing_samples(instance, sequencing_samples_data) + return instance + + +class FastqSerializer(SeqAutoViewMixin, serializers.ModelSerializer): + class Meta: + model = Fastq + fields = ("path", "name", "read") + + +class UnalignedReadsSerializer(serializers.ModelSerializer): + """ UnalignedReads is a joiner class - not represented by a file thus not SeqAutoRecord """ + sequencing_sample = SequencingSampleLookupSerializer() + fastq_r1 = FastqSerializer() + fastq_r2 = FastqSerializer() + + class Meta: + model = UnalignedReads + fields = "__all__" + + def create(self, validated_data): + sequencing_sample = validated_data.pop('sequencing_sample') + sequencing_run = sequencing_sample.sequencing_run + unaligned_reads_kwargs = {} + + for field_name in ["fastq_r1", "fastq_r2"]: + if fastq_data := validated_data.pop(field_name): + fastq = Fastq.objects.update_or_create(sequencing_sample=sequencing_sample, + defaults=fastq_data) + unaligned_reads_kwargs[field_name] = fastq + + instance, _created = UnalignedReads.objects.update_or_create(sequencing_run=sequencing_run, + sequencing_sample=sequencing_sample, + defaults=unaligned_reads_kwargs) + return instance + + +class FlagstatsSerializer(SeqAutoViewMixin, serializers.ModelSerializer): + class Meta: + model = Flagstats + fields = ("total", "read1", "read2", "mapped", "properly_paired") + + +class BamFilePathSerializer(serializers.ModelSerializer): + class Meta: + model = BamFile + fields = ("path", ) + + +class BamFileSerializer(SeqAutoViewMixin, serializers.ModelSerializer): + unaligned_reads = UnalignedReadsSerializer() + aligner = AlignerSerializer() + flagstats = FlagstatsSerializer() # 1-to-1 field + + class Meta: + model = BamFile + fields = ("path", "unaligned_reads", "name", "aligner", "flagstats") + + def create(self, validated_data): + flagstats_data = validated_data.pop('flagstats', None) + bam_file = BamFile.objects.create(**validated_data) + + if flagstats_data: + Flagstats.objects.create(bam_file=bam_file, **flagstats_data) + + return bam_file + + def update(self, instance, validated_data): + flagstats_data = validated_data.pop('flagstats', None) + instance = super().update(instance, validated_data) + if flagstats_data: + flagstats_instance = getattr(instance, 'flagstats', None) + flagstats_serializer = FlagstatsSerializer(instance=flagstats_instance, data=flagstats_data) + flagstats_serializer.is_valid(raise_exception=True) + flagstats_serializer.save(bam_file=instance) + return instance + + +class VCFFilePathSerializer(serializers.ModelSerializer): + class Meta: + model = VCFFile + fields = ("path", ) + + +class VCFFileSerializer(SeqAutoViewMixin, serializers.ModelSerializer): + bam_file = BamFileSerializer() + variant_caller = VariantCallerSerializer() + + class Meta: + model = VCFFile + fields = ("path", "bam_file", "variant_caller") + + +class SampleSheetCombinedVCFFileSerializer(SeqAutoViewMixin, serializers.ModelSerializer): + sample_sheet = SampleSheetLookupSerializer() + variant_caller = VariantCallerSerializer() + + class Meta: + model = SampleSheetCombinedVCFFile + fields = ("path", "sample_sheet", "variant_caller") diff --git a/seqauto/test_data/clinical_hg38/idt_exome/Exome_20_022_200920_NB501009_0410_AHNLYFBGXG/SampleSheet.csv b/seqauto/test_data/clinical_hg38/idt_exome/Exome_20_022_200920_NB501009_0410_AHNLYFBGXG/SampleSheet.csv index 789adbf86..8a6e18d3b 100644 --- a/seqauto/test_data/clinical_hg38/idt_exome/Exome_20_022_200920_NB501009_0410_AHNLYFBGXG/SampleSheet.csv +++ b/seqauto/test_data/clinical_hg38/idt_exome/Exome_20_022_200920_NB501009_0410_AHNLYFBGXG/SampleSheet.csv @@ -1,4 +1,4 @@ -FCID,Lane,SampleID,SampleRef,Index,Description,Control,Recipe,Operator,SampleProject,ProjectType,Group,Bioinformatician,E-mail,Adapter,Priming,Notes,conc pM -test_hiseq_flowcell,3,hiseq_sample1,hg19,GCCAAT,Foo,N,,XX,SampleProject,WES,tau,,foo@bar.com,KAPA,,,16 -test_hiseq_flowcell,3,hiseq_sample2,hg19,CAGATC,Foo,N,,XX,SampleProject,WES,tau,,foo@bar.com,KAPA,,,16 -#_IEMVERSION_3_TruSeq LT,#_IEMVERSION_3_TruSeq LT,,,,,,,,,,,,,,,, +FCID,Lane,SampleID,SampleRef,Index,Description,Control,Recipe,Operator,SampleProject,ProjectType,SAPOrderNumber,Group,Bioinformatician,E-mail,Adapter,Priming,Notes,conc pM +test_hiseq_flowcell,3,hiseq_sample1,hg19,GCCAAT,Foo,N,,XX,SampleProject,WES,SAP123456,tau,,foo@bar.com,KAPA,,,16 +test_hiseq_flowcell,3,hiseq_sample2,hg19,CAGATC,Foo,N,,XX,SampleProject,WES,SAP123457,tau,,foo@bar.com,KAPA,,,16 +#_IEMVERSION_3_TruSeq LT,#_IEMVERSION_3_TruSeq LT,,,,,,,,,,,,,,,,, diff --git a/seqauto/urls.py b/seqauto/urls.py index de4e54ee1..d685e0105 100644 --- a/seqauto/urls.py +++ b/seqauto/urls.py @@ -1,3 +1,5 @@ +from django.urls import include, path +from rest_framework import routers from rest_framework.urlpatterns import format_suffix_patterns from library.django_utils.jqgrid_view import JQGridView @@ -13,6 +15,10 @@ AssayGrid, AlignerGrid, VariantCallerGrid, VariantCallingPipelineGrid from seqauto.views import SequencerUpdate, LibraryUpdate, AssayUpdate, VariantCallerUpdate, \ AlignerUpdate, VariantCallingPipelineUpdate +from seqauto.views_rest import SequencingRunViewSet, EnrichmentKitViewSet, SequencerModelViewSet, SequencerViewSet, \ + ExperimentViewSet, VariantCallerViewSet, VCFFileViewSet, SampleSheetCombinedVCFFileViewSet, FastQCViewSet, \ + SampleSheetViewSet, IlluminaFlowcellQCViewSet, QCViewSet, QCGeneListViewSet, QCGeneCoverageViewSet, \ + QCExecSummaryViewSet from snpdb.views.datatable_view import DatabaseTableView from variantgrid.perm_path import perm_path @@ -105,10 +111,30 @@ name='sequencing_run_autocomplete'), ] -#router = routers.DefaultRouter() +router = routers.DefaultRouter() +router.register(r'api/v1/enrichment_kit', EnrichmentKitViewSet, basename='api_enrichment_kit') +router.register(r'api/v1/sequencer_model', SequencerModelViewSet, basename='api_sequencer_model') +router.register(r'api/v1/sequencer', SequencerViewSet, basename='api_sequencer') +router.register(r'api/v1/experiment', ExperimentViewSet, basename='api_experiment') +router.register(r'api/v1/variant_caller', VariantCallerViewSet, basename='api_variant_caller') +router.register(r'api/v1/sequencing_run', SequencingRunViewSet, basename='api_sequencing_run') +router.register(r'api/v1/sample_sheet', SampleSheetViewSet, basename='api_sample_sheet') +router.register(r'api/v1/vcf_file', VCFFileViewSet, basename='api_vcf_file') +router.register(r'api/v1/sample_sheet_combined_vcf_file', SampleSheetCombinedVCFFileViewSet, basename='api_sample_sheet_combined_vcf_file') +router.register(r'api/v1/fastqc', FastQCViewSet, basename='api_fastqc') +router.register(r'api/v1/illumina_flowcell_qc', IlluminaFlowcellQCViewSet, basename='api_illumina_flowcell_qc') +router.register(r'api/v1/qc_gene_list', QCGeneListViewSet, basename='api_qc_gene_list') +router.register(r'api/v1/qc_gene_coverage', QCGeneCoverageViewSet, basename='api_qc_gene_coverage') +router.register(r'api/v1/qc_exec_summary', QCExecSummaryViewSet, basename='api_qc_exec_summary') + +urlpatterns += [ + path('', include(router.urls)), +] + rest_urlpatterns = [ perm_path('api/view_enrichment_kit_summary/', views_rest.EnrichmentKitSummaryView.as_view(), name='api_view_enrichment_kit_summary'), - perm_path('api/view_enrichment_kit/', views_rest.EnrichmentKitView.as_view(), name='api_view_enrichment_kit'), + perm_path('api/view_enrichment_kit/', EnrichmentKitViewSet.as_view({'get': 'retrieve'}), + name='api_view_enrichment_kit'), # Deprecated, used for backwards compatability perm_path('api/enrichment_kit_gene_coverage//', views_rest.EnrichmentKitGeneCoverageView.as_view(), name='api_enrichment_kit_gene_coverage'), perm_path('api/enrichment_kit_gene_gold_coverage//', views_rest.EnrichmentKitGeneGoldCoverageView.as_view(), name='api_enrichment_kit_gene_gold_coverage'), perm_path('api/enrichment_kit_gene_gold_coverage_summary//', views_rest.GoldCoverageSummaryView.as_view(), name='api_enrichment_kit_gene_gold_coverage_summary'), diff --git a/seqauto/views_rest.py b/seqauto/views_rest.py index 4f2695fd0..c8ea5fc34 100644 --- a/seqauto/views_rest.py +++ b/seqauto/views_rest.py @@ -12,14 +12,22 @@ from rest_framework.generics import get_object_or_404, RetrieveAPIView from rest_framework.response import Response from rest_framework.views import APIView +from rest_framework.viewsets import ModelViewSet from genes.models import GeneVersion from genes.views.views import get_coverage_stats from library.constants import WEEK_SECS from library.utils import defaultdict_to_dict -from seqauto.models import GoldCoverageSummary, EnrichmentKit +from seqauto.models import GoldCoverageSummary, EnrichmentKit, SequencerModel, Sequencer, Experiment, VariantCaller, \ + SequencingRun, SampleSheet, VCFFile, SampleSheetCombinedVCFFile, FastQC, QCExecSummary, QCGeneCoverage, QCGeneList, \ + QC, IlluminaFlowcellQC from seqauto.serializers import EnrichmentKitSerializer, \ GoldCoverageSummarySerializer, EnrichmentKitSummarySerializer +from seqauto.serializers.seqauto_qc_serializers import FastQCSerializer, QCExecSummarySerializer, \ + QCGeneCoverageSerializer, QCGeneListSerializer, QCSerializer, IlluminaFlowcellQCSerializer +from seqauto.serializers.sequencing_serializers import SequencerModelSerializer, SequencerSerializer, \ + ExperimentSerializer, VariantCallerSerializer, SequencingRunSerializer, SampleSheetSerializer, VCFFileSerializer, \ + SampleSheetCombinedVCFFileSerializer class EnrichmentKitSummaryView(RetrieveAPIView): @@ -31,12 +39,81 @@ def get_queryset(self): return EnrichmentKit.objects.all() -class EnrichmentKitView(RetrieveAPIView): +class EnrichmentKitViewSet(ModelViewSet): + queryset = EnrichmentKit.objects.all() serializer_class = EnrichmentKitSerializer lookup_field = 'pk' - def get_queryset(self): - return EnrichmentKit.objects.all() + +class SequencerModelViewSet(ModelViewSet): + queryset = SequencerModel.objects.all() + serializer_class = SequencerModelSerializer + + +class SequencerViewSet(ModelViewSet): + queryset = Sequencer.objects.all() + serializer_class = SequencerSerializer + + +class ExperimentViewSet(ModelViewSet): + queryset = Experiment.objects.all() + serializer_class = ExperimentSerializer + + +class VariantCallerViewSet(ModelViewSet): + queryset = VariantCaller.objects.all() + serializer_class = VariantCallerSerializer + + +class SequencingRunViewSet(ModelViewSet): + queryset = SequencingRun.objects.filter(hidden=False) + serializer_class = SequencingRunSerializer + + +class SampleSheetViewSet(ModelViewSet): + queryset = SampleSheet.objects.all() + serializer_class = SampleSheetSerializer + + +class VCFFileViewSet(ModelViewSet): + queryset = VCFFile.objects.all() + serializer_class = VCFFileSerializer + + +class SampleSheetCombinedVCFFileViewSet(ModelViewSet): + queryset = SampleSheetCombinedVCFFile.objects.all() + serializer_class = SampleSheetCombinedVCFFileSerializer + + +class FastQCViewSet(ModelViewSet): + queryset = FastQC.objects.all() + serializer_class = FastQCSerializer + + +class IlluminaFlowcellQCViewSet(ModelViewSet): + queryset = IlluminaFlowcellQC.objects.all() + serializer_class = IlluminaFlowcellQCSerializer + + +class QCViewSet(ModelViewSet): + queryset = QC.objects.all() + serializer_class = QCSerializer + + +class QCGeneListViewSet(ModelViewSet): + queryset = QCGeneList.objects.all() + serializer_class = QCGeneListSerializer + + +class QCGeneCoverageViewSet(ModelViewSet): + queryset = QCGeneCoverage.objects.all() + serializer_class = QCGeneCoverageSerializer + + +class QCExecSummaryViewSet(ModelViewSet): + queryset = QCExecSummary.objects.all() + serializer_class = QCExecSummarySerializer + class EnrichmentKitGeneCoverageView(APIView): diff --git a/upload/urls.py b/upload/urls.py index 5481f3ed0..91a94f287 100644 --- a/upload/urls.py +++ b/upload/urls.py @@ -1,9 +1,12 @@ +from django.urls import path + from library.django_utils.jqgrid_view import JQGridView from snpdb.views.datatable_view import DatabaseTableView from upload.grids import UploadPipelineModifiedVariantsGrid, UploadPipelineSkippedAnnotationGrid, \ UploadStepColumns from upload.views import views from upload.views.views import view_upload_step_detail +from upload.views.views_rest import APIFileUploadView from variantgrid.perm_path import perm_path urlpatterns = [ @@ -27,4 +30,8 @@ perm_path('jfu_upload/', views.jfu_upload, name='jfu_upload'), perm_path('jfu_delete/', views.jfu_upload_delete, name='jfu_delete'), perm_path('uploaded_file/download/', views.DownloadUploadedFile.as_view(), name='download_uploaded_file'), + + # APIs - Django REST framework + perm_path('api/v1/file_upload', APIFileUploadView.as_view(), name='api_file_upload'), + ] diff --git a/upload/views/views.py b/upload/views/views.py index 70e228d0c..e50ed30d2 100644 --- a/upload/views/views.py +++ b/upload/views/views.py @@ -49,7 +49,7 @@ def get_icon_for_uploaded_file_status(status): return None -def uploadedfile_dict(uploaded_file): +def uploadedfile_dict(uploaded_file) -> dict: try: size = uploaded_file.uploaded_file.size except: @@ -104,6 +104,23 @@ def uploadedfile_dict(uploaded_file): return data +def handle_file_upload(user, django_uploaded_file) -> UploadedFile: + original_filename = django_uploaded_file._name + kwargs = { + "name": original_filename, + "uploaded_file": django_uploaded_file, + "import_source": ImportSource.WEB_UPLOAD, + "user": user + } + uploaded_file = UploadedFile.objects.create(**kwargs) + # Save 1st to actually create file (need to open handling unicode) + uploaded_file.file_type = get_uploaded_file_type(uploaded_file, original_filename) + uploaded_file.save() + + if uploaded_file.file_type: + upload_processing.process_uploaded_file(uploaded_file) + return uploaded_file + @require_POST def jfu_upload(request): # The assumption here is that jQuery File Upload @@ -121,23 +138,9 @@ def jfu_upload(request): raise ValueError(message) django_uploaded_file = upload_receive(request) - original_filename = django_uploaded_file._name - kwargs = {"name": original_filename, - "uploaded_file": django_uploaded_file, - "import_source": ImportSource.WEB_UPLOAD, - "user": request.user} - - uploaded_file = UploadedFile.objects.create(**kwargs) - # Save 1st to actually create file (need to open handling unicode) - uploaded_file.file_type = get_uploaded_file_type(uploaded_file, original_filename) - uploaded_file.save() - - file_dict = {} - - if uploaded_file.file_type: - upload_processing.process_uploaded_file(uploaded_file) + uploaded_file = handle_file_upload(request.user, django_uploaded_file) - file_dict.update(uploadedfile_dict(uploaded_file)) + file_dict = uploadedfile_dict(uploaded_file) except Exception as e: logging.error(e) log_traceback() diff --git a/upload/views/views_rest.py b/upload/views/views_rest.py new file mode 100644 index 000000000..5c15d67a3 --- /dev/null +++ b/upload/views/views_rest.py @@ -0,0 +1,21 @@ +from http import HTTPStatus + +from django.http import JsonResponse +from django.utils.decorators import method_decorator +from django.views.decorators.csrf import csrf_exempt + +from rest_framework.views import APIView + +from upload.views.views import handle_file_upload + + +@method_decorator(csrf_exempt, name='dispatch') +class APIFileUploadView(APIView): + """ Re-implemented uploads in DRF so we can use API tokens for all client work """ + def post(self, request, *args, **kwargs): + try: + django_uploaded_file = request.FILES['file'] + uploaded_file = handle_file_upload(request.user, django_uploaded_file) + return JsonResponse({"uploaded_file_id": uploaded_file.pk}) + except Exception as e: + return JsonResponse({"error": str(e)}, status=HTTPStatus.INTERNAL_SERVER_ERROR) diff --git a/variantgrid/settings/components/default_settings.py b/variantgrid/settings/components/default_settings.py index 59544af7c..6ad96bd10 100644 --- a/variantgrid/settings/components/default_settings.py +++ b/variantgrid/settings/components/default_settings.py @@ -721,6 +721,12 @@ CACHE_GENERATED_FILES = True REST_FRAMEWORK = { + # NOTE: Middleware is run first - so GlobalLoginRequiredMiddleware will reject tokens w/o logins + # before DRF even sees it. You need to add your APIs to PUBLIC_PATHS + 'DEFAULT_AUTHENTICATION_CLASSES': [ + 'rest_framework.authentication.SessionAuthentication', + 'rest_framework.authentication.TokenAuthentication', + ], 'DEFAULT_PERMISSION_CLASSES': ( 'rest_framework.permissions.IsAuthenticated', ), @@ -780,6 +786,7 @@ 'leaflet', "psqlextra", 'rest_framework', + 'rest_framework.authtoken', 'termsandconditions', 'crispy_forms', # used to make bootstrap compatible forms 'crispy_bootstrap4', @@ -908,7 +915,9 @@ PUBLIC_PATHS = [ r'^/accounts/.*', # allow public access to all django registration views, r'^/oidc/.*', # all oidc URLs - r'^/classification/api/.*' # REST framework used by command line tools + r'^/classification/api/.*', # REST framework used by command line tools + r'^/seqauto/api/.*', + r'^/upload/api/.*', ] # Both need to be set to enable - and use get_secret in server settings files to keep out of source control