diff --git a/blast/api.py b/blast/api.py index 2dc155a3d..20570346b 100644 --- a/blast/api.py +++ b/blast/api.py @@ -1,4 +1,9 @@ from rest_framework import renderers +from rest_framework import viewsets +from app.models import Organism +from blast.models import SequenceType, BlastDb +from blast.serializers import OrganismSerializer, BlastDbSerializer, SequenceTypeSerializer + class FASTARenderer(renderers.BaseRenderer): media_type = 'text/plain' @@ -12,11 +17,6 @@ def render(self, data, media_type=None, renderer_context=None): else: return '' -from rest_framework.renderers import JSONRenderer, BrowsableAPIRenderer -from rest_framework import viewsets -from django.contrib.auth.models import User -from .models import * -from .serializers import * class OrganismViewSet(viewsets.ReadOnlyModelViewSet): """ @@ -26,6 +26,7 @@ class OrganismViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = OrganismSerializer lookup_field = 'short_name' + class SequenceTypeViewSet(viewsets.ReadOnlyModelViewSet): """ Retrieve sequence types. @@ -34,6 +35,7 @@ class SequenceTypeViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = SequenceTypeSerializer lookup_field = 'dataset_type' + class BlastDbViewSet(viewsets.ReadOnlyModelViewSet): """ Retrieve BLAST databases. @@ -42,59 +44,3 @@ class BlastDbViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = BlastDbSerializer lookup_field = 'title' lookup_value_regex = '[^/]+' - - #@link() - #def sequence_set(self, request, title=None): - # empty_error = "Empty list and '%(class_name)s.allow_empty' is False." - # blastdb = self.get_object() - # object_list = self.filter_queryset(blastdb.sequence_set.all()) - - # # Default is to allow empty querysets. This can be altered by setting - # # `.allow_empty = False`, to raise 404 errors on empty querysets. - # if not self.allow_empty and not object_list: - # warnings.warn( - # 'The `allow_empty` parameter is deprecated. ' - # 'To use `allow_empty=False` style behavior, You should override ' - # '`get_queryset()` and explicitly raise a 404 on empty querysets.', - # DeprecationWarning - # ) - # class_name = self.__class__.__name__ - # error_msg = self.empty_error % {'class_name': class_name} - # raise Http404(error_msg) - - # # Switch between paginated or standard style responses - # page = self.paginate_queryset(object_list) - # if page is not None: - # serializer = PaginatedSequenceSerializer(instance=page, context={'request': request}) - # else: - # serializer = SequenceSerializer(object_list, many=True, context={'request': request}) - - # return Response(serializer.data) - -class SequenceViewSet(viewsets.ReadOnlyModelViewSet): - """ - Retrieve fasta sequences. - """ - queryset = Sequence.objects.all() - serializer_class = SequenceSerializer - renderer_classes = (JSONRenderer, BrowsableAPIRenderer, FASTARenderer) - lookup_field = 'id' - lookup_value_regex = '[^/]+' - -class BlastQueryRecordViewSet(viewsets.ReadOnlyModelViewSet): - """ - Retrieve tasks. - """ - queryset = BlastQueryRecord.objects.all() - serializer_class = BlastQueryRecordSerializer - lookup_field = 'task_id' - -class UserViewSet(viewsets.ReadOnlyModelViewSet): - """ - Retrieve users. - """ - queryset = User.objects.all() - serializer_class = UserSerializer - lookup_field = 'pk' - - diff --git a/blast/models.py b/blast/models.py index 4551eee37..aa2c93960 100755 --- a/blast/models.py +++ b/blast/models.py @@ -59,12 +59,10 @@ class BlastDb(models.Model): objects = BlastDbManager() organism = models.ForeignKey(app.models.Organism) type = models.ForeignKey(SequenceType) - # fasta_file = models.FileField(upload_to='blastdb') # upload file fasta_file = FileBrowseField('FASTA file path', max_length=200, directory='blast/db/', extensions='FASTA', format='FASTA') title = models.CharField(max_length=200, unique=True, help_text='This is passed into makeblast -title') # makeblastdb -title description = models.TextField(blank=True) # shown in blast db selection ui is_shown = models.BooleanField(default=None, help_text='Display this database in the BLAST submit form') # to temporarily remove from blast db selection ui - # sequence_count = models.PositiveIntegerField(null=True, blank=True) # number of sequences in this fasta # properties def fasta_file_exists(self): diff --git a/blast/serializers.py b/blast/serializers.py index 0e5dd3022..3e5dab95e 100644 --- a/blast/serializers.py +++ b/blast/serializers.py @@ -1,62 +1,36 @@ -from rest_framework import serializers -from django.contrib.auth.models import User +from rest_framework.serializers import HyperlinkedModelSerializer, ReadOnlyField, HyperlinkedRelatedField from app.models import Organism -from blast.models import SequenceType, BlastDb, Sequence, BlastQueryRecord +from blast.models import SequenceType, BlastDb -class OrganismSerializer(serializers.HyperlinkedModelSerializer): - url = serializers.HyperlinkedIdentityField(view_name='blast:organism-detail', lookup_field='short_name') +class OrganismSerializer(HyperlinkedModelSerializer): class Meta: model = Organism + fields = ('display_name', 'short_name', 'description', 'tax_id') -class SequenceTypeSerializer(serializers.HyperlinkedModelSerializer): - url = serializers.HyperlinkedIdentityField(view_name='blast:sequencetype-detail', lookup_field='dataset_type') - +class SequenceTypeSerializer(HyperlinkedModelSerializer): class Meta: model = SequenceType - - -class BlastDbSerializer(serializers.HyperlinkedModelSerializer): - url = serializers.HyperlinkedIdentityField(view_name='blast:blastdb-detail', lookup_field='title') - organism = serializers.HyperlinkedRelatedField(view_name='blast:organism-detail', lookup_field='short_name', read_only=True) - type = serializers.HyperlinkedRelatedField(view_name='blast:sequencetype-detail', lookup_field='dataset_type', read_only=True) - sequence_set = serializers.HyperlinkedRelatedField(view_name='blast:blastdb-sequence-set', lookup_field='title', read_only=True) - fasta_file_exists = serializers.Field() - blast_db_files_exists = serializers.Field() - sequence_set_exists = serializers.Field() - db_ready = serializers.Field() + fields = ('molecule_type', 'dataset_type') + + +class BlastDbSerializer(HyperlinkedModelSerializer): + organism = HyperlinkedRelatedField( + view_name='blast:organism-detail', + lookup_field='short_name', + read_only=True) + type = HyperlinkedRelatedField( + view_name='blast:sequencetype-detail', + lookup_field='dataset_type', + read_only=True) + fasta_file_exists = ReadOnlyField() + blast_db_files_exists = ReadOnlyField() + sequence_set_exists = ReadOnlyField() + db_ready = ReadOnlyField() class Meta: model = BlastDb - fields = ('url', 'organism', 'type', 'fasta_file', 'title', 'description', 'is_shown', 'fasta_file_exists', 'blast_db_files_exists', 'sequence_set_exists', 'db_ready', 'sequence_set', ) - - -class SequenceSerializer(serializers.HyperlinkedModelSerializer): - blast_db = serializers.HyperlinkedRelatedField(view_name='blast:blastdb-detail', lookup_field='title', read_only=True) - fasta_seq = serializers.ReadOnlyField() - - class Meta: - model = Sequence - fields = ('blast_db', 'id', 'length', 'seq_start_pos', 'seq_end_pos', 'modified_date', 'fasta_seq', ) - - -class BlastQueryRecordSerializer(serializers.HyperlinkedModelSerializer): - url = serializers.HyperlinkedIdentityField(view_name='blast:blastqueryrecord-detail', lookup_field='task_id') - - class Meta: - model = BlastQueryRecord - - -class UserSerializer(serializers.ModelSerializer): - url = serializers.HyperlinkedIdentityField(view_name='blast:user-detail', lookup_field='pk') - - class Meta: - model = User - fields = ('id', 'url',) - - -class UserBlastQueryRecordSerializer(serializers.ModelSerializer): - class Meta: - model = BlastQueryRecord - fields = ('task_id', 'enqueue_date', 'result_status') + fields = ('organism', 'type', 'fasta_file', 'title', 'description', + 'is_shown', 'fasta_file_exists', 'blast_db_files_exists', + 'sequence_set_exists', 'db_ready') diff --git a/blast/static/blast/scripts/blast-results.js b/blast/static/blast/scripts/blast-results.js index 6c1893731..fc74faedd 100755 --- a/blast/static/blast/scripts/blast-results.js +++ b/blast/static/blast/scripts/blast-results.js @@ -155,7 +155,7 @@ $(function () { // document ready var url_root = /(https?:\/\/.*(?:blast)*)\/task\//g.exec(document.URL)[1]; function get_fasta(sseqid) { // Returns a jqXHR or true if already in cache, for use with $.when - //http://localhost:8000/api/seq/gnl%7CLoxosceles_reclusa_transcript_v0.5.3%7CLREC000002-RA/?format=fasta + // base_url/blast/api/seq/gnl%7CLoxosceles_reclusa_transcript_v0.5.3%7CLREC000002-RA/?format=fasta if (sseqid in fasta_cache) return true; else if (sseqid in fasta_loading) diff --git a/blast/urls.py b/blast/urls.py index 4c112e89f..1236c8dc6 100644 --- a/blast/urls.py +++ b/blast/urls.py @@ -1,16 +1,6 @@ from django.conf.urls import url, include from django.conf import settings from blast import views -from blast.api import OrganismViewSet, SequenceTypeViewSet, BlastDbViewSet, SequenceViewSet, BlastQueryRecordViewSet, UserViewSet -from rest_framework import routers - -router = routers.DefaultRouter() -router.register(r'organism', OrganismViewSet) -router.register(r'seqtype', SequenceTypeViewSet) -router.register(r'blastdb', BlastDbViewSet) -router.register(r'seq', SequenceViewSet) -router.register(r'task', BlastQueryRecordViewSet) -router.register(r'user', UserViewSet) urlpatterns = [ # ex: /blast/ @@ -18,15 +8,13 @@ # url(r'^iframe$', views.create, {'iframe': True}, name='iframe'), # ex: /blast/5/ url(r'^task/(?P[0-9a-zA-Z]+)$', views.retrieve, name='retrieve'), - url(r'^task/(?P[0-9a-zA-Z]+)/status$', views.status, name='status'), + url(r'^task/(?P[0-9a-zA-Z]+)/status$', + views.status, + name='status'), # url(r'^read_gff3/(?P[0-9a-fA-F]*)/*(?P[\w\-\|.]*)/*$', views.read_gff3, name='read_gff3'), - url(r'^api/', include(router.urls)), - # url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')), - url(r'^user-tasks/(?P[0-9]+)$', views.user_tasks), + url(r'^api/seq/(?P[a-zA-Z0-9_-]+)/', views.get_seq, name='seq'), ] if settings.DEBUG: from blast import test_views - urlpatterns += [ - url(r'^test/$', test_views.test_main) - ] + urlpatterns += [url(r'^test/$', test_views.test_main)] diff --git a/blast/views.py b/blast/views.py index a40bfcff7..2ac6eba7c 100755 --- a/blast/views.py +++ b/blast/views.py @@ -1,13 +1,13 @@ from __future__ import absolute_import from django.shortcuts import render -from django.shortcuts import redirect +from django.shortcuts import redirect, get_object_or_404 from django.http import Http404 from django.http import HttpResponse from django.conf import settings from django.core.cache import cache from uuid import uuid4 from os import path, makedirs, chmod, remove -from blast.models import BlastQueryRecord, BlastDb +from blast.models import BlastQueryRecord, BlastDb, Sequence from blast.tasks import run_blast_task from datetime import datetime, timedelta from django.utils.timezone import localtime, now @@ -18,18 +18,41 @@ from itertools import groupby from multiprocessing import cpu_count from util.get_bin_name import get_bin_name +from rest_framework.renderers import JSONRenderer -blast_customized_options = {'blastn': ['max_target_seqs', 'evalue', 'word_size', 'reward', 'penalty', 'gapopen', 'gapextend', 'strand', 'low_complexity', 'soft_masking'], - 'tblastn': ['max_target_seqs', 'evalue', 'word_size', 'matrix', 'threshold', 'gapopen', 'gapextend', 'low_complexity', 'soft_masking'], - 'tblastx': ['max_target_seqs', 'evalue', 'word_size', 'matrix', 'threshold', 'strand', 'low_complexity', 'soft_masking'], - 'blastp': ['max_target_seqs', 'evalue', 'word_size', 'matrix', 'threshold', 'gapopen', 'gapextend', 'low_complexity', 'soft_masking'], - 'blastx': ['max_target_seqs', 'evalue', 'word_size', 'matrix', 'threshold', 'strand', 'gapopen', 'gapextend', 'low_complexity', 'soft_masking']} +blast_customized_options = { + 'blastn': [ + 'max_target_seqs', 'evalue', 'word_size', 'reward', 'penalty', + 'gapopen', 'gapextend', 'strand', 'low_complexity', 'soft_masking' + ], + 'tblastn': [ + 'max_target_seqs', 'evalue', 'word_size', 'matrix', 'threshold', + 'gapopen', 'gapextend', 'low_complexity', 'soft_masking' + ], + 'tblastx': [ + 'max_target_seqs', 'evalue', 'word_size', 'matrix', 'threshold', + 'strand', 'low_complexity', 'soft_masking' + ], + 'blastp': [ + 'max_target_seqs', 'evalue', 'word_size', 'matrix', 'threshold', + 'gapopen', 'gapextend', 'low_complexity', 'soft_masking' + ], + 'blastx': [ + 'max_target_seqs', 'evalue', 'word_size', 'matrix', 'threshold', + 'strand', 'gapopen', 'gapextend', 'low_complexity', 'soft_masking' + ] +} # blast_col_name = 'qseqid sseqid evalue qlen slen length nident mismatch positive gapopen gaps qstart qend sstart send bitscore qcovs qframe sframe' blast_col_name = 'qseqid sseqid pident length mismatch gapopen qstart qend sstart send evalue bitscore nident qcovs qlen slen qframe sframe' -blast_col_names_display = 'Query sequence ID,Subject sequence ID,Percentage of identical matches,Alignment length,Number of mismatches,Number of gap openings,Start of alignment in query,End of alignment in query,Start of alignment in subject,End of alignment in subject,Expect value,Bit score,Number of identical matches,Query coverage per subject,Query sequence length,Subject sequence length,Query frame,Subject frame'.split(',') +blast_col_names_display = 'Query sequence ID,Subject sequence ID,Percentage of identical matches,Alignment length,Number of mismatches,Number of gap openings,Start of alignment in query,End of alignment in query,Start of alignment in subject,End of alignment in subject,Expect value,Bit score,Number of identical matches,Query coverage per subject,Query sequence length,Subject sequence length,Query frame,Subject frame'.split( + ',') blast_info = { - 'col_types': ['str', 'str', 'float', 'int', 'int', 'int', 'int', 'int', 'int', 'int', 'float', 'float', 'int', 'int', 'int', 'int', 'int', 'int'], - 'col_names': blast_col_name.split(), + 'col_types': [ + 'str', 'str', 'float', 'int', 'int', 'int', 'int', 'int', 'int', 'int', + 'float', 'float', 'int', 'int', 'int', 'int', 'int', 'int' + ], + 'col_names': + blast_col_name.split(), 'ext': { '.0': '0', '.html': '0', @@ -43,25 +66,40 @@ def create(request, iframe=False): if request.method == 'GET': - blastdb_list = sorted([[db.type.dataset_type, db.type.get_molecule_type_display(), db.title, db.organism.display_name, db.description] for db in BlastDb.objects.select_related('organism').select_related('type').filter(is_shown=True) if db.db_ready()], key=lambda x: (x[3], x[1], x[0], x[2])) - blastdb_type_counts = dict([(k.lower().replace(' ', '_'), len(list(g))) for k, g in groupby(sorted(blastdb_list, key=lambda x: x[0]), key=lambda x: x[0])]) - return render(request, 'blast/main.html', { - 'title': 'BLAST Query', - 'blastdb_list': json.dumps(blastdb_list), - 'blastdb_type_counts': blastdb_type_counts, - 'iframe': iframe - }) + blastdb_list = sorted( + [[ + db.type.dataset_type, + db.type.get_molecule_type_display(), db.title, + db.organism.display_name, db.description + ] for db in BlastDb.objects.select_related('organism') + .select_related('type').filter(is_shown=True) if db.db_ready()], + key=lambda x: (x[3], x[1], x[0], x[2])) + blastdb_type_counts = dict([(k.lower().replace(' ', '_'), len( + list(g))) for k, g in groupby( + sorted(blastdb_list, key=lambda x: x[0]), key=lambda x: x[0])]) + return render( + request, 'blast/main.html', { + 'title': 'BLAST Query', + 'blastdb_list': json.dumps(blastdb_list), + 'blastdb_type_counts': blastdb_type_counts, + 'iframe': iframe + }) elif request.method == 'OPTIONS': return HttpResponse("OPTIONS METHOD NOT SUPPORTED", status=202) elif request.method == 'POST': # setup file paths - task_id = uuid4().hex # TODO: Create from hash of input to check for duplicate inputs - file_prefix = path.join(settings.MEDIA_ROOT, 'blast', 'task', task_id, task_id) + task_id = uuid4( + ).hex # TODO: Create from hash of input to check for duplicate inputs + file_prefix = path.join(settings.MEDIA_ROOT, 'blast', 'task', task_id, + task_id) query_filename = file_prefix + '.in' asn_filename = file_prefix + '.asn' if not path.exists(path.dirname(query_filename)): makedirs(path.dirname(query_filename)) - chmod(path.dirname(query_filename), Perm.S_IRWXU | Perm.S_IRWXG | Perm.S_IRWXO) # ensure the standalone dequeuing process can open files in the directory + chmod( + path.dirname(query_filename), + Perm.S_IRWXU | Perm.S_IRWXG | Perm.S_IRWXO + ) # ensure the standalone dequeuing process can open files in the directory bin_name = get_bin_name() # write query to file if 'query-file' in request.FILES: @@ -70,59 +108,102 @@ def create(request, iframe=False): query_f.write(chunk) elif 'query-sequence' in request.POST: with open(query_filename, 'wb') as query_f: - query_text = [x.encode('ascii','ignore').strip() for x in request.POST['query-sequence'].split('\n')] + query_text = [ + x.encode('ascii', 'ignore').strip() + for x in request.POST['query-sequence'].split('\n') + ] query_f.write('\n'.join(query_text)) else: - return render(request, 'blast/invalid_query.html', {'title': 'Invalid Query'}) + return render(request, 'blast/invalid_query.html', + {'title': 'Invalid Query'}) - if (path.getsize(query_filename) > int(settings.BLAST_QUERY_SIZE_MAX) * 1024): - return render(request, 'blast/invalid_query.html', {'title': 'Your query size is ' + str(path.getsize(query_filename)) + ' bytes, but exceeds our query size limit of ' + str(settings.BLAST_QUERY_SIZE_MAX) + ' kbytes, Please try again with a smaller query size.',}) + if (path.getsize(query_filename) > + int(settings.BLAST_QUERY_SIZE_MAX) * 1024): + return render( + request, 'blast/invalid_query.html', { + 'title': + 'Your query size is ' + str(path.getsize(query_filename)) + + ' bytes, but exceeds our query size limit of ' + str( + settings.BLAST_QUERY_SIZE_MAX) + + ' kbytes, Please try again with a smaller query size.', + }) - chmod(query_filename, Perm.S_IRWXU | Perm.S_IRWXG | Perm.S_IRWXO) # ensure the standalone dequeuing process can access the file + chmod(query_filename, Perm.S_IRWXU | Perm.S_IRWXG | Perm.S_IRWXO + ) # ensure the standalone dequeuing process can access the file # build blast command - db_list = ' '.join([db.fasta_file.path_full for db in BlastDb.objects.filter(title__in=set(request.POST.getlist('db-name'))) if db.db_ready()]) + db_list = ' '.join([ + db.fasta_file.path_full for db in BlastDb.objects.filter( + title__in=set(request.POST.getlist('db-name'))) + if db.db_ready() + ]) if not db_list: - return render(request, 'blast/invalid_query.html', {'title': 'Invalid Query',}) + return render(request, 'blast/invalid_query.html', { + 'title': 'Invalid Query', + }) # check if program is in list for security - if request.POST['program'] in ['blastn', 'tblastn', 'tblastx', 'blastp', 'blastx']: + if request.POST['program'] in [ + 'blastn', 'tblastn', 'tblastx', 'blastp', 'blastx' + ]: with open(query_filename, 'r') as f: qstr = f.read() - if(qstr.count('>') > int(settings.BLAST_QUERY_MAX)): + if (qstr.count('>') > int(settings.BLAST_QUERY_MAX)): query_cnt = str(qstr.count('>')) remove(query_filename) - return render(request, 'blast/invalid_query.html', - {'title': 'Your search includes ' + query_cnt + ' sequences, but blast allows a maximum of ' + str(settings.BLAST_QUERY_MAX) + ' sequences per submission.', }) + return render( + request, 'blast/invalid_query.html', { + 'title': + 'Your search includes ' + query_cnt + + ' sequences, but blast allows a maximum of ' + str( + settings.BLAST_QUERY_MAX) + + ' sequences per submission.', + }) # generate customized_options input_opt = [] max_target_seqs = request.POST.get('max_target_seqs', 50) - for blast_option in blast_customized_options[request.POST['program']]: + for blast_option in blast_customized_options[request.POST[ + 'program']]: if blast_option == 'low_complexity': if request.POST['program'] == 'blastn': - input_opt.extend(['-dust', request.POST['low_complexity']]) + input_opt.extend( + ['-dust', request.POST['low_complexity']]) else: - input_opt.extend(['-seg', request.POST['low_complexity']]) + input_opt.extend( + ['-seg', request.POST['low_complexity']]) else: - input_opt.extend(['-'+blast_option, request.POST[blast_option]]) + input_opt.extend( + ['-' + blast_option, request.POST[blast_option]]) - program_path = path.join(settings.BASE_DIR, 'blast', bin_name, request.POST['program']) + program_path = path.join(settings.BASE_DIR, 'blast', bin_name, + request.POST['program']) num_threads = '4' if cpu_count() >= 4 else str(cpu_count()) - args_list = [[program_path, '-query', query_filename, '-db', db_list, '-outfmt', '11', '-out', asn_filename, '-num_threads', num_threads]] + args_list = [[ + program_path, '-query', query_filename, '-db', db_list, + '-outfmt', '11', '-out', asn_filename, '-num_threads', + num_threads + ]] args_list[0].extend(input_opt) # convert to multiple formats - blast_formatter_path = path.join(settings.BASE_DIR, 'blast', bin_name, 'blast_formatter') + blast_formatter_path = path.join(settings.BASE_DIR, 'blast', + bin_name, 'blast_formatter') for ext, outfmt in blast_info['ext'].items(): - args = [blast_formatter_path, '-archive', asn_filename, '-outfmt', outfmt, '-out', file_prefix + ext] + args = [ + blast_formatter_path, '-archive', asn_filename, '-outfmt', + outfmt, '-out', file_prefix + ext + ] if ext == '.html': args.append('-html') if int(outfmt.split()[0]) > 4: args.extend(['-max_target_seqs', max_target_seqs]) else: - args.extend(['-num_descriptions', max_target_seqs, '-num_alignments', max_target_seqs]) + args.extend([ + '-num_descriptions', max_target_seqs, + '-num_alignments', max_target_seqs + ]) args_list.append(args) record = BlastQueryRecord() record.task_id = task_id @@ -137,7 +218,9 @@ def create(request, iframe=False): seq_count = qstr.count('>') if (seq_count == 0): seq_count = 1 - with open(path.join(path.dirname(file_prefix), 'status.json'), 'wb') as f: + with open( + path.join(path.dirname(file_prefix), 'status.json'), + 'wb') as f: json.dump({'status': 'pending', 'seq_count': seq_count}, f) run_blast_task.delay(task_id, args_list, file_prefix, blast_info) @@ -153,11 +236,15 @@ def retrieve(request, task_id='1'): try: r = BlastQueryRecord.objects.get(task_id=task_id) # if result is generated and not expired - if r.result_date and (r.result_date.replace(tzinfo=None) >= (datetime.utcnow()+ timedelta(days=-7))): + if r.result_date and (r.result_date.replace(tzinfo=None) >= + (datetime.utcnow() + timedelta(days=-7))): if r.result_status in ['SUCCESS', 'NO_GFF']: - file_prefix = path.join(settings.MEDIA_ROOT, 'blast', 'task', task_id, task_id) + file_prefix = path.join(settings.MEDIA_ROOT, 'blast', 'task', + task_id, task_id) results_info = '' - with open(path.join(settings.MEDIA_ROOT, 'blast', 'task', task_id, 'info.json'), 'rb') as f: + with open( + path.join(settings.MEDIA_ROOT, 'blast', 'task', + task_id, 'info.json'), 'rb') as f: results_info = f.read() results_data = '' with open(file_prefix + '.json', 'rb') as f: @@ -168,42 +255,54 @@ def retrieve(request, task_id='1'): results_col_names = blast_info['col_names'] results_col_names_display = blast_col_names_display results_col_names = ['blastdb'] + results_col_names - results_col_names_display = ['BLAST Database Title'] + results_col_names_display + results_col_names_display = ['BLAST Database Title' + ] + results_col_names_display return render( - request, - 'blast/results.html', { - 'title': 'BLAST Result', - 'results_col_names': json.dumps(results_col_names), - 'results_col_names_display': json.dumps(results_col_names_display), - 'results_detail': json.dumps(results_detail), - 'results_data': results_data, - 'results_info': results_info, - 'task_id': task_id, + request, 'blast/results.html', { + 'title': + 'BLAST Result', + 'results_col_names': + json.dumps(results_col_names), + 'results_col_names_display': + json.dumps(results_col_names_display), + 'results_detail': + json.dumps(results_detail), + 'results_data': + results_data, + 'results_info': + results_info, + 'task_id': + task_id, }) else: # if .tsv file size is 0, no hits found - return render(request, 'blast/results_not_existed.html', { - 'title': 'No Hits Found', - 'isNoHits': True, - 'isExpired': False, - }) + return render( + request, 'blast/results_not_existed.html', { + 'title': 'No Hits Found', + 'isNoHits': True, + 'isExpired': False, + }) else: - enqueue_date = r.enqueue_date.astimezone(timezone('US/Eastern')).strftime('%d %b %Y %X %Z') + enqueue_date = r.enqueue_date.astimezone( + timezone('US/Eastern')).strftime('%d %b %Y %X %Z') if r.dequeue_date: - dequeue_date = r.dequeue_date.astimezone(timezone('US/Eastern')).strftime('%d %b %Y %X %Z') + dequeue_date = r.dequeue_date.astimezone( + timezone('US/Eastern')).strftime('%d %b %Y %X %Z') else: dequeue_date = None # result is exipired isExpired = False - if r.result_date and (r.result_date.replace(tzinfo=None) < (datetime.utcnow()+ timedelta(days=-7))): + if r.result_date and (r.result_date.replace(tzinfo=None) < + (datetime.utcnow() + timedelta(days=-7))): isExpired = True - return render(request, 'blast/results_not_existed.html', { - 'title': 'Query Submitted', - 'task_id': task_id, - 'isExpired': isExpired, - 'enqueue_date': enqueue_date, - 'dequeue_date': dequeue_date, - 'isNoHits': False, - }) + return render( + request, 'blast/results_not_existed.html', { + 'title': 'Query Submitted', + 'task_id': task_id, + 'isExpired': isExpired, + 'enqueue_date': enqueue_date, + 'dequeue_date': dequeue_date, + 'isNoHits': False, + }) except Exception: raise Http404 @@ -212,7 +311,9 @@ def read_gff3(request, task_id, dbname): output = '##gff-version 3\n' try: if request.method == 'GET': - with open(path.join(settings.MEDIA_ROOT, 'blast', 'task', task_id, dbname) + '.gff', 'rb') as f: + with open( + path.join(settings.MEDIA_ROOT, 'blast', 'task', task_id, + dbname) + '.gff', 'rb') as f: output = f.read() finally: return HttpResponse(output) @@ -220,7 +321,8 @@ def read_gff3(request, task_id, dbname): def status(request, task_id): if request.method == 'GET': - status_file_path = path.join(settings.MEDIA_ROOT, 'blast', 'task', task_id, 'status.json') + status_file_path = path.join(settings.MEDIA_ROOT, 'blast', 'task', + task_id, 'status.json') status = {'status': 'unknown'} if path.isfile(status_file_path): with open(status_file_path, 'rb') as f: @@ -235,7 +337,8 @@ def status(request, task_id): break statusdata['num_preceding'] = num_preceding elif statusdata['status'] == 'running': - asn_path = path.join(settings.MEDIA_ROOT, 'blast', 'task', task_id, (task_id+'.asn')) + asn_path = path.join(settings.MEDIA_ROOT, 'blast', 'task', + task_id, (task_id + '.asn')) if path.isfile(asn_path): with open(asn_path, 'r') as asn_f: astr = asn_f.read() @@ -249,24 +352,6 @@ def status(request, task_id): return HttpResponse('Invalid Post') -# to-do: integrate with existing router of restframework -from rest_framework.renderers import JSONRenderer -from .serializers import UserBlastQueryRecordSerializer -class JSONResponse(HttpResponse): - """ - An HttpResponse that renders its content into JSON. - """ - def __init__(self, data, **kwargs): - content = JSONRenderer().render(data) - kwargs['content_type'] = 'application/json' - super(JSONResponse, self).__init__(content, **kwargs) - -def user_tasks(request, user_id): - """ - Return tasks performed by the user. - """ - if request.method == 'GET': - records = BlastQueryRecord.objects.filter(user__id=user_id, is_shown=True, result_date__gt=(localtime(now())+ timedelta(days=-7))) - serializer = UserBlastQueryRecordSerializer(records, many=True) - return JSONResponse(serializer.data) - +def get_seq(request, seq_id): + seq = get_object_or_404(Sequence, id=seq_id) + return HttpResponse(seq.fasta_seq(), content_type='text/plain') diff --git a/clustal/serializers.py b/clustal/serializers.py deleted file mode 100644 index bf1944ead..000000000 --- a/clustal/serializers.py +++ /dev/null @@ -1,14 +0,0 @@ -from rest_framework import serializers -from django.contrib.auth.models import User -from .models import ClustalQueryRecord - -class UserSerializer(serializers.ModelSerializer): - url = serializers.HyperlinkedIdentityField(view_name='user-detail', lookup_field='pk') - class Meta: - model = User - fields = ('id',) - -class UserClustalQueryRecordSerializer(serializers.ModelSerializer): - class Meta: - model = ClustalQueryRecord - fields = ('task_id', 'enqueue_date', 'result_status') diff --git a/clustal/urls.py b/clustal/urls.py index c1af49445..3ba4305c2 100644 --- a/clustal/urls.py +++ b/clustal/urls.py @@ -1,15 +1,13 @@ from django.conf.urls import url from clustal import views -# from .api import * urlpatterns = [ # ex: /clustal/ url(r'^$', views.create, name='create'), # ex: /clustal/5/ url(r'^task/(?P[0-9a-zA-Z]+)$', views.retrieve, name='retrieve'), - url(r'^task/(?P[0-9a-zA-Z]+)/status$', views.status, name='status'), - # url(r'^api/', include(router.urls)), - # url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')), - url('^user-tasks/(?P[0-9]+)$', views.user_tasks), + url(r'^task/(?P[0-9a-zA-Z]+)/status$', + views.status, + name='status'), url(r'^manual/$', views.manual, name='manual'), ] diff --git a/clustal/views.py b/clustal/views.py index 35c752c67..c9aa4d67e 100644 --- a/clustal/views.py +++ b/clustal/views.py @@ -20,7 +20,7 @@ def manual(request): ''' Manual page of Clustal ''' - return render(request, 'clustal/manual.html', {'title':'Clustal Manual'}) + return render(request, 'clustal/manual.html', {'title': 'Clustal Manual'}) def create(request): @@ -47,30 +47,42 @@ def create(request): query_filename = '' if 'query-file' in request.FILES: - query_filename = path.join(settings.MEDIA_ROOT, 'clustal', 'task', task_id, request.FILES['query-file'].name) + query_filename = path.join(settings.MEDIA_ROOT, 'clustal', 'task', + task_id, + request.FILES['query-file'].name) with open(query_filename, 'wb') as query_f: for chunk in request.FILES['query-file'].chunks(): query_f.write(chunk) elif 'query-sequence' in request.POST and request.POST['query-sequence']: - query_filename = path.join(settings.MEDIA_ROOT, 'clustal', 'task', task_id, task_id + '.in') + query_filename = path.join(settings.MEDIA_ROOT, 'clustal', 'task', + task_id, task_id + '.in') with open(query_filename, 'wb') as query_f: - query_text = [x.encode('ascii', 'ignore').strip() for x in request.POST['query-sequence'].split('\n')] + query_text = [ + x.encode('ascii', 'ignore').strip() + for x in request.POST['query-sequence'].split('\n') + ] query_f.write('\n'.join(query_text)) else: - return render(request, 'clustal/invalid_query.html', {'title': '',}) + return render(request, 'clustal/invalid_query.html', { + 'title': '', + }) chmod(query_filename, Perm.S_IRWXU | Perm.S_IRWXG | Perm.S_IRWXO) # ensure the standalone dequeuing process can access the file - bin_name = get_bin_name() # note that we didn't support Clustal on windows yet + bin_name = get_bin_name( + ) # note that we didn't support Clustal on windows yet program_path = path.join(settings.BASE_DIR, 'clustal', bin_name) # count number of query sequence by counting '>' with open(query_filename, 'r') as f: qstr = f.read() seq_count = qstr.count('>') - if(seq_count > 600): - return render(request, 'clustal/invalid_query.html', - {'title': 'Clustal: Max number of query sequences: 600 sequences.',}) + if (seq_count > 600): + return render( + request, 'clustal/invalid_query.html', { + 'title': + 'Clustal: Max number of query sequences: 600 sequences.', + }) is_color = False # check if program is in list for security @@ -81,81 +93,114 @@ def create(request): if request.POST['program'] == 'clustalw': # clustalw - option_params.append("-type="+request.POST['sequenceType']) + option_params.append("-type=" + request.POST['sequenceType']) # parameters setting for full option or fast option if request.POST['pairwise'] == "full": if request.POST['sequenceType'] == "dna": if request.POST['PWDNAMATRIX'] != "": - option_params.append('-PWDNAMATRIX='+request.POST['PWDNAMATRIX']) + option_params.append('-PWDNAMATRIX=' + + request.POST['PWDNAMATRIX']) if request.POST['dna-PWGAPOPEN'] != "": - option_params.append('-PWGAPOPEN='+request.POST['dna-PWGAPOPEN']) + option_params.append('-PWGAPOPEN=' + + request.POST['dna-PWGAPOPEN']) if request.POST['dna-PWGAPEXT'] != "": - option_params.append('-PWGAPEXT='+request.POST['dna-PWGAPEXT']) + option_params.append('-PWGAPEXT=' + + request.POST['dna-PWGAPEXT']) elif request.POST['sequenceType'] == "protein": if request.POST['PWMATRIX'] != "": - option_params.append('-PWMATRIX='+request.POST['PWMATRIX']) + option_params.append('-PWMATRIX=' + + request.POST['PWMATRIX']) if request.POST['protein-PWGAPOPEN'] != "": - option_params.append('-PWGAPOPEN='+request.POST['protein-PWGAPOPEN']) + option_params.append( + '-PWGAPOPEN=' + + request.POST['protein-PWGAPOPEN']) if request.POST['protein-PWGAPEXT'] != "": - option_params.append('-PWGAPEXT='+request.POST['protein-PWGAPEXT']) + option_params.append( + '-PWGAPEXT=' + + request.POST['protein-PWGAPEXT']) elif request.POST['pairwise'] == "fast": option_params.append('-QUICKTREE') if request.POST['KTUPLE'] != "": - option_params.append('-KTUPLE='+request.POST['KTUPLE']) + option_params.append('-KTUPLE=' + + request.POST['KTUPLE']) if request.POST['WINDOW'] != "": - option_params.append('-WINDOW='+request.POST['WINDOW']) + option_params.append('-WINDOW=' + + request.POST['WINDOW']) if request.POST['PAIRGAP'] != "": - option_params.append('-PAIRGAP='+request.POST['PAIRGAP']) + option_params.append('-PAIRGAP=' + + request.POST['PAIRGAP']) if request.POST['TOPDIAGS'] != "": - option_params.append('-TOPDIAGS='+request.POST['TOPDIAGS']) + option_params.append('-TOPDIAGS=' + + request.POST['TOPDIAGS']) if request.POST['SCORE'] != "": - option_params.append('-SCORE='+request.POST['SCORE']) + option_params.append('-SCORE=' + request.POST['SCORE']) # prarmeters setting for mutliple alignment if request.POST['sequenceType'] == "dna": if request.POST['DNAMATRIX'] != "": - option_params.append('-DNAMATRIX='+request.POST['DNAMATRIX']) + option_params.append('-DNAMATRIX=' + + request.POST['DNAMATRIX']) if request.POST['dna-GAPOPEN'] != "": - option_params.append('-GAPOPEN='+request.POST['dna-GAPOPEN']) + option_params.append('-GAPOPEN=' + + request.POST['dna-GAPOPEN']) if request.POST['dna-GAPEXT'] != "": - option_params.append('-GAPEXT='+request.POST['dna-GAPEXT']) + option_params.append('-GAPEXT=' + + request.POST['dna-GAPEXT']) if request.POST['dna-GAPDIST'] != "": - option_params.append('-GAPDIST='+request.POST['dna-GAPDIST']) + option_params.append('-GAPDIST=' + + request.POST['dna-GAPDIST']) if request.POST['dna-ITERATION'] != "": - option_params.append('-ITERATION='+request.POST['dna-ITERATION']) + option_params.append('-ITERATION=' + + request.POST['dna-ITERATION']) if request.POST['dna-NUMITER'] != "": - option_params.append('-NUMITER='+request.POST['dna-NUMITER']) + option_params.append('-NUMITER=' + + request.POST['dna-NUMITER']) if request.POST['dna-CLUSTERING'] != "": - option_params.append('-CLUSTERING='+request.POST['dna-CLUSTERING']) + option_params.append('-CLUSTERING=' + + request.POST['dna-CLUSTERING']) elif request.POST['sequenceType'] == "protein": if request.POST['MATRIX'] != "": - option_params.append('-MATRIX='+request.POST['MATRIX']) + option_params.append('-MATRIX=' + + request.POST['MATRIX']) if request.POST['protein-GAPOPEN'] != "": - option_params.append('-GAPOPEN='+request.POST['protein-GAPOPEN']) + option_params.append('-GAPOPEN=' + + request.POST['protein-GAPOPEN']) if request.POST['protein-GAPEXT'] != "": - option_params.append('-GAPEXT='+request.POST['protein-GAPEXT']) + option_params.append('-GAPEXT=' + + request.POST['protein-GAPEXT']) if request.POST['protein-GAPDIST'] != "": - option_params.append('-GAPDIST='+request.POST['protein-GAPDIST']) + option_params.append('-GAPDIST=' + + request.POST['protein-GAPDIST']) if request.POST['protein-ITERATION'] != "": - option_params.append('-ITERATION='+request.POST['protein-ITERATION']) + option_params.append('-ITERATION=' + + request.POST['protein-ITERATION']) if request.POST['protein-NUMITER'] != "": - option_params.append('-NUMITER='+request.POST['protein-NUMITER']) + option_params.append('-NUMITER=' + + request.POST['protein-NUMITER']) if request.POST['protein-CLUSTERING'] != "": - option_params.append('-CLUSTERING='+request.POST['protein-CLUSTERING']) + option_params.append( + '-CLUSTERING=' + + request.POST['protein-CLUSTERING']) # parameters setting of output - is_color = True if request.POST['OUTPUT'] == 'clustal' else False - option_params.append('-OUTPUT='+request.POST['OUTPUT']) - option_params.append('-OUTORDER='+request.POST['OUTORDER']) - - args_list.append([path.join(program_path, 'clustalw2'), '-infile='+query_filename, - '-OUTFILE='+path.join(settings.MEDIA_ROOT, 'clustal', 'task', task_id, task_id+'.aln'), - '-type=protein'] + option_params) + is_color = True if request.POST[ + 'OUTPUT'] == 'clustal' else False + option_params.append('-OUTPUT=' + request.POST['OUTPUT']) + option_params.append('-OUTORDER=' + request.POST['OUTORDER']) + + args_list.append([ + path.join(program_path, 'clustalw2'), '-infile=' + + query_filename, '-OUTFILE=' + + path.join(settings.MEDIA_ROOT, 'clustal', 'task', task_id, + task_id + '.aln'), '-type=protein' + ] + option_params) args_list_log = [] - args_list_log.append(['clustalw2', '-infile='+path.basename(query_filename), - '-OUTFILE='+task_id+'.aln', '-type=protein'] + option_params) + args_list_log.append([ + 'clustalw2', '-infile=' + path.basename(query_filename), + '-OUTFILE=' + task_id + '.aln', '-type=protein' + ] + option_params) else: # clustalo if request.POST['dealing_input'] == "yes": @@ -166,26 +211,38 @@ def create(request): option_params.append("--full-iter") if request.POST['combined_iter'] != "": - option_params.append("--iterations="+request.POST['combined_iter']) + option_params.append("--iterations=" + + request.POST['combined_iter']) if request.POST['max_gt_iter'] != "": - option_params.append("--max-guidetree-iterations="+request.POST['max_gt_iter']) + option_params.append("--max-guidetree-iterations=" + + request.POST['max_gt_iter']) if request.POST['max_hmm_iter'] != "": - option_params.append("--max-hmm-iterations="+request.POST['max_hmm_iter']) + option_params.append("--max-hmm-iterations=" + + request.POST['max_hmm_iter']) if request.POST['omega_output'] != "": - option_params.append("--outfmt="+request.POST['omega_output']) - is_color = True if request.POST['omega_output'] == 'clu' else False + option_params.append("--outfmt=" + + request.POST['omega_output']) + is_color = True if request.POST[ + 'omega_output'] == 'clu' else False if request.POST['omega_order'] != "": - option_params.append("--output-order="+request.POST['omega_order']) - - args_list.append([path.join(program_path,'clustalo'), '--infile='+query_filename, - '--guidetree-out='+path.join(settings.MEDIA_ROOT, 'clustal', 'task', task_id, task_id)+'.ph', - '--outfile='+path.join(settings.MEDIA_ROOT, 'clustal', 'task', task_id, task_id)+'.aln'] - + option_params) + option_params.append("--output-order=" + + request.POST['omega_order']) + + args_list.append([ + path.join(program_path, 'clustalo'), '--infile=' + + query_filename, '--guidetree-out=' + + path.join(settings.MEDIA_ROOT, 'clustal', 'task', task_id, + task_id) + '.ph', '--outfile=' + + path.join(settings.MEDIA_ROOT, 'clustal', 'task', task_id, + task_id) + '.aln' + ] + option_params) args_list_log = [] - args_list_log.append(['clustalo', '--infile='+path.basename(query_filename), - '--guidetree-out=' + task_id + '.ph', - '--outfile=' + task_id +'.aln'] + option_params) + args_list_log.append([ + 'clustalo', '--infile=' + + path.basename(query_filename), '--guidetree-out=' + + task_id + '.ph', '--outfile=' + task_id + '.aln' + ] + option_params) record = ClustalQueryRecord() record.task_id = task_id @@ -194,15 +251,23 @@ def create(request): record.save() # generate status.json for frontend status checking - with open(query_filename, 'r') as f: # count number of query sequence by counting '>' + with open(query_filename, 'r' + ) as f: # count number of query sequence by counting '>' qstr = f.read() seq_count = qstr.count('>') if (seq_count == 0): seq_count = 1 - with open(path.join(settings.MEDIA_ROOT, 'clustal', 'task', task_id, 'status.json'), 'wb') as f: - json.dump({'status': 'pending', 'seq_count': seq_count, 'program':request.POST['program'], - 'cmd': " ".join(args_list_log[0]), 'is_color': is_color, - 'query_filename': path.basename(query_filename)}, f) + with open( + path.join(settings.MEDIA_ROOT, 'clustal', 'task', + task_id, 'status.json'), 'wb') as f: + json.dump({ + 'status': 'pending', + 'seq_count': seq_count, + 'program': request.POST['program'], + 'cmd': " ".join(args_list_log[0]), + 'is_color': is_color, + 'query_filename': path.basename(query_filename) + }, f) run_clustal_task.delay(task_id, args_list, file_prefix) return redirect('clustal:retrieve', task_id) @@ -217,9 +282,12 @@ def retrieve(request, task_id='1'): try: r = ClustalQueryRecord.objects.get(task_id=task_id) # if result is generated and not expired - if r.result_date and (r.result_date.replace(tzinfo=None) >= (datetime.utcnow()+ timedelta(days=-7))): - url_base_prefix = path.join(settings.MEDIA_URL, 'clustal', 'task', task_id) - dir_base_prefix = path.join(settings.MEDIA_ROOT, 'clustal', 'task', task_id) + if r.result_date and (r.result_date.replace(tzinfo=None) >= + (datetime.utcnow() + timedelta(days=-7))): + url_base_prefix = path.join(settings.MEDIA_URL, 'clustal', 'task', + task_id) + dir_base_prefix = path.join(settings.MEDIA_ROOT, 'clustal', 'task', + task_id) url_prefix = path.join(url_base_prefix, task_id) dir_prefix = path.join(dir_base_prefix, task_id) @@ -231,7 +299,7 @@ def retrieve(request, task_id='1'): aln_url = dir_prefix + '.aln' if path.isfile(aln_url): - if(path.getsize(aln_url) > 1024 * 1024 * 10): + if (path.getsize(aln_url) > 1024 * 1024 * 10): report = 'The Clustal reports exceed 10 Megabyte, please download it.' out_txt.append(report) else: @@ -241,17 +309,16 @@ def retrieve(request, task_id='1'): line = line.rstrip('\n') report.append(line + "
") - out_txt.append(''.join(report).replace(' ',' ')) + out_txt.append(''.join(report).replace(' ', ' ')) else: - return render(request, 'clustal/results_not_existed.html', - { + return render(request, 'clustal/results_not_existed.html', { 'title': 'Internal Error', 'isError': True, }) dnd_url, ph_url = None, None query_prefix = path.splitext(statusdata['query_filename'])[0] - if path.isfile(path.join(dir_base_prefix,query_prefix + '.dnd')): + if path.isfile(path.join(dir_base_prefix, query_prefix + '.dnd')): dnd_url = path.join(url_base_prefix, query_prefix + '.dnd') ph_file = path.join(dir_base_prefix + '.ph') @@ -260,8 +327,7 @@ def retrieve(request, task_id='1'): if r.result_status in set(['SUCCESS']): return render( - request, - 'clustal/result.html', { + request, 'clustal/result.html', { 'title': 'CLUSTAL Result', 'aln': url_prefix + '.aln', 'ph': ph_url, @@ -272,30 +338,34 @@ def retrieve(request, task_id='1'): 'task_id': task_id, }) else: - return render(request, 'clustal/results_not_existed.html', - { - 'title': 'No Hits Found', - 'isNoHits': True, - 'isExpired': False, - }) + return render( + request, 'clustal/results_not_existed.html', { + 'title': 'No Hits Found', + 'isNoHits': True, + 'isExpired': False, + }) else: - enqueue_date = r.enqueue_date.astimezone(timezone('US/Eastern')).strftime('%d %b %Y %X %Z') + enqueue_date = r.enqueue_date.astimezone( + timezone('US/Eastern')).strftime('%d %b %Y %X %Z') if r.dequeue_date: - dequeue_date = r.dequeue_date.astimezone(timezone('US/Eastern')).strftime('%d %b %Y %X %Z') + dequeue_date = r.dequeue_date.astimezone( + timezone('US/Eastern')).strftime('%d %b %Y %X %Z') else: dequeue_date = None # result is exipired isExpired = False - if r.result_date and (r.result_date.replace(tzinfo=None) < (datetime.utcnow()+ timedelta(days=-7))): + if r.result_date and (r.result_date.replace(tzinfo=None) < + (datetime.utcnow() + timedelta(days=-7))): isExpired = True - return render(request, 'clustal/results_not_existed.html', { - 'title': 'Query Submitted', - 'task_id': task_id, - 'isExpired': isExpired, - 'enqueue_date': enqueue_date, - 'dequeue_date': dequeue_date, - 'isNoHits': False, - }) + return render( + request, 'clustal/results_not_existed.html', { + 'title': 'Query Submitted', + 'task_id': task_id, + 'isExpired': isExpired, + 'enqueue_date': enqueue_date, + 'dequeue_date': dequeue_date, + 'isNoHits': False, + }) except Exception: raise Http404 @@ -305,14 +375,15 @@ def status(request, task_id): function for front-end to check task status ''' if request.method == 'GET': - status_file_path = path.join(settings.MEDIA_ROOT, 'clustal', 'task', task_id, 'status.json') + status_file_path = path.join(settings.MEDIA_ROOT, 'clustal', 'task', + task_id, 'status.json') status = {'status': 'unknown'} if path.isfile(status_file_path): with open(status_file_path, 'rb') as f: statusdata = json.load(f) if statusdata['status'] == 'pending' and settings.USE_CACHE: tlist = cache.get('task_list_cache', []) - num_preceding = -1; + num_preceding = -1 if tlist: for index, tuple in enumerate(tlist): if task_id in tuple: @@ -325,31 +396,3 @@ def status(request, task_id): return HttpResponse(json.dumps(status)) else: return HttpResponse('Invalid Post') - - -# to-do: integrate with existing router of restframework -from rest_framework.renderers import JSONRenderer -from .serializers import UserClustalQueryRecordSerializer - - -class JSONResponse(HttpResponse): - """ - An HttpResponse that renders its content into JSON. - """ - def __init__(self, data, **kwargs): - content = JSONRenderer().render(data) - kwargs['content_type'] = 'application/json' - super(JSONResponse, self).__init__(content, **kwargs) - - -def user_tasks(request, user_id): - """ - Return tasks performed by the user. - """ - if request.method == 'GET': - records = ClustalQueryRecord.objects.filter(user__id=user_id, result_date__gt=(localtime(now()) + timedelta(days=-7))) - serializer = UserClustalQueryRecordSerializer(records, many=True) - return JSONResponse(serializer.data) - - - diff --git a/hmmer/serializers.py b/hmmer/serializers.py deleted file mode 100644 index 21587c7f2..000000000 --- a/hmmer/serializers.py +++ /dev/null @@ -1,20 +0,0 @@ -from rest_framework import serializers -from django.contrib.auth.models import User -from .models import HmmerQueryRecord - -class BlastQueryRecordSerializer(serializers.HyperlinkedModelSerializer): - url = serializers.HyperlinkedIdentityField(view_name='blastqueryrecord-detail', lookup_field='task_id') - - class Meta: - model = HmmerQueryRecord - -class UserSerializer(serializers.ModelSerializer): - url = serializers.HyperlinkedIdentityField(view_name='user-detail', lookup_field='pk') - class Meta: - model = User - fields = ('id',) - -class UserHmmerQueryRecordSerializer(serializers.ModelSerializer): - class Meta: - model = HmmerQueryRecord - fields = ('task_id', 'enqueue_date', 'result_status') \ No newline at end of file diff --git a/hmmer/urls.py b/hmmer/urls.py index c5b8ec51c..5b5213b4d 100644 --- a/hmmer/urls.py +++ b/hmmer/urls.py @@ -7,9 +7,8 @@ url(r'^$', views.create, name='create'), # ex: /hmmer/task/5/ url(r'^task/(?P[0-9a-zA-Z]+)$', views.retrieve, name='retrieve'), - url(r'^task/(?P[0-9a-zA-Z]+)/status$', views.status, name='status'), - # url(r'^api/', include(router.urls)), - # url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')), - url('^user-tasks/(?P[0-9]+)$', views.user_tasks), + url(r'^task/(?P[0-9a-zA-Z]+)/status$', + views.status, + name='status'), url(r'^manual/$', views.manual, name='manual'), ] diff --git a/hmmer/views.py b/hmmer/views.py index f5ed944cc..b011db25d 100644 --- a/hmmer/views.py +++ b/hmmer/views.py @@ -23,7 +23,7 @@ def manual(request): ''' Manual page of Hmmer ''' - return render(request, 'hmmer/manual.html',{'title':'HMMER Manual'}) + return render(request, 'hmmer/manual.html', {'title': 'HMMER Manual'}) def create(request): @@ -35,63 +35,77 @@ def create(request): (1). Phmmer, Max number of query sequences: 10 sequences ''' if request.method == 'GET': - hmmerdb_list = sorted([['Protein', "Protein", db.title, db.organism.display_name, db.description] for db in - HmmerDB.objects.select_related('organism').filter(is_shown=True)], - key=lambda x: (x[3], x[1], x[0], x[2])) - hmmerdb_type_counts = dict([(k.lower().replace(' ', '_'), len(list(g))) for k, g in - groupby(sorted(hmmerdb_list, key=lambda x: x[0]), key=lambda x: x[0])]) + hmmerdb_list = sorted( + [[ + 'Protein', "Protein", db.title, db.organism.display_name, + db.description + ] for db in HmmerDB.objects.select_related('organism').filter( + is_shown=True)], + key=lambda x: (x[3], x[1], x[0], x[2])) + hmmerdb_type_counts = dict([(k.lower().replace(' ', '_'), len( + list(g))) for k, g in groupby( + sorted(hmmerdb_list, key=lambda x: x[0]), key=lambda x: x[0])]) ''' Redirect from clustal result ''' clustal_content = [] if ("clustal_task_id" in request.GET): - clustal_aln = path.join(settings.MEDIA_ROOT, 'clustal', - 'task', request.GET['clustal_task_id'], + clustal_aln = path.join(settings.MEDIA_ROOT, 'clustal', 'task', + request.GET['clustal_task_id'], request.GET['clustal_task_id'] + ".aln") with open(clustal_aln, 'r') as content_file: for line in content_file: clustal_content.append(line) - return render(request, 'hmmer/main.html', { - 'title': 'HMMER Query', - 'hmmerdb_list': json.dumps(hmmerdb_list), - 'hmmerdb_type_counts': hmmerdb_type_counts, - 'clustal_content': "".join(clustal_content), - }) + return render( + request, 'hmmer/main.html', { + 'title': 'HMMER Query', + 'hmmerdb_list': json.dumps(hmmerdb_list), + 'hmmerdb_type_counts': hmmerdb_type_counts, + 'clustal_content': "".join(clustal_content), + }) elif request.method == 'POST': # setup file paths task_id = uuid4().hex task_dir = path.join(settings.MEDIA_ROOT, 'hmmer', 'task', task_id) # file_prefix only for task... - file_prefix = path.join(settings.MEDIA_ROOT, 'hmmer', 'task', task_id, task_id) + file_prefix = path.join(settings.MEDIA_ROOT, 'hmmer', 'task', task_id, + task_id) if not path.exists(task_dir): makedirs(task_dir) - chmod(task_dir, - Perm.S_IRWXU | Perm.S_IRWXG | Perm.S_IRWXO) + chmod(task_dir, Perm.S_IRWXU | Perm.S_IRWXG | Perm.S_IRWXO) # ensure the standalone dequeuing process can open files in the directory # change directory to task directory if 'query-file' in request.FILES: - query_filename = path.join(settings.MEDIA_ROOT, 'hmmer', 'task', task_id, request.FILES['query-file'].name) + query_filename = path.join(settings.MEDIA_ROOT, 'hmmer', 'task', + task_id, + request.FILES['query-file'].name) with open(query_filename, 'wb') as query_f: for chunk in request.FILES['query-file'].chunks(): query_f.write(chunk) elif 'query-sequence' in request.POST and request.POST['query-sequence']: - query_filename = path.join(settings.MEDIA_ROOT, 'hmmer', 'task', task_id, task_id + '.in') + query_filename = path.join(settings.MEDIA_ROOT, 'hmmer', 'task', + task_id, task_id + '.in') with open(query_filename, 'wb') as query_f: - query_text = [x.encode('ascii', 'ignore').strip() for x in request.POST['query-sequence'].split('\n')] + query_text = [ + x.encode('ascii', 'ignore').strip() + for x in request.POST['query-sequence'].split('\n') + ] query_f.write('\n'.join(query_text)) else: - return render(request, 'hmmer/invalid_query.html', {'title': '', }) + return render(request, 'hmmer/invalid_query.html', { + 'title': '', + }) - chmod(query_filename, - Perm.S_IRWXU | Perm.S_IRWXG | Perm.S_IRWXO) + chmod(query_filename, Perm.S_IRWXU | Perm.S_IRWXG | Perm.S_IRWXO) # ensure the standalone dequeuing process can access the file - bin_name = get_bin_name() # Note that currently we didn't support HMMER on windows + bin_name = get_bin_name( + ) # Note that currently we didn't support HMMER on windows program_path = path.join(settings.BASE_DIR, 'hmmer', bin_name, 'bin') if request.POST['program'] == 'phmmer': @@ -100,41 +114,70 @@ def create(request): if qstr.count('>') > int(HMMER_QUERY_MAX): query_cnt = str(qstr.count('>')) remove(query_filename) - return render(request, 'hmmer/invalid_query.html', - {'title': 'Your search includes ' + query_cnt + ' sequences, but HMMER allows a maximum of ' + str(HMMER_QUERY_MAX) + ' sequences per submission.', }) + return render( + request, 'hmmer/invalid_query.html', { + 'title': + 'Your search includes ' + query_cnt + + ' sequences, but HMMER allows a maximum of ' + + str(HMMER_QUERY_MAX) + + ' sequences per submission.', + }) elif request.POST['program'] == 'hmmsearch': ''' Format validation by hmmsearch fast mode If the machine can't perform it in short time, it could be marked. But you need find a good to check format in front-end ''' - p = Popen([path.join(program_path, "hmmbuild"), "--fast", '--amino', - path.join(settings.MEDIA_ROOT, 'hmmer', 'task', 'hmmbuild.test'), query_filename], - stdout=PIPE, stderr=PIPE) + p = Popen( + [ + path.join(program_path, "hmmbuild"), "--fast", '--amino', + path.join(settings.MEDIA_ROOT, 'hmmer', 'task', + 'hmmbuild.test'), query_filename + ], + stdout=PIPE, + stderr=PIPE) p.wait() result = p.communicate()[1] - if(result != ''): - return render(request, 'hmmer/invalid_query.html', - {'title': 'Invalid MSA format', - 'info' :' \ - Valid MSA format descriptions ' }) + if (result != ''): + return render( + request, 'hmmer/invalid_query.html', { + 'title': + 'Invalid MSA format', + 'info': + ' \ + Valid MSA format descriptions ' + }) else: # check if program is in list for security raise Http404 # build hmmer command - db_list = [db.fasta_file.path_full for db in HmmerDB.objects.filter(title__in=set(request.POST.getlist('db-name')))] + db_list = [ + db.fasta_file.path_full for db in HmmerDB.objects.filter( + title__in=set(request.POST.getlist('db-name'))) + ] for db in db_list: - symlink(db, path.join(settings.MEDIA_ROOT, 'hmmer', 'task', task_id, db[db.rindex('/') + 1:])) + symlink( + db, + path.join(settings.MEDIA_ROOT, 'hmmer', 'task', task_id, + db[db.rindex('/') + 1:])) if not db_list: - return render(request, 'hmmer/invalid_query.html', {'title': '', }) + return render(request, 'hmmer/invalid_query.html', { + 'title': '', + }) if request.POST['cutoff'] == 'evalue': - option_params = ['--incE', request.POST['s_sequence'], '--incdomE', request.POST['s_hit'], - '-E', request.POST['r_sequence'], '--domE', request.POST['r_hit']] + option_params = [ + '--incE', request.POST['s_sequence'], '--incdomE', + request.POST['s_hit'], '-E', request.POST['r_sequence'], + '--domE', request.POST['r_hit'] + ] elif request.POST['cutoff'] == 'bitscore': - option_params = ['--incT', request.POST['s_sequence'], '--incdomT', request.POST['s_hit'], - '-T', request.POST['r_sequence'], '--domT', request.POST['r_hit']] + option_params = [ + '--incT', request.POST['s_sequence'], '--incdomT', + request.POST['s_hit'], '-T', request.POST['r_sequence'], + '--domT', request.POST['r_hit'] + ] else: raise Http404 @@ -149,13 +192,20 @@ def create(request): seq_count = qstr.count('>') if (seq_count == 0): seq_count = 1 - with open(path.join(settings.MEDIA_ROOT, 'hmmer', 'task', task_id, 'status.json'), 'wb') as f: - json.dump({'status': 'pending', 'seq_count': seq_count, - 'db_list': [db[db.rindex('/') + 1:] for db in db_list], - 'program': request.POST['program'], - 'params': option_params, - 'input': path.basename(query_filename)}, f) - args = generate_hmmer_args(request.POST['program'], program_path, query_filename, option_params, db_list) + with open( + path.join(settings.MEDIA_ROOT, 'hmmer', 'task', task_id, + 'status.json'), 'wb') as f: + json.dump( + { + 'status': 'pending', + 'seq_count': seq_count, + 'db_list': [db[db.rindex('/') + 1:] for db in db_list], + 'program': request.POST['program'], + 'params': option_params, + 'input': path.basename(query_filename) + }, f) + args = generate_hmmer_args(request.POST['program'], program_path, + query_filename, option_params, db_list) run_hmmer_task.delay(task_id, args, file_prefix) return redirect('hmmer:retrieve', task_id) @@ -167,9 +217,12 @@ def retrieve(request, task_id='1'): try: r = HmmerQueryRecord.objects.get(task_id=task_id) # if result is generated and not expired - if r.result_date and (r.result_date.replace(tzinfo=None) >= (datetime.utcnow() + timedelta(days=-7))): - url_base_prefix = path.join(settings.MEDIA_URL, 'hmmer', 'task', task_id) - dir_base_prefix = path.join(settings.MEDIA_ROOT, 'hmmer', 'task', task_id) + if r.result_date and (r.result_date.replace(tzinfo=None) >= + (datetime.utcnow() + timedelta(days=-7))): + url_base_prefix = path.join(settings.MEDIA_URL, 'hmmer', 'task', + task_id) + dir_base_prefix = path.join(settings.MEDIA_ROOT, 'hmmer', 'task', + task_id) url_prefix = path.join(url_base_prefix, task_id) dir_prefix = path.join(dir_base_prefix, task_id) @@ -182,7 +235,8 @@ def retrieve(request, task_id='1'): report = ["
"] # 1mb limitation - if path.isfile(dir_prefix + '.merge') and path.getsize(dir_prefix + '.merge') > 1024000: + if path.isfile(dir_prefix + '.merge' + ) and path.getsize(dir_prefix + '.merge') > 1024000: out_txt = 'The Hmmer reports exceed 1 Megabyte, please download it.' isExceed = True else: @@ -192,7 +246,8 @@ def retrieve(request, task_id='1'): for line in content_file: line = line.rstrip('\n') if line == '[ok]': - out_txt.append(''.join(report).replace(' ', ' ')) + out_txt.append(''.join(report).replace( + ' ', ' ')) report = ["
"] else: report.append(line + "
") @@ -200,7 +255,8 @@ def retrieve(request, task_id='1'): if r.result_status == 'SUCCESS': return render( request, - 'hmmer/result.html', { + 'hmmer/result.html', + { 'title': 'HMMER Result', 'output': url_prefix + '.merge', 'status': path.join(url_base_prefix, 'status.json'), @@ -211,37 +267,42 @@ def retrieve(request, task_id='1'): 'isExceed': isExceed }) else: - return render(request, 'hmmer/results_not_existed.html', - { - 'title': 'No Hits Found', - 'isNoHits': True, - 'isExpired': False, - }) + return render( + request, 'hmmer/results_not_existed.html', { + 'title': 'No Hits Found', + 'isNoHits': True, + 'isExpired': False, + }) else: - enqueue_date = r.enqueue_date.astimezone(timezone('US/Eastern')).strftime('%d %b %Y %X %Z') + enqueue_date = r.enqueue_date.astimezone( + timezone('US/Eastern')).strftime('%d %b %Y %X %Z') if r.dequeue_date: - dequeue_date = r.dequeue_date.astimezone(timezone('US/Eastern')).strftime('%d %b %Y %X %Z') + dequeue_date = r.dequeue_date.astimezone( + timezone('US/Eastern')).strftime('%d %b %Y %X %Z') else: dequeue_date = None # result is exipired isExpired = False - if r.result_date and (r.result_date.replace(tzinfo=None) < (datetime.utcnow() + timedelta(days=-7))): + if r.result_date and (r.result_date.replace(tzinfo=None) < + (datetime.utcnow() + timedelta(days=-7))): isExpired = True - return render(request, 'hmmer/results_not_existed.html', { - 'title': 'Query Submitted', - 'task_id': task_id, - 'isExpired': isExpired, - 'enqueue_date': enqueue_date, - 'dequeue_date': dequeue_date, - 'isNoHits': False, - }) + return render( + request, 'hmmer/results_not_existed.html', { + 'title': 'Query Submitted', + 'task_id': task_id, + 'isExpired': isExpired, + 'enqueue_date': enqueue_date, + 'dequeue_date': dequeue_date, + 'isNoHits': False, + }) except Exception: raise Http404 def status(request, task_id): if request.method == 'GET': - status_file_path = path.join(settings.MEDIA_ROOT, 'hmmer', 'task', task_id, 'status.json') + status_file_path = path.join(settings.MEDIA_ROOT, 'hmmer', 'task', + task_id, 'status.json') status = {'status': 'unknown'} if path.isfile(status_file_path): with open(status_file_path, 'rb') as f: @@ -262,45 +323,23 @@ def status(request, task_id): else: return HttpResponse('Invalid Post') -# to-do: integrate with existing router of restframework -from rest_framework.renderers import JSONRenderer -from .serializers import UserHmmerQueryRecordSerializer - - -class JSONResponse(HttpResponse): - """ - An HttpResponse that renders its content into JSON. - """ - - def __init__(self, data, **kwargs): - content = JSONRenderer().render(data) - kwargs['content_type'] = 'application/json' - super(JSONResponse, self).__init__(content, **kwargs) - - -def user_tasks(request, user_id): - """ - Return tasks performed by the user. - """ - if request.method == 'GET': - records = HmmerQueryRecord.objects.filter(user__id=user_id, result_date__gt=(localtime(now())+ timedelta(days=-7))) - serializer = UserHmmerQueryRecordSerializer(records, many=True) - return JSONResponse(serializer.data) - -def generate_hmmer_args(program, program_path, query_filename, - option_params, db_list): +def generate_hmmer_args(program, program_path, query_filename, option_params, + db_list): args = [] if program == 'hmmsearch': - args.append([path.join(program_path, 'hmmbuild'), '--amino', - '-o', 'hmm.sumary', - query_filename + '.hmm', - query_filename]) + args.append([ + path.join(program_path, 'hmmbuild'), '--amino', '-o', 'hmm.sumary', + query_filename + '.hmm', query_filename + ]) for idx, db in enumerate(db_list): - args.append([path.join(program_path, 'hmmsearch'), '-o', str(idx) + '.out'] - + option_params + [query_filename + '.hmm', db]) + args.append([ + path.join(program_path, 'hmmsearch'), '-o', + str(idx) + '.out' + ] + option_params + [query_filename + '.hmm', db]) else: # phmmer for idx, db in enumerate(db_list): - args.append([path.join(program_path, 'phmmer'), '-o', str(idx) + '.out'] - + option_params + [query_filename, db]) + args.append( + [path.join(program_path, 'phmmer'), '-o', + str(idx) + '.out'] + option_params + [query_filename, db]) return args diff --git a/i5k/settings.py b/i5k/settings.py index be9b452a2..f861baaf2 100755 --- a/i5k/settings.py +++ b/i5k/settings.py @@ -285,7 +285,9 @@ REST_FRAMEWORK = { 'DEFAULT_PERMISSION_CLASSES': [ 'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly' - ] + ], + 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', + 'PAGE_SIZE': 20 } # django-pipeline