diff --git a/.dockerignore b/.dockerignore index 6fa1727..d9d231a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,6 @@ myenv env -db.sqlite3 \ No newline at end of file +db.sqlite3 +cover +media/uploads/*.rdf +media/uploads/*.ttl \ No newline at end of file diff --git a/.gitignore b/.gitignore index 520aaf9..04346d5 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,5 @@ s_wsgy.py .env_dev vocabseditor/settings/pg_local.py .env_secret +media/uploads/*.rdf +media/uploads/*.ttl \ No newline at end of file diff --git a/README.md b/README.md index dfef9f4..8129c0b 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ REDMINE_ID=12345 MIGRATE=yes ``` -`docker run -it -p 8020:8020 --rm --env-file .env vocabseditor` +`docker run -it -p 8020:8020 --rm --name vocabseditor --env-file .env vocabseditor` ### docker-compose (using external database) diff --git a/build_and_run.sh b/build_and_run.sh new file mode 100755 index 0000000..c137f0f --- /dev/null +++ b/build_and_run.sh @@ -0,0 +1,4 @@ +#/bin/bash + +docker build -t vocabseditor:latest . +docker run -it -p 8020:8020 --rm --name vocabseditor --env-file .env vocabseditor \ No newline at end of file diff --git a/media/uploads/.gitkeep b/media/uploads/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/vocabs/export_views.py b/vocabs/export_views.py deleted file mode 100644 index bc78e5c..0000000 --- a/vocabs/export_views.py +++ /dev/null @@ -1,18 +0,0 @@ -from django.shortcuts import redirect -from django.views.generic.list import ListView -from django_celery_results.models import TaskResult - -from vocabs.tasks import export_concept_schema - - -def export_async(request): - get_format = request.GET.get('format', default='pretty-xml') - schema_id = request.GET.get('schema-id') - export_concept_schema.delay(schema_id, get_format) - return redirect('vocabs:export-status') - - -class TaskResultListView(ListView): - model = TaskResult - paginate_by = 100 - template_name = "vocabs/taskresult_list.j2" diff --git a/vocabs/import_export_views.py b/vocabs/import_export_views.py new file mode 100644 index 0000000..a614ab2 --- /dev/null +++ b/vocabs/import_export_views.py @@ -0,0 +1,47 @@ +from django.contrib import messages +from django.shortcuts import redirect, render +from django.contrib.auth.decorators import login_required +from django.views.generic.list import ListView +from django_celery_results.models import TaskResult + +from vocabs.forms import UploadFileForm +from vocabs.tasks import export_concept_schema, import_concept_schema +from vocabs.utils import handle_uploaded_file + + +def export_async(request): + get_format = request.GET.get('format', default='pretty-xml') + schema_id = request.GET.get('schema-id') + export_concept_schema.delay(schema_id, get_format) + return redirect('vocabs:job-status') + + +@login_required +def import_async(request): + if request.method == 'POST': + form = UploadFileForm(request.POST, request.FILES) + if form.is_valid(): + file = request.FILES['file'] + file_format = file.name.split('.')[-1] + if file_format in ['ttl', 'rdf']: + file_format = file.name.split('.')[-1] + full_path = handle_uploaded_file(file) + import_concept_schema.delay( + full_path, + request.user.username, + file_format=file_format, + language=form.cleaned_data['language'] + ) + messages.info(request, f"Started Import of {file.name}") + else: + messages.error(request, "Upload rdf or ttl file") + return redirect('vocabs:job-status') + else: + form = UploadFileForm() + return render(request, 'vocabs/upload.html', {'form': form}) + + +class TaskResultListView(ListView): + model = TaskResult + paginate_by = 100 + template_name = "vocabs/taskresult_list.j2" diff --git a/vocabs/skos_import.py b/vocabs/skos_import.py index b64a2b2..ddf6984 100644 --- a/vocabs/skos_import.py +++ b/vocabs/skos_import.py @@ -486,7 +486,8 @@ def upload_data(self, user): logging.info(e) else: pass - return SkosConcept.objects.rebuild() + SkosConcept.objects.rebuild() + return concept_scheme.get_absolute_url() else: pass return diff --git a/vocabs/tasks.py b/vocabs/tasks.py index 51940c7..163216c 100644 --- a/vocabs/tasks.py +++ b/vocabs/tasks.py @@ -3,11 +3,12 @@ from django.conf import settings from django.utils.text import slugify -from vocabs.rdf_utils import graph_construct_qs, RDF_FORMATS from vocabs.models import SkosConceptScheme, SkosConcept +from vocabs.rdf_utils import graph_construct_qs, RDF_FORMATS +from vocabs.skos_import import SkosImporter -@shared_task +@shared_task(name="Export") def export_concept_schema(schema_id, export_format): schema = SkosConceptScheme.objects.get(id=schema_id) qs = SkosConcept.objects.filter(scheme=schema) @@ -16,4 +17,21 @@ def export_concept_schema(schema_id, export_format): export_path = os.path.join(settings.MEDIA_ROOT, file_name) g.serialize(export_path, format=export_format) os.chmod(export_path, 0o0755) # this is needed because I don't get docker permission/user things - return file_name + return f"/media/{file_name}" + + +@shared_task(name="Import") +def import_concept_schema(full_path, user_name, file_format=None, language=None): + if file_format == 'ttl': + skos_vocab = SkosImporter( + file=full_path, + file_format="ttl", + language=language + ) + else: + skos_vocab = SkosImporter( + file=full_path, + language=language + ) + result = skos_vocab.upload_data(user=user_name) + return result diff --git a/vocabs/templates/vocabs/taskresult_list.j2 b/vocabs/templates/vocabs/taskresult_list.j2 index 0bb18be..93d03ba 100644 --- a/vocabs/templates/vocabs/taskresult_list.j2 +++ b/vocabs/templates/vocabs/taskresult_list.j2 @@ -11,7 +11,7 @@ - + @@ -20,7 +20,7 @@ {% for x in object_list %} - +
IDJob Status Start End
{{ x.task_id }}{{ x.task_name }} {% if x.status == "SUCCESS" %} {{ x.status }} {% elif x.status == "FAILURE" %} @@ -32,7 +32,7 @@ {{ x.date_created }} {{ x.date_done }} {% if x.status == "SUCCESS" %} - {{ x.result |cut:'"' }} + {{ x.result |cut:'"' }} {% else %} {{ x.result }} {% endif %} diff --git a/vocabs/urls.py b/vocabs/urls.py index 95b4b47..13ed9be 100644 --- a/vocabs/urls.py +++ b/vocabs/urls.py @@ -1,6 +1,6 @@ from django.urls import path from . import views -from vocabs.export_views import export_async, TaskResultListView +from vocabs.import_export_views import import_async, export_async, TaskResultListView app_name = 'vocabs' @@ -45,7 +45,7 @@ 'collection/delete/', views.SkosCollectionDelete.as_view(), name='skoscollection_delete', ), - path('import/', views.file_upload, name='import'), + path('import/', import_async, name='import'), path('export/', export_async, name='export'), - path('export-status/', TaskResultListView.as_view(), name='export-status'), + path('job-status/', TaskResultListView.as_view(), name='job-status'), ] diff --git a/vocabs/utils.py b/vocabs/utils.py index 94774ab..dfb68c9 100644 --- a/vocabs/utils.py +++ b/vocabs/utils.py @@ -1,3 +1,6 @@ +import os +from django.conf import settings + RDF_FORMATS = { "xml": "rdf", "n3": "n3", @@ -9,3 +12,16 @@ "nquads": "nq", "json-ld": ".jsonld", } + + +def handle_uploaded_file(file): + file_name = file.name + full_file_name = os.path.join( + settings.MEDIA_ROOT, + 'uploads', + file_name + ) + with open(full_file_name, 'wb+') as destination: + for chunk in file.chunks(): + destination.write(chunk) + return full_file_name diff --git a/vocabs/views.py b/vocabs/views.py index 8ef7d2b..e4ae528 100644 --- a/vocabs/views.py +++ b/vocabs/views.py @@ -1,15 +1,19 @@ import time import datetime -from django.shortcuts import render +from guardian.shortcuts import get_objects_for_user +from django.contrib.auth.decorators import login_required +from reversion.models import Version +from django.db import transaction +from browsing.browsing_utils import GenericListView, BaseCreateView, BaseUpdateView from django.http import HttpResponse from django.views.generic.detail import DetailView from django.views.generic.edit import DeleteView from django.utils.decorators import method_decorator from django.urls import reverse_lazy from django_tables2 import RequestConfig -from .models import SkosConcept, SkosConceptScheme, SkosCollection -from .forms import ( - UploadFileForm, + +from vocabs.models import SkosConcept, SkosConceptScheme, SkosCollection +from vocabs.forms import ( SkosConceptSchemeFormHelper, SkosConceptSchemeForm, SkosCollectionFormHelper, @@ -36,17 +40,8 @@ SkosConceptSchemeListFilter, SkosCollectionListFilter ) -from browsing.browsing_utils import GenericListView, BaseCreateView, BaseUpdateView from vocabs.rdf_utils import graph_construct_qs, RDF_FORMATS -from guardian.shortcuts import get_objects_for_user -from django.contrib.auth.decorators import login_required -from reversion.models import Version -from django.db import transaction -from django.shortcuts import redirect -from vocabs.skos_import import SkosImporter -from django.contrib import messages - class BaseDetailView(DetailView): @@ -597,31 +592,3 @@ def render_to_response(self, context): g = graph_construct_qs(qs) g.serialize(destination=response, format=get_format) return response - - -################################################### -# SKOS vocabulary upload -################################################### - -@login_required -def file_upload(request): - if request.method == 'POST': - form = UploadFileForm(request.POST, request.FILES) - if form.is_valid(): - file = request.FILES['file'] - file_format = file.name.split('.')[-1] - if file_format in ['ttl', 'rdf']: - if file_format == 'ttl': - skos_vocab = SkosImporter(file=file, file_format="ttl", language=form.cleaned_data['language']) - else: - skos_vocab = SkosImporter(file=file, language=form.cleaned_data['language']) - try: - skos_vocab.upload_data(user=request.user.username) - return redirect('vocabs:browse_schemes') - except Exception as error: - messages.error(request, error) - else: - messages.error(request, "Upload rdf or ttl file") - else: - form = UploadFileForm() - return render(request, 'vocabs/upload.html', {'form': form}) diff --git a/vocabseditor/settings.py b/vocabseditor/settings.py index 2a14596..de77121 100644 --- a/vocabseditor/settings.py +++ b/vocabseditor/settings.py @@ -182,6 +182,7 @@ CELERY_RESULT_BACKEND = 'django-db' CELERY_BROKER_URL = os.environ.get('amqp://') CELERY_TASK_TRACK_STARTED = True +CELERY_RESULT_EXTENDED = True # Django guardian settings