From 015804e7840a9bd58aaf80d0b0b84412e1de2dff Mon Sep 17 00:00:00 2001 From: Simon Rey Date: Mon, 22 Aug 2022 10:45:06 +0200 Subject: [PATCH] Write dataset command + new info file --- info.txt | 0 preflibApp/management/commands/adddataset.py | 4 +- preflibApp/management/commands/deldataset.py | 16 ++- .../management/commands/initializedb.py | 2 +- .../management/commands/writedataset.py | 127 ++++++++++++++++++ preflibApp/models.py | 15 ++- 6 files changed, 147 insertions(+), 17 deletions(-) create mode 100644 info.txt create mode 100644 preflibApp/management/commands/writedataset.py diff --git a/info.txt b/info.txt new file mode 100644 index 0000000..e69de29 diff --git a/preflibApp/management/commands/adddataset.py b/preflibApp/management/commands/adddataset.py index 44195ba..1987b87 100644 --- a/preflibApp/management/commands/adddataset.py +++ b/preflibApp/management/commands/adddataset.py @@ -33,7 +33,7 @@ def handle(self, *args, **options): try: # Initializing the log - newLogNum = Log.objects.filter(logType = "dataset").aggregate(Max('logNum'))['logNum__max'] + newLogNum = Log.objects.filter(logType = "add_dataset").aggregate(Max('logNum'))['logNum__max'] if newLogNum == None: newLogNum = 0 else: @@ -112,7 +112,7 @@ def handle(self, *args, **options): os.remove(os.path.join(dataToAddDir, "dataset.lock")) Log.objects.create( log = ''.join(log), - logType = "dataset", + logType = "add_dataset", logNum = newLogNum, publicationDate = timezone.now()) diff --git a/preflibApp/management/commands/deldataset.py b/preflibApp/management/commands/deldataset.py index 42bb571..40301d9 100644 --- a/preflibApp/management/commands/deldataset.py +++ b/preflibApp/management/commands/deldataset.py @@ -1,10 +1,7 @@ from django.core.management.base import BaseCommand from django.contrib.staticfiles import finders -from django.core import management from django.utils import timezone -from django.conf import settings from django.db.models import Max -from django.apps import apps from preflibApp.choices import * from preflibApp.models import * @@ -18,13 +15,14 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument('--abb', nargs = '*', type = str) + parser.add_argument('--all', action = 'store_true') def handle(self, *args, **options): try: # Initializing the log - newLogNum = Log.objects.filter(logType = "dataset").aggregate(Max('logNum'))['logNum__max'] + newLogNum = Log.objects.filter(logType = "del_dataset").aggregate(Max('logNum'))['logNum__max'] if newLogNum == None: newLogNum = 0 else: @@ -41,8 +39,12 @@ def handle(self, *args, **options): log.append("\n

There is no data folder in the static folder, that is weird...

") # Starting the real stuff - log.append("

Adding datasets

\n\n

The datasets have been successfully deleted in ") log.append(str((timezone.now() - startTime).total_seconds() / 60)) log.append(" minutes.

") @@ -66,6 +68,6 @@ def handle(self, *args, **options): finally: Log.objects.create( log = ''.join(log), - logType = "dataset", + logType = "del_dataset", logNum = newLogNum, publicationDate = timezone.now()) \ No newline at end of file diff --git a/preflibApp/management/commands/initializedb.py b/preflibApp/management/commands/initializedb.py index e0d3dc2..5628993 100644 --- a/preflibApp/management/commands/initializedb.py +++ b/preflibApp/management/commands/initializedb.py @@ -29,7 +29,7 @@ def initialize_tags(self): sport_tag = DataTag.objects.update_or_create( name = "Sport", defaults = { - "description": "The preferences apply to scenario in which some alternatives are to be selected", + "description": "The data represent sport tournaments, interpreted as elections", "parent": election_tag[0] } ) diff --git a/preflibApp/management/commands/writedataset.py b/preflibApp/management/commands/writedataset.py new file mode 100644 index 0000000..fb4a5d0 --- /dev/null +++ b/preflibApp/management/commands/writedataset.py @@ -0,0 +1,127 @@ +from django.core.management.base import BaseCommand +from django.contrib.staticfiles import finders +from django.utils import timezone +from django.db.models import Max + +from preflibApp.choices import * +from preflibApp.models import * + +import traceback +import shutil +import os + + +class Command(BaseCommand): + help = "Add datasets to database" + + def add_arguments(self, parser): + parser.add_argument('-d', type = str, required = True) + parser.add_argument('--abb', nargs = '*', type = str) + parser.add_argument('--all', action = 'store_true') + + + def handle(self, *args, **options): + if not options['d']: + print("ERROR: you need to pass a target directory as argument (with option -d path/to/your/dic).") + return + else: + if not os.path.isdir(options['d']): + print("ERROR: {} is not a directory.".format(options['d'])) + return + + if not options['all'] and not options['abb']: + print("ERROR: you need to pass at least one dataset to write (with option --abb DATASET_ABBREVIATION) or the option --all.") + return + + try: + # Initializing the log + newLogNum = Log.objects.filter(logType = "write_dataset").aggregate(Max('logNum'))['logNum__max'] + if newLogNum == None: + newLogNum = 0 + else: + newLogNum += 1 + + # Starting the log + log = ["

Writing dataset #" + str(newLogNum) + " - " + str(timezone.now()) + "

\n"] + log.append("\n") + + # Looking for the data folder + data_dir = finders.find("data") + if not data_dir: + log.append("\n

There is no data folder in the static folder, that is weird...

") + + # Starting the real stuff + log.append("

Deleting datasets

\n\n

The datasets have been successfully deleted in ") + log.append(str((timezone.now() - startTime).total_seconds() / 60)) + log.append(" minutes.

") + + except Exception as e: + # If anything happened during the execution, we log it and move on + log.append("\n

" + str(e) + "
\n" + str(traceback.format_exc()) + "

") + print(traceback.format_exc()) + print(e) + finally: + Log.objects.create( + log = ''.join(log), + logType = "write_dataset", + logNum = newLogNum, + publicationDate = timezone.now()) + + def write_info_file(self, dataset, ds_dir): + with open(os.path.join(ds_dir, "info.txt"), "w") as f: + # File Header + f.write("Name: {}\n\n".format(dataset.name)) + f.write("Abbreviation: {}\n\n".format(dataset.abbreviation)) + f.write("Category: {}\n\n".format(dataset.category)) + f.write("Tags: {}\n\n".format(dataset.get_tag_list())) + f.write("Series Number: {}\n\n".format(dataset.seriesNumber)) + f.write("Publication Date: {}\n\n".format(dataset.publicationDate)) + f.write("Description: {}\n\n".format(dataset.description)) + f.write("Required Citations: {}\n\n".format(dataset.requiredCitations)) + f.write("Selected Studies: {}\n\n".format(dataset.selectedStudies)) + + # Patch Section + f.write("patch_name, description, series number, publication date, representative\n") + write_file_str = "\npatch_number, file_name, modification type, publication date\n" + for data_patch in dataset.datapatch_set.all(): + f.write("{}, {}, {}, {}, {}\n".format(data_patch.name, data_patch.description, data_patch.seriesNumber, + data_patch.publicationDate, data_patch.representative.fileName)) + + for data_file in data_patch.datafile_set.all(): + write_file_str += "{}, {}, {}, {}\n".format(data_patch.seriesNumber, data_file.fileName, + data_file.modificationType, data_file.publicationDate) + + # File Section + f.write(write_file_str) + f.close() diff --git a/preflibApp/models.py b/preflibApp/models.py index 95c3ce4..6ae7a12 100644 --- a/preflibApp/models.py +++ b/preflibApp/models.py @@ -54,10 +54,11 @@ class DataSet(models.Model): description = models.TextField() tags = models.ManyToManyField(DataTag, blank = True) - requiredCitations = models.TextField() + requiredCitations = models.TextField(blank = True, + null = True) selectedStudies = models.TextField() - publicationDate = models.DateTimeField(auto_now = True) - modificationDate = models.DateTimeField(auto_now = True) + publicationDate = models.DateField(auto_now = True) + modificationDate = models.DateField(auto_now = True) def get_tag_list(self): return ', '.join([str(tag) for tag in self.tags.all()]) @@ -77,8 +78,8 @@ class DataPatch(models.Model): name = models.CharField(max_length = 1000) description = models.CharField(max_length = 1000) seriesNumber = models.SlugField() - publicationDate = models.DateTimeField(auto_now = True) - modificationDate = models.DateTimeField(auto_now = True) + publicationDate = models.DateField(auto_now = True) + modificationDate = models.DateField(auto_now = True) class Meta: unique_together = ('dataSet', 'seriesNumber') @@ -133,8 +134,8 @@ class DataFile(models.Model): unique = True) fileSize = models.FloatField(default = 0) image = models.CharField(max_length = 1000, null = True) - publicationDate = models.DateTimeField(auto_now = True) - modificationDate = models.DateTimeField(auto_now = True) + publicationDate = models.DateField(auto_now = True) + modificationDate = models.DateField(auto_now = True) class Meta: ordering = ['fileName']