diff --git a/manage.py b/manage.py index 6969298..774fc57 100644 --- a/manage.py +++ b/manage.py @@ -5,7 +5,7 @@ def main(): - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'preflib.settings') + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "preflib.settings") try: from django.core.management import execute_from_command_line except ImportError as exc: @@ -17,5 +17,5 @@ def main(): execute_from_command_line(sys.argv) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/preflib/settings.py b/preflib/settings.py index d3901a0..9d07f89 100755 --- a/preflib/settings.py +++ b/preflib/settings.py @@ -23,52 +23,51 @@ # Application definition INSTALLED_APPS = [ - 'preflibapp.apps.PreflibappConfig', - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'django_distill', + "preflibapp.apps.PreflibappConfig", + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "django_distill", ] MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", ] -ROOT_URLCONF = 'preflib.urls' +ROOT_URLCONF = "preflib.urls" TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [os.path.join(BASE_DIR, 'templates')], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [os.path.join(BASE_DIR, "templates")], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", ], }, }, ] -WSGI_APPLICATION = 'preflib.wsgi.application' +WSGI_APPLICATION = "preflib.wsgi.application" # Database # https://docs.djangoproject.com/en/2.2/ref/settings/#databases -DATABASES = { -} +DATABASES = {} # Password validation @@ -76,16 +75,16 @@ AUTH_PASSWORD_VALIDATORS = [ { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", }, ] @@ -93,9 +92,9 @@ # Internationalization # https://docs.djangoproject.com/en/2.2/topics/i18n/ -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = "en-us" -TIME_ZONE = 'UTC' +TIME_ZONE = "UTC" USE_I18N = True @@ -104,12 +103,11 @@ USE_TZ = True +STATIC_URL = "/static/" -STATIC_URL = '/static/' - -LOGIN_URL = '/login' +LOGIN_URL = "/login" # Auto primary keys -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' +DEFAULT_AUTO_FIELD = "django.db.models.AutoField" from preflib.local_settings import * diff --git a/preflib/urls.py b/preflib/urls.py index 130825b..5fce1a3 100644 --- a/preflib/urls.py +++ b/preflib/urls.py @@ -13,16 +13,17 @@ 1. Import the include() function: from django.urls import include, path 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ + from django.contrib import admin from django.urls import path, include from django.conf.urls import handler400, handler403, handler404, handler500 -handler400 = 'preflibapp.views.error_400_view' -handler403 = 'preflibapp.views.error_403_view' -handler404 = 'preflibapp.views.error_404_view' -handler500 = 'preflibapp.views.error_500_view' +handler400 = "preflibapp.views.error_400_view" +handler403 = "preflibapp.views.error_403_view" +handler404 = "preflibapp.views.error_404_view" +handler500 = "preflibapp.views.error_500_view" urlpatterns = [ - path('djangoadmin/', admin.site.urls), - path('', include('preflibapp.urls')) + path("djangoadmin/", admin.site.urls), + path("", include("preflibapp.urls")), ] diff --git a/preflib/wsgi.py b/preflib/wsgi.py index 3ba442d..635f0cc 100644 --- a/preflib/wsgi.py +++ b/preflib/wsgi.py @@ -11,6 +11,6 @@ from django.core.wsgi import get_wsgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'preflib.settings') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "preflib.settings") application = get_wsgi_application() diff --git a/preflibapp/admin.py b/preflibapp/admin.py index 5996cee..e300657 100644 --- a/preflibapp/admin.py +++ b/preflibapp/admin.py @@ -8,4 +8,4 @@ admin.site.register(Metadata) admin.site.register(DataProperty) admin.site.register(Paper) -admin.site.register(Log) \ No newline at end of file +admin.site.register(Log) diff --git a/preflibapp/apps.py b/preflibapp/apps.py index 1dd4476..693b4a5 100644 --- a/preflibapp/apps.py +++ b/preflibapp/apps.py @@ -2,4 +2,4 @@ class PreflibappConfig(AppConfig): - name = 'preflibapp' + name = "preflibapp" diff --git a/preflibapp/choices.py b/preflibapp/choices.py index 35a1b68..6931c0e 100644 --- a/preflibapp/choices.py +++ b/preflibapp/choices.py @@ -1,5 +1,5 @@ def find_choice_value(choices, key): - for (k, v) in choices: + for k, v in choices: if k == key: return v return None @@ -13,35 +13,32 @@ def is_choice(choices, choice): DATATYPES = [ - ('soc', 'strict order complete'), - ('soi', 'strict order incomplete'), - ('toc', 'tie order complete'), - ('toi', 'tie order incomplete'), - ('cat', 'categorical'), + ("soc", "strict order complete"), + ("soi", "strict order incomplete"), + ("toc", "tie order complete"), + ("toi", "tie order incomplete"), + ("cat", "categorical"), # ('tog', 'tournament graph'), # ('mjg', 'majority graph'), # ('wmg', 'weighted majority graph'), # ('pwg', 'pairwise graph'), - ('wmd', 'weighted matching data'), - ('dat', 'extra data file'), - ('csv', 'comma-separated values') + ("wmd", "weighted matching data"), + ("dat", "extra data file"), + ("csv", "comma-separated values"), ] MODIFICATIONTYPES = [ - ('original', 'original'), - ('induced', 'induced'), - ('imbued', 'imbued'), - ('synthetic', 'synthetic') + ("original", "original"), + ("induced", "induced"), + ("imbued", "imbued"), + ("synthetic", "synthetic"), ] METADATACATEGORIES = [ - ('general', 'general properties'), - ('preference', 'preference structure'), - ('ballot', 'ballot structure'), - ('aggregation', 'aggregtated structure') + ("general", "general properties"), + ("preference", "preference structure"), + ("ballot", "ballot structure"), + ("aggregation", "aggregtated structure"), ] -SEARCHWIDGETS = [ - ('ternary', 'ternary choices'), - ('range', 'range') -] +SEARCHWIDGETS = [("ternary", "ternary choices"), ("range", "range")] diff --git a/preflibapp/management/commands/adddataset.py b/preflibapp/management/commands/adddataset.py index 03f466e..c6a74ca 100644 --- a/preflibapp/management/commands/adddataset.py +++ b/preflibapp/management/commands/adddataset.py @@ -2,168 +2,131 @@ from django.contrib.staticfiles import finders from django.core import management from django.db.models import Max +from preflibtools.instances.dataset import read_info_file import preflibapp from preflibapp.models import * -from preflibtools.instances.preflibinstance import OrdinalInstance, CategoricalInstance, MatchingInstance +from preflibtools.instances.preflibinstance import ( + OrdinalInstance, + CategoricalInstance, + MatchingInstance, +) import traceback import zipfile import os -def read_info_file(file_name): - infos = {'files': {}} - with open(file_name, 'r') as file: - # We go line per line trying to match the beginning of the line to a known header tag - lines = file.readlines() - line_index = 0 - for line_index in range(len(lines)): - line = lines[line_index] - if len(line) > 1: - if line.startswith('Name:'): - infos['name'] = line[5:].strip() - elif line.startswith('Abbreviation:'): - infos['abb'] = line[13:].strip() - elif line.startswith('Tags:'): - infos['tags'] = [tag.strip() for tag in line[5:].strip().split(',')] - elif line.startswith('Series Number:'): - infos['series'] = line[14:].strip() - elif line.startswith('Publication Date:'): - infos['publication_date'] = line[17:].strip() - elif line.startswith('Description:'): - infos['description'] = line[12:].strip() - elif line.startswith('Required Citations:'): - infos['citations'] = line[19:].strip() if line[19:].strip() != "None" else "" - elif line.startswith('Selected Studies:'): - infos['studies'] = line[17:].strip() if line[17:].strip() != "None" else "" - elif line.startswith('file_name, modification_type, relates_to, title, description, publication_date'): - break - # We are now reading the description of the files - for line in lines[line_index + 1:]: - line = line.strip() - if len(line) > 0: - split_line = line.split(',') - new_split_line = [] - inside_quotes = False - tmp_split = '' - for split in split_line: - split = split.strip() - if len(split) > 0: - if inside_quotes: - if split[-3:] == '"""': - tmp_split += split[:-3] - new_split_line.append(tmp_split) - inside_quotes = False - else: - tmp_split += split + ', ' - else: - if split[0:3] == '"""': - tmp_split += split[3:] + ', ' - inside_quotes = True - else: - new_split_line.append(split) - else: - new_split_line.append('') - infos['files'][new_split_line[0].strip()] = { - 'file_name': new_split_line[0].strip(), - 'modification_type': new_split_line[1].strip(), - 'relates_to': new_split_line[2].strip(), - 'title': new_split_line[3].strip(), - 'description': new_split_line[4].strip(), - 'publication_date': new_split_line[5].strip() - } - return infos - - def add_dataset(file_path, tmp_dir, data_dir, keepzip, log): # We start by extracting the zip file - with zipfile.ZipFile(file_path, 'r') as archive: + with zipfile.ZipFile(file_path, "r") as archive: archive.extractall(tmp_dir) # We try to read and parse the info file, if we don't find the info.txt file, we skip the dataset if os.path.exists(os.path.join(tmp_dir, "info.txt")): infos = read_info_file(os.path.join(tmp_dir, "info.txt")) else: - raise Exception("No info.txt file has been found for " + str(file_path) + " ... skipping it.") + raise Exception( + "No info.txt file has been found for " + + str(file_path) + + " ... skipping it." + ) # Now that we have all the infos, we can create the dataset object in the database dataset_obj, _ = DataSet.objects.update_or_create( - abbreviation=infos['abb'], + abbreviation=infos["abb"], defaults={ - 'name': infos['name'], - 'series_number': infos['series'], - 'zip_file_path': None, - 'zip_file_size': 0, - 'description': infos['description'], - 'required_citations': infos['citations'], - 'selected_studies': infos['studies'], - 'publication_date': infos['publication_date']}) + "name": infos["name"], + "series_number": infos["series"], + "zip_file_path": None, + "zip_file_size": 0, + "description": infos["description"], + "required_citations": infos["citations"], + "selected_studies": infos["studies"], + "publication_date": infos["publication_date"], + }, + ) # We add the tags, creating them in the database if needed for tag in infos["tags"]: tag_obj, _ = DataTag.objects.get_or_create( - name=tag, - defaults={ - 'description': 'No description provided.' - } + name=tag, defaults={"description": "No description provided."} ) dataset_obj.tags.add(tag_obj) dataset_obj.save() # We create a folder for the dataset in the data folder try: - os.makedirs(os.path.join(data_dir, infos['abb'])) + os.makedirs(os.path.join(data_dir, infos["abb"])) except FileExistsError: pass - if os.path.exists(os.path.join(data_dir, infos['abb'], "info.txt")): - os.remove(os.path.join(data_dir, infos['abb'], "info.txt")) - os.rename(os.path.join(tmp_dir, "info.txt"), os.path.join(data_dir, infos['abb'], "info.txt")) + if os.path.exists(os.path.join(data_dir, infos["abb"], "info.txt")): + os.remove(os.path.join(data_dir, infos["abb"], "info.txt")) + os.rename( + os.path.join(tmp_dir, "info.txt"), + os.path.join(data_dir, infos["abb"], "info.txt"), + ) # Let's now add the datafiles to the database relates_to_dict = {} for file_name in os.listdir(tmp_dir): - extension = os.path.splitext(file_name)[1][1:] # Using [1:] here to remove the dot + extension = os.path.splitext(file_name)[1][ + 1: + ] # Using [1:] here to remove the dot # We only do it if it actually is a file we're interested in if is_choice(DATATYPES, extension): print("\t{}".format(file_name)) # Move the file to the folder of the dataset - if os.path.exists(os.path.join(data_dir, infos['abb'], file_name)): - os.remove(os.path.join(data_dir, infos['abb'], file_name)) - os.rename(os.path.join(tmp_dir, file_name), os.path.join(data_dir, infos['abb'], file_name)) + if os.path.exists(os.path.join(data_dir, infos["abb"], file_name)): + os.remove(os.path.join(data_dir, infos["abb"], file_name)) + os.rename( + os.path.join(tmp_dir, file_name), + os.path.join(data_dir, infos["abb"], file_name), + ) # Parsing the parsable files or looking through the infos we collected to see if the file appears there if extension in ["soc", "soi", "toc", "toi", "cat", "wmd"]: if extension == "cat": - instance = CategoricalInstance(os.path.join(data_dir, infos['abb'], file_name)) + instance = CategoricalInstance( + os.path.join(data_dir, infos["abb"], file_name) + ) elif extension == "wmd": - instance = MatchingInstance(os.path.join(data_dir, infos['abb'], file_name)) + instance = MatchingInstance( + os.path.join(data_dir, infos["abb"], file_name) + ) else: - instance = OrdinalInstance(os.path.join(data_dir, infos['abb'], file_name)) + instance = OrdinalInstance( + os.path.join(data_dir, infos["abb"], file_name) + ) file_info = { - 'modification_type': instance.modification_type, - 'title': instance.title, - 'description': instance.description, - 'relates_to': instance.relates_to, - 'publication_date': instance.publication_date, + "modification_type": instance.modification_type, + "title": instance.title, + "description": instance.description, + "relates_to": instance.relates_to, + "publication_date": instance.publication_date, } else: - file_info = infos['files'].get(file_name) + file_info = infos["files"].get(file_name) if not file_info: file_info = { - 'modification_type': '-', - 'title': '', - 'description': '-', - 'relates_to': '', - 'publication_date': timezone.now(), + "modification_type": "-", + "title": "", + "description": "-", + "relates_to": "", + "publication_date": timezone.now(), } print("No info found for {}".format(file_name)) - log.append("\n\n

No info has been found for the file " + str(file_name) + - " in the info file of " + str(file_path) + "

\n\n

No info has been found for the file " + + str(file_name) + + " in the info file of " + + str(file_path) + + "

\n