diff --git a/coldfront/core/department/templates/department/department_detail.html b/coldfront/core/department/templates/department/department_detail.html index 64db0d728..0abae2927 100644 --- a/coldfront/core/department/templates/department/department_detail.html +++ b/coldfront/core/department/templates/department/department_detail.html @@ -89,12 +89,12 @@

Usage Summary For: {{ departm {% comment %} {% endcomment %} - + {% else %} Your Labs: @@ -113,7 +113,7 @@

Usage Summary For: {{ departm {{ allocation_users.count }} - Amount Due for Projects: + Total Amount Due, Monthly Storage: ${{ storage_full_price|floatformat:2 }} {% comment %} {% endcomment %} @@ -224,7 +224,7 @@

Monthly Stor
-

Quarterly Compute Invoice Details

+

Quarterly Cluster Usage Details

{{compute_pi_dict.keys|length}}
@@ -238,7 +238,7 @@

Quarterly Co Allocation Users Quota (TB) - Quarterly Cost to Date + @@ -256,7 +256,6 @@

Quarterly Co {% for allocation in allocs %} - {% if allocation.resources.first.resource_type.name == "Cluster" %} @@ -274,9 +273,8 @@

Quarterly Co {{ allocation.allocationuser_set.count }} {{ allocation.size|floatformat:1 }} - ${{ allocation.cost|floatformat:2 }} + - {% endif %} {% endfor %} {% endfor %} diff --git a/coldfront/core/utils/management/commands/add_scheduled_tasks.py b/coldfront/core/utils/management/commands/add_scheduled_tasks.py index f59dd6fdd..9ee297ddb 100644 --- a/coldfront/core/utils/management/commands/add_scheduled_tasks.py +++ b/coldfront/core/utils/management/commands/add_scheduled_tasks.py @@ -28,6 +28,8 @@ def handle(self, *args, **options): 'fasrc': ['import_quotas', 'id_import_allocations'], 'sftocf': ['pullsf_pushcf_redash', 'pull_resource_data'], 'ldap': ['update_group_membership_ldap', 'id_add_projects'], + 'slurm': ['slurm_sync'], + 'xdmod': ['xdmod_usage'], } scheduled = [task.func for task in Schedule.objects.all()] diff --git a/coldfront/core/utils/management/commands/create_superuser.py b/coldfront/core/utils/management/commands/create_superuser.py index 714ceb6ac..4d4fbdd4b 100644 --- a/coldfront/core/utils/management/commands/create_superuser.py +++ b/coldfront/core/utils/management/commands/create_superuser.py @@ -1,10 +1,6 @@ -import os - from django.conf import settings -from django.contrib.auth.models import Group from django.contrib.auth import get_user_model -from django.core.management.base import BaseCommand, CommandError -from django.core.exceptions import ObjectDoesNotExist +from django.core.management.base import BaseCommand base_dir = settings.BASE_DIR class Command(BaseCommand): @@ -14,7 +10,3 @@ def handle(self, *args, **options): admin_user.is_superuser = True admin_user.is_staff = True admin_user.save() - - for user in get_user_model().objects.all(): - user.set_password('test1234') - user.save() \ No newline at end of file diff --git a/coldfront/core/utils/management/commands/import_add_allocations.py b/coldfront/core/utils/management/commands/import_add_allocations.py index ef9af34b8..1d6631746 100644 --- a/coldfront/core/utils/management/commands/import_add_allocations.py +++ b/coldfront/core/utils/management/commands/import_add_allocations.py @@ -10,8 +10,7 @@ from django.core.exceptions import ValidationError, MultipleObjectsReturned from django.core.management.base import BaseCommand -from coldfront.core.allocation.models import (Allocation, - AllocationUser, +from coldfront.core.allocation.models import (AllocationUser, AllocationAttribute, AllocationAttributeType, AllocationStatusChoice, diff --git a/coldfront/core/utils/management/commands/import_allocationattribute.py b/coldfront/core/utils/management/commands/import_allocationattribute.py index 3171b3234..f0c706773 100644 --- a/coldfront/core/utils/management/commands/import_allocationattribute.py +++ b/coldfront/core/utils/management/commands/import_allocationattribute.py @@ -1,28 +1,12 @@ -import datetime -import os -import json - -from dateutil.relativedelta import relativedelta from django.conf import settings -from django.core.management import call_command from django.core.management.base import BaseCommand from coldfront.core.allocation.models import (Allocation, AllocationAttribute, AllocationAttributeType, - AllocationStatusChoice, - AllocationUser, - AllocationUserStatusChoice) -from coldfront.core.field_of_science.models import FieldOfScience -from coldfront.core.grant.models import (Grant, GrantFundingAgency, - GrantStatusChoice) -from coldfront.core.project.models import (Project, ProjectStatusChoice, - ProjectUser, ProjectUserRoleChoice, - ProjectUserStatusChoice) -from coldfront.core.publication.models import Publication, PublicationSource -from coldfront.core.resource.models import (Resource, ResourceAttribute, - ResourceAttributeType, - ResourceType) -from coldfront.core.user.models import UserProfile + AllocationStatusChoice) +from coldfront.core.resource.models import (Resource, + ResourceAttribute, + ResourceAttributeType) base_dir = settings.BASE_DIR diff --git a/coldfront/core/utils/management/commands/import_allocationquotas.py b/coldfront/core/utils/management/commands/import_allocationquotas.py index 52349c16c..560a258ec 100644 --- a/coldfront/core/utils/management/commands/import_allocationquotas.py +++ b/coldfront/core/utils/management/commands/import_allocationquotas.py @@ -1,19 +1,13 @@ import csv -import datetime import os -import json import logging -from dateutil.relativedelta import relativedelta from django.conf import settings -from django.contrib.auth import get_user, get_user_model -from django.core.management import call_command from django.core.management.base import BaseCommand from coldfront.core.allocation.models import (Allocation, AllocationAttribute, AllocationAttributeType) from coldfront.core.project.models import (Project) -from coldfront.core.resource.models import (Resource, ) from coldfront.config.env import ENV import pandas as pd @@ -38,14 +32,14 @@ def add_arguments(self, parser): default='tier0', help='Storage tier', ) - + def handle(self, *args, **options): LOCALDATA_ROOT = ENV.str('LOCALDATA_ROOT', default=base_dir) - storage = options['storage'] + storage = options['storage'] print(storage) fileName = storage + "_allocation.csv" - tier = options['tier'] + tier = options['tier'] resource_name = storage + '/' + tier print("Loading data for: " + resource_name) lab_list_file = os.path.join(LOCALDATA_ROOT, 'local_data/',fileName) @@ -66,14 +60,14 @@ def handle(self, *args, **options): try: filtered_query = Project.objects.get(title = lab_name) # find project allocations = Allocation.objects.filter(project = filtered_query, resources__name=resource_name, status__name='Active') - if(allocations.count() == 0): + if allocations.count() == 0: print("Allocation not found:" + lab_name + ": "+resource_name) tocsv = [lab_name,resource_name,"Allocation"] - writer.writerow(tocsv) + writer.writerow(tocsv) continue allocation= allocations[0] - if (allocation): # get allocation + if allocation: # get allocation allocation_attribute_type_obj = AllocationAttributeType.objects.get( name='Storage Quota (TB)') try: @@ -87,28 +81,27 @@ def handle(self, *args, **options): except AllocationAttribute.DoesNotExist: allocation_attribute_exist = False - if (not allocation_attribute_exist): + if not allocation_attribute_exist: allocation_attribute_obj,_ =AllocationAttribute.objects.get_or_create( allocation_attribute_type=allocation_attribute_type_obj, allocation=allocation, value = lab_allocation) allocation_attribute_type_obj.save() - + allocation_attribute_obj.allocationattributeusage.value = lab_usage allocation_attribute_obj.allocationattributeusage.save() - + allocation_attribute_type_payment = AllocationAttributeType.objects.get( name='RequiresPayment') allocation_attribute_payment, _ = AllocationAttribute.objects.get_or_create( allocation_attribute_type=allocation_attribute_type_payment, allocation=allocation, - value=True) + value=True) allocation_attribute_payment.save() except Project.DoesNotExist: print("Project not found: " + lab_name) tocsv = [lab_name,resource_name,"Project"] - writer.writerow(tocsv) + writer.writerow(tocsv) continue proj_allocation.close() - diff --git a/coldfront/core/utils/management/commands/import_allocationuser.py b/coldfront/core/utils/management/commands/import_allocationuser.py index f57f1a23f..54596f05c 100644 --- a/coldfront/core/utils/management/commands/import_allocationuser.py +++ b/coldfront/core/utils/management/commands/import_allocationuser.py @@ -87,20 +87,6 @@ def handle(self, *args, **options): with open(lab_data) as f: data = json.load(f) - # if (not filtered_query): # if not found project, then create project - # project_obj, _ = Project.objects.get_or_create( - # #pi = pi1, - # title = lab_name, - # description= lab_name + ' storage allocation', - # field_of_science=FieldOfScience.objects.get( - # #FIXME: Description of the DEPT - # description='Other'), - # status=ProjectStatusChoice.objects.get(name='Active'), - # force_review=True - # ) - # start_date = datetime.datetime.now() - # end_date = datetime.datetime.now() + relativedelta(days=365) - # else: # found project allocations = Allocation.objects.filter(project=filtered_query, resources__name=resource_name, status__name='Active') if allocations.count() == 0: @@ -140,22 +126,17 @@ def handle(self, *args, **options): lab_usage_in_tb = round(lab_usage_in_tb, 2) # lab_usage_in_tb_str = str(lab_usage_in_tb) allocation= allocations[0] - if (allocation): # get allocation + if allocation: # get allocation allocation_attribute_type_obj = AllocationAttributeType.objects.get( name='Storage Quota (TB)') - try: - allocation_attribute_obj = AllocationAttribute.objects.get( - allocation_attribute_type=allocation_attribute_type_obj, - allocation=allocation, - ) - allocation_attribute_obj.value = lab_allocation_in_tb_str - allocation_attribute_obj.save() - allocation_attribute_exist = True - except AllocationAttribute.DoesNotExist: - allocation_attribute_exist = False - - if (not allocation_attribute_exist): - allocation_attribute_obj,_ =AllocationAttribute.objects.get_or_create( + allocation_attribute_obj, created = AllocationAttribute.objects.get_or_create( + allocation_attribute_type=allocation_attribute_type_obj, + allocation=allocation, + defaults={'value': lab_allocation_in_tb_str} + ) + + if created: + allocation_attribute_obj, _ = AllocationAttribute.objects.get_or_create( allocation_attribute_type=allocation_attribute_type_obj, allocation=allocation, value = lab_allocation_in_tb_str) @@ -175,9 +156,9 @@ def handle(self, *args, **options): # checking my user_json_dictinary # loop through my allocation_users set for allocation_user in allocation_users: - allocation_user_username = (allocation_user.user.username) + allocation_user_username = allocation_user.user.username + user_obj = get_user_model().objects.get(username=allocation_user_username) if allocation_user_username in user_json_dict: - user_obj = get_user_model().objects.get(username = allocation_user_username) allocationuser_obj = AllocationUser.objects.get(user=user_obj) allocationuser_obj.status = AllocationUserStatusChoice.objects.get(name='Active') one_user_logical_usage = user_json_dict[allocation_user_username]['logical_usage'] @@ -189,20 +170,16 @@ def handle(self, *args, **options): allocation.save() user_json_dict.pop(allocation_user_username) else: - try: - user_obj = get_user_model().objects.get(username = allocation_user_username) - allocationuser_obj = AllocationUser.objects.get(user=user_obj) - allocationuser_obj.status = AllocationUserStatusChoice.objects.get(name='Removed') - allocationuser_obj.usage = 0 - allocationuser_obj.usage_bytes = 0 - allocationuser_obj.unit = '' - allocationuser_obj.allocation_group_usage_bytes = lab_data["kbytes"] - allocationuser_obj.allocation_group_quota = lab_data["quota"] - allocationuser_obj.save() - allocation.save() - except Exception as e: - print(f'Error: {e}') - # allocation_users.remove(allocation_user) # remove this particular allocation_user + allocationuser_obj, _ = AllocationUser.objects.update_or_create( + user=user_obj, + allocation=allocation, + defaults={ + 'status': AllocationUserStatusChoice.objects.get(name='Removed'), + 'usage': 0, + 'usage_bytes': 0, + 'unit': '', + } + ) for json_user in user_json_dict: try: @@ -228,31 +205,18 @@ def handle(self, *args, **options): # ) # get_user_model().objects.get(username=json_user).save() - # raise Exception(f'Cannot find user {json_user}') - - try: - allocationuser_obj = AllocationUser.objects.get(user=user_obj) - print(allocationuser_obj.status) - allocationuser_obj.status= AllocationUserStatusChoice.objects.get(name='Active') - except AllocationUser.DoesNotExist: - # create allocationuser object - allocationuser_obj = AllocationUser( - allocation=allocation, - user=user_obj, - status=AllocationUserStatusChoice.objects.get(name='Active'), - ) - - # only updating allocation user object usage_string = user_json_dict[json_user]['usage'] num, alpha = splitString(usage_string) - allocationuser_obj.usage = num - allocationuser_obj.usage_bytes = user_json_dict[json_user]['logical_usage'] - allocationuser_obj.unit = alpha - allocationuser_obj.allocation_group_usage_bytes = lab_data["kbytes"] - allocationuser_obj.allocation_group_quota = lab_data["quota"] - allocationuser_obj.save() - allocation.save() - # get_user_model().objects.get(username=json_user).save() + allocationuser_obj, created = AllocationUser.objects.get_or_create( + user=user_obj, + allocation=allocation, + defaults={ + 'status':AllocationUserStatusChoice.objects.get(name='Active'), + 'usage': num, + 'usage_bytes': user_json_dict[json_user]['logical_usage'], + 'unit': alpha, + } + ) + except Exception as e: - # logger.exception(e) print(f'Error: {e}') diff --git a/coldfront/core/utils/management/commands/import_projects_fas.py b/coldfront/core/utils/management/commands/import_projects_fas.py index 03bca2d57..c1eb11eb7 100644 --- a/coldfront/core/utils/management/commands/import_projects_fas.py +++ b/coldfront/core/utils/management/commands/import_projects_fas.py @@ -4,7 +4,6 @@ import datetime from django.conf import settings -from django.contrib.auth.models import Group from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand from django.utils import timezone @@ -14,7 +13,6 @@ from coldfront.core.project.models import (Project, ProjectStatusChoice, ProjectUser, ProjectUserRoleChoice, ProjectUserStatusChoice) -from coldfront.core.user.models import (UserProfile) from coldfront.config.env import ENV base_dir = settings.BASE_DIR @@ -120,12 +118,12 @@ def handle(self, *args, **options): writer.writerow(tocsv) continue project_obj = Project.objects.get(title = title) - if (project_obj != ""): + if project_obj != "": for project_user in user_dict: - if (project_user != ""): + if project_user != "": username = project_user['samaccountname'] enable_email = False - if (username == pi_username): + if username == pi_username: role = 'PI' enable_email = True else: diff --git a/coldfront/core/utils/management/commands/import_publications.py b/coldfront/core/utils/management/commands/import_publications.py index 950b04b39..522b7f14b 100644 --- a/coldfront/core/utils/management/commands/import_publications.py +++ b/coldfront/core/utils/management/commands/import_publications.py @@ -4,10 +4,7 @@ from django.conf import settings from django.core.management.base import BaseCommand -from coldfront.core.field_of_science.models import FieldOfScience -from coldfront.core.project.models import (Project, ProjectStatusChoice, - ProjectUser, ProjectUserRoleChoice, - ProjectUserStatusChoice) +from coldfront.core.project.models import Project from coldfront.core.publication.models import Publication, PublicationSource base_dir = settings.BASE_DIR diff --git a/coldfront/core/utils/management/commands/import_subscriptions.py b/coldfront/core/utils/management/commands/import_subscriptions.py deleted file mode 100644 index a137945a7..000000000 --- a/coldfront/core/utils/management/commands/import_subscriptions.py +++ /dev/null @@ -1,166 +0,0 @@ -import csv -import datetime -import os - -from django.conf import settings -from django.contrib.auth.models import Group -from django.contrib.auth import get_user_model -from django.core.management.base import BaseCommand - -from coldfront.core.project.models import Project -from coldfront.core.resource.models import Resource -from coldfront.core.subscription.models import (AttributeType, Subscription, - SubscriptionAttribute, - SubscriptionAttributeType, - SubscriptionAttributeUsage, - SubscriptionStatusChoice, - SubscriptionUser, - SubscriptionUserStatusChoice) - -base_dir = settings.BASE_DIR - - -class Command(BaseCommand): - - def handle(self, *args, **options): - print('Adding subscriptions ...') - AttributeType.objects.all().delete() - filepath = os.path.join(base_dir, 'local_data', 's_atttriute_type.tsv') - with open(filepath, 'r') as fp: - for line in fp: - AttributeType.objects.create(name=line.strip()) - - SubscriptionAttributeType.objects.all().delete() - filepath = os.path.join(base_dir, 'local_data', 'subscription_attribute_type.tsv') - # print(filepath) - with open(filepath, 'r') as fp: - for line in fp: - attribute_type, name, has_usage = line.strip().split('\t') - if has_usage == "True": - has_usage = True - else: - has_usage = False - # print(attribute_type, name, has_usage) - subscription_attribute_type_obj = SubscriptionAttributeType.objects.create( - attribute_type=AttributeType.objects.get(name=attribute_type), - name=name, - has_usage=has_usage - ) - # print(subscription_attribute_type_obj) - - Subscription.objects.all().delete() - SubscriptionUser.objects.all().delete() - SubscriptionAttribute.objects.all().delete() - SubscriptionAttributeUsage.objects.all().delete() - - subscription_status_choices = {} - subscription_status_choices['ACT'] = SubscriptionStatusChoice.objects.get(name='Active') - subscription_status_choices['PEN'] = SubscriptionStatusChoice.objects.get(name='Pending') - # subscription_status_choices['EXP'] = SubscriptionStatusChoice.objects.get(name='Expired') - # subscription_status_choices['DEN'] = SubscriptionStatusChoice.objects.get(name='Denied') - # subscription_status_choices['REV'] = SubscriptionStatusChoice.objects.get(name='Revoked') - subscription_status_choices['UNP'] = SubscriptionStatusChoice.objects.get(name='Unpaid') - subscription_status_choices['NEW'] = SubscriptionStatusChoice.objects.get(name='New') - # subscription_status_choices['INA'] = SubscriptionStatusChoice.objects.get(name='Inactive (Renewed)') - subscription_status_choices['APR'] = SubscriptionStatusChoice.objects.get(name='Approved') - - subscription_user_status_choices = {} - subscription_user_status_choices['ACT'] = SubscriptionUserStatusChoice.objects.get(name='Active') - # subscription_user_status_choices['PEA'] = SubscriptionUserStatusChoice.objects.get(name='Denied') - # subscription_user_status_choices['PER'] = SubscriptionUserStatusChoice.objects.get(name='Pending - Add') - # subscription_user_status_choices['DEN'] = SubscriptionUserStatusChoice.objects.get(name='Pending - Remove') - subscription_user_status_choices['REM'] = SubscriptionUserStatusChoice.objects.get(name='Removed') - - filepath = os.path.join(base_dir, 'local_data', 'subscriptions.tsv') - with open(filepath, 'r') as fp: - lines = fp.read().split('$$$$$$$$$$-new-line-$$$$$$$$$$') - for line in lines: - if not line.strip(): - continue - created, modified, title, pi_username, project_status, quantity, resource_name, status, active_util, justification, attributes, users = line.split( - '\t') - - # print(title, pi_username, project_status, quantity, resource_name, status, active_util, justification, attributes, users) - - # print(title, pi_username, project_status) - pi_user = get_user_model().objects.get(username=pi_username) - try: - project_obj = Project.objects.get( - title=title.strip(), pi__username=pi_username.strip(), status__name=project_status) - except: - print(title.strip(), pi_username.strip()) - - resource_obj = Resource.objects.get(name=resource_name) - end_date_datetime_obj = datetime.datetime.strptime(active_util, '%Y-%m-%d') - - created = datetime.datetime.strptime(created.strip(), '%Y-%m-%d %H:%M:%S').date() - modified = datetime.datetime.strptime(modified.strip(), '%Y-%m-%d %H:%M:%S').date() - subscription_obj = Subscription.objects.create( - created=created, - modified=modified, - project=project_obj, - status=subscription_status_choices[status], - end_date=end_date_datetime_obj, - quantity=int(quantity), - justification=justification.strip() - ) - subscription_obj.resources.add(resource_obj) - - for linked_resource in resource_obj.linked_resources.all(): - subscription_obj.resources.add(linked_resource) - - subscription_attribute_type_obj = SubscriptionAttributeType.objects.get(name='slurm_user_specs') - subscription_attribute_obj = SubscriptionAttribute.objects.create( - subscription=subscription_obj, - subscription_attribute_type=subscription_attribute_type_obj, - value='Fairshare=parent', - is_private=True - ) - - if attributes != 'N/A': - for subscription_attribute in attributes.split(';'): - name, value, is_private, usage_value = subscription_attribute.split(',') - if is_private == 'True': - is_private = True - else: - is_private = False - # print(subscription_attribute) - subscription_attribute_type_obj = SubscriptionAttributeType.objects.get(name=name) - subscription_attribute_obj = SubscriptionAttribute.objects.create( - subscription=subscription_obj, - subscription_attribute_type=subscription_attribute_type_obj, - value=value, - is_private=is_private - ) - - if subscription_attribute_obj.subscription_attribute_type.has_usage and usage_value != 'N/A': - subscription_attribute_usage_obj = SubscriptionAttributeUsage( - subscription_attribute=subscription_attribute_obj, - value=usage_value - ) - subscription_attribute_usage_obj.save() - - try: - if users != 'N/A': - for user in users.split(';'): - username, user_status = user.split(',') - user_obj = get_user_model().objects.get(username=username) - subscription_user_obj = SubscriptionUser.objects.create( - subscription=subscription_obj, - user=user_obj, - status=subscription_user_status_choices[user_status] - ) - except: - print(title, pi_username, project_status, quantity, resource_name, - status, active_util, justification, attributes, users) - - # subscription_user_array = [] - # for subscription_user in subscription.subscriptionuserstatus_set.all(): - # subscription_user_array.append(','.join((subscription_user.user.username, subscription_user.status))) - - # subscription_user_array_joined = ';'.join(subscription_user_array) - # row.append(subscription_user_array_joined) - # print(row) - # csvfile.writerow(row) - - print('Finished adding subscriptions.') diff --git a/coldfront/core/utils/management/commands/import_users_from_files.py b/coldfront/core/utils/management/commands/import_users_from_files.py index eb65be8a2..545f8208c 100644 --- a/coldfront/core/utils/management/commands/import_users_from_files.py +++ b/coldfront/core/utils/management/commands/import_users_from_files.py @@ -4,8 +4,6 @@ from django.contrib.auth.models import Group from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand -from django.core.exceptions import ObjectDoesNotExist -from django.db.utils import OperationalError from csv import reader @@ -16,68 +14,47 @@ class Command(BaseCommand): def handle(self, *args, **options): print('Adding users now ...') - file_path = os.path.join(base_dir, 'local_data/labs') - print("line27, file path is", file_path) - # arr = os.listdir() - arr = os.listdir(file_path) + local_path = os.path.join(base_dir, 'local_data/labs') + files = os.listdir(local_path) - print("type of arr is", type(arr)) - print("line 30",arr) lab_list = [] - for f in arr: + for f in files: f_name = f.split('.') - if (f_name[len(f_name)-1] == 'csv'): + if f_name[len(f_name)-1] == 'csv': my_file = f_name[len(f_name)-2]+('.csv') lab_list.append(my_file) - print("line 41: lab_list", lab_list) for lab in lab_list: - print("line 39 lab is:", lab) file_name = lab lab_list = list(lab.split('.')) lab_name = lab_list[0] file_path = os.path.join(base_dir, 'local_data/labs', file_name) - print("line 43 file path is:", file_path) - # if (file_name != "rc_admin"): - # open file in read mode - with open (file_path, 'r') as read_obj: + with open(file_path, 'r') as read_obj: csv_reader = reader(read_obj) # opt out the first line first_line = read_obj.readline() for row in csv_reader: - try: - username = row[0] - print("line52, my username is", username) - user = get_user_model().objects.get(username=username) - # print("line 54 user is", user) - # print("line47",username, "already exist, don't add to database") - # if the user exists, I only need to append this existing user's group - if not user.groups.filter(name = lab_name).exists(): - print("line 45",lab_name) - my_group = Group.objects.get(name=lab_name) - my_group.user_set.add(user) - print ("user do not exist in", lab_name) - continue - # the type of row is - except ObjectDoesNotExist: - - print("jumped to line 68") - username = row[0] - full_name = row[1] - full_name_list = full_name.split() - first_name = full_name_list[0] - - if (len(full_name_list) > 1): - last_name = full_name_list[1] - - else: - last_name = "N/A" - print("line76 my username is", username) - print("line77 my fullname is", full_name) - - email = row[2] - is_active = True - is_staff = False - is_superuser = False + username = row[0] + full_name = row[1].split() + user, created = get_user_model().objects.get_or_create( + username=username, + defaults={ + 'username': username, + 'first_name': full_name[0], + 'last_name':full_name[-1], + 'email': row[2], + 'is_active': True, + 'is_staff': False, + 'is_superuser': False, + + } + ) + # if the user exists, I only need to append this existing user's group + if not user.groups.filter(name=lab_name).exists(): + my_group = Group.objects.get(name=lab_name) + my_group.user_set.add(user) + print ("user do not exist in", lab_name) + + if created: groups = lab_name # creates my user object to load data from csv to GUI @@ -87,62 +64,8 @@ def handle(self, *args, **options): group_obj, _ = Group.objects.get_or_create(name=group.strip()) # gets u the group object based on the group name group_objs.append(group_obj) - - user_obj = get_user_model().objects.create( - username=username, - first_name=first_name, - last_name=last_name, - email=email, - is_active=is_active, - is_staff=is_staff, - is_superuser=is_superuser, - ) # add user to group if group_objs: - user_obj.groups.add(*group_objs) # add the group object to the user - user_obj.save() - except OperationalError: - print("line110") - print("jumped to line 64") - username = row[0] - full_name = row[1] - full_name_list = full_name.split() - first_name = full_name_list[0] - - - if (len(full_name_list) > 1): - last_name = full_name_list[1] - - else: - last_name = "N/A" - print("line124 my username is", username) - print("line125 my fullname is", full_name) - - email = row[2] - is_active = True - is_staff = False - is_superuser = False - groups = lab_name - - # creates my user object to load data from csv to GUI - # create user object - group_objs = [] - for group in groups.split(','): - group_obj, _ = Group.objects.get_or_create(name=group.strip()) # gets u the group object based on the group name - group_objs.append(group_obj) - - - user_obj = get_user_model().objects.create( - username=username, - first_name=first_name, - last_name=last_name, - email=email, - is_active=is_active, - is_staff=is_staff, - is_superuser=is_superuser, - ) - # # add user to group - if group_objs: - user_obj.groups.add(*group_objs) # add the group object to the user - user_obj.save() + user.groups.add(*group_objs) # add the group object to the user + user.save() print('Finished adding users for lab:', lab) diff --git a/coldfront/core/utils/management/commands/import_users_from_files2.py b/coldfront/core/utils/management/commands/import_users_from_files2.py deleted file mode 100644 index 305a15798..000000000 --- a/coldfront/core/utils/management/commands/import_users_from_files2.py +++ /dev/null @@ -1,179 +0,0 @@ -import datetime -import os - -from django.conf import settings -from django.contrib.auth.models import Group -from django.contrib.auth import get_user_model -from django.core.management.base import BaseCommand, CommandError -from django.core.exceptions import ObjectDoesNotExist - -from csv import reader - -from coldfront.core.field_of_science.models import FieldOfScience -from coldfront.core.project.models import (Project, ProjectStatusChoice, - ProjectUser, ProjectUserRoleChoice, - ProjectUserStatusChoice) -from coldfront.core.allocation.models import (AllocationUser, - AllocationUserStatusChoice) -from coldfront.config.env import ENV, PROJECT_ROOT - -base_dir = settings.BASE_DIR - - -class Command(BaseCommand): - - def handle(self, *args, **options): - print('Adding users now ...') - # lab_list = ['zhuang_lab.csv', 'moorcroft_lab.csv', 'kuang_lab.csv', 'kovac_lab.csv', - # 'holman_lab.csv', 'giribet_lab.csv', 'edwards_lab.csv', 'denolle_lab.csv', - # 'wofsy_lab.csv', 'rc_admin.csv'] - - local_path = os.path.join(base_dir, 'local_data') - - files = os.listdir(local_path) - - lab_list = [] - for f in files: - f_name = f.split('.') - if (f_name[len(f_name)-1] == 'csv'): - if f_name[len(f_name)-2] != 'Quota': - print("line39:",f_name[len(f_name)-2]) - file = f_name[len(f_name)-2]+('.csv') - lab_list.append(file) - - print(lab_list) - for lab in lab_list: - file_name = lab - lab_list = list(lab.split('.')) - lab_name = lab_list[0] - file_path = os.path.join(base_dir, 'local_data', file_name) - print("line34 file_name is:", file_name) - if (file_name != "rc_admin.csv"): - # open file in read mode - with open (file_path, 'r') as read_obj: - csv_reader = reader(read_obj) # opt out the first line - first_line = read_obj.readline() - for row in csv_reader: - try: - username = row[0] - user = get_user_model().objects.get(username=username) - #(username, "already exist, don't add to database") - # if the user exists, I only need to append this existing user's group - if not user.groups.filter(name = lab_name).exists(): - # print("line 45",lab_name) - my_group = Group.objects.get(name=lab_name) - my_group.user_set.add(user) - # print ("user do not exist in", lab_name) - continue - # the type of row is - except ObjectDoesNotExist: - - username = row[0] - full_name = row[1] - full_name_list = full_name.split() - first_name = full_name_list[0] - - - if (len(full_name_list) > 1): - last_name = full_name_list[1] - - else: - last_name = "N/A" - - - email = row[2] - is_active = True - is_staff = False - is_superuser = False - groups = lab_name - - # creates my user object to load data from csv to GUI - # create user object - group_objs = [] - for group in groups.split(','): - group_obj, _ = Group.objects.get_or_create(name=group.strip()) # gets u the group object based on the group name - group_objs.append(group_obj) - - - user_obj = get_user_model().objects.create( - username=username, - first_name=first_name, - last_name=last_name, - email=email, - is_active=is_active, - is_staff=is_staff, - is_superuser=is_superuser, - ) - # add user to group - if group_objs: - user_obj.groups.add(*group_objs) # add the group object to the user - user_obj.save() - print('Finished adding users for lab:', lab) - else: - with open (file_path, 'r') as read_obj: - csv_reader = reader(read_obj) # opt out the first line - first_line = read_obj.readline() - for row in csv_reader: - print("line101 row is:", row) - try: - username = row[0] - user, created = get_user_model().objects.get_or_create(username=username) - #(username, "already exist, don't add to database") - # if the user exists, I only need to append this existing user's group - if not user.groups.filter(name = lab_name).exists(): - print("line 45",lab_name) - my_group = Group.objects.get(name=lab_name) - my_group.user_set.add(user) - print ("user do not exist in", lab_name) - print("line112, created is", created) - print("line113, user is:", user) - if not created: - #user was retrieved - username = row[0] - get_user_model().objects.filter(username=username).update(is_superuser=True, is_staff=True) - - continue - # the type of row is - except ObjectDoesNotExist: - - username = row[0] - full_name = row[1] - full_name_list = full_name.split() - first_name = full_name_list[0] - - - if (len(full_name_list) > 1): - last_name = full_name_list[1] - - else: - last_name = "N/A" - - - email = row[2] - is_active = True - is_staff = True - is_superuser = True - groups = lab_name - - # creates my user object to load data from csv to GUI - # create user object - group_objs = [] - for group in groups.split(','): - group_obj, _ = Group.objects.get_or_create(name=group.strip()) # gets u the group object based on the group name - group_objs.append(group_obj) - - - user_obj = get_user_model().objects.create( - username=username, - first_name=first_name, - last_name=last_name, - email=email, - is_active=is_active, - is_staff=is_staff, - is_superuser=is_superuser, - ) - # add user to group - if group_objs: - user_obj.groups.add(*group_objs) # add the group object to the user - user_obj.save() - print('Finished adding users for lab:', lab) diff --git a/coldfront/core/utils/management/commands/import_users_from_files3.py b/coldfront/core/utils/management/commands/import_users_from_files3.py deleted file mode 100644 index e66ed042c..000000000 --- a/coldfront/core/utils/management/commands/import_users_from_files3.py +++ /dev/null @@ -1,179 +0,0 @@ -import datetime -import os - -from django.conf import settings -from django.contrib.auth.models import Group -from django.contrib.auth import get_user_model -from django.core.management.base import BaseCommand, CommandError -from django.core.exceptions import ObjectDoesNotExist - -from csv import reader - -from coldfront.core.field_of_science.models import FieldOfScience -from coldfront.core.project.models import (Project, ProjectStatusChoice, - ProjectUser, ProjectUserRoleChoice, - ProjectUserStatusChoice) -from coldfront.core.allocation.models import (AllocationUser, - AllocationUserStatusChoice) -from coldfront.config.env import ENV, PROJECT_ROOT - -base_dir = settings.BASE_DIR - - -class Command(BaseCommand): - - def handle(self, *args, **options): - print('Adding users now ...') - # lab_list = ['zhuang_lab.csv', 'moorcroft_lab.csv', 'kuang_lab.csv', 'kovac_lab.csv', - # 'holman_lab.csv', 'giribet_lab.csv', 'edwards_lab.csv', 'denolle_lab.csv', - # 'wofsy_lab.csv', 'rc_admin.csv'] - - local_path = os.path.join(base_dir, 'local_data') - - files = os.listdir(local_path) - - lab_list = [] - for f in files: - f_name = f.split('.') - if (f_name[len(f_name)-1] == 'csv'): - if f_name[len(f_name)-2] != 'Quota': - print("line39:",f_name[len(f_name)-2]) - file = f_name[len(f_name)-2]+('.csv') - lab_list.append(file) - - print(lab_list) - for lab in lab_list: - file_name = lab - lab_list = list(lab.split('.')) - lab_name = lab_list[0] - file_path = os.path.join(base_dir, 'local_data', file_name) - print("line34 file_name is:", file_name) - if (file_name != "rc_admin.csv"): - # open file in read mode - with open (file_path, 'r') as read_obj: - csv_reader = reader(read_obj) # opt out the first line - first_line = read_obj.readline() - for row in csv_reader: - try: - username = row[0] - user = get_user_model().objects.get(username=username) - #(username, "already exist, don't add to database") - # if the user exists, I only need to append this existing user's group - if not user.groups.filter(name = lab_name).exists(): - # print("line 45",lab_name) - my_group = Group.objects.get(name=lab_name) - my_group.user_set.add(user) - # print ("user do not exist in", lab_name) - continue - # the type of row is - except ObjectDoesNotExist: - - username = row[0] - full_name = row[1] - full_name_list = full_name.split() - first_name = full_name_list[0] - - - if (len(full_name_list) > 1): - last_name = full_name_list[1] - - else: - last_name = "N/A" - - - email = row[2] - is_active = True - is_staff = False - is_superuser = False - groups = lab_name - - # creates my user object to load data from csv to GUI - # create user object - group_objs = [] - for group in groups.split(','): - group_obj, _ = Group.objects.get_or_create(name=group.strip()) # gets u the group object based on the group name - group_objs.append(group_obj) - - - user_obj = get_user_model().objects.create( - username=username, - first_name=first_name, - last_name=last_name, - email=email, - is_active=is_active, - is_staff=is_staff, - is_superuser=is_superuser, - ) - # add user to group - if group_objs: - user_obj.groups.add(*group_objs) # add the group object to the user - user_obj.save() - print('Finished adding users for lab:', lab) - else: - with open (file_path, 'r') as read_obj: - csv_reader = reader(read_obj) # opt out the first line - first_line = read_obj.readline() - for row in csv_reader: - print("line101 row is:", row) - try: - username = row[0] - user, created = get_user_model().objects.get_or_create(username=username) - #(username, "already exist, don't add to database") - # if the user exists, I only need to append this existing user's group - if not user.groups.filter(name = lab_name).exists(): - print("line 45",lab_name) - my_group = Group.objects.get(name=lab_name) - my_group.user_set.add(user) - print ("user do not exist in", lab_name) - print("line112, created is", created) - print("line113, user is:", user) - if not created: - #user was retrieved - username = row[0] - get_user_model().objects.filter(username=username).update(is_superuser=True, is_staff=True) - - continue - # the type of row is - except ObjectDoesNotExist: - print("never get to except in line 137") - username = row[0] - full_name = row[1] - full_name_list = full_name.split() - first_name = full_name_list[0] - - - if (len(full_name_list) > 1): - last_name = full_name_list[1] - - else: - last_name = "N/A" - - - email = row[2] - is_active = True - is_staff = True - is_superuser = True - groups = lab_name - - # creates my user object to load data from csv to GUI - # create user object - group_objs = [] - for group in groups.split(','): - group_obj, _ = Group.objects.get_or_create(name=group.strip()) # gets u the group object based on the group name - group_objs.append(group_obj) - - - user_obj = get_user_model().objects.create( - username=username, - first_name=first_name, - last_name=last_name, - email=email, - is_active=is_active, - is_staff=is_staff, - is_superuser=is_superuser, - ) - # add user to group - if group_objs: - user_obj.groups.add(*group_objs) # add the group object to the user - user_obj.save() - print('Finished adding users for lab:', lab) diff --git a/coldfront/core/utils/management/commands/import_users_original.py b/coldfront/core/utils/management/commands/import_users_original.py index 3792cb10e..11b8cbeab 100644 --- a/coldfront/core/utils/management/commands/import_users_original.py +++ b/coldfront/core/utils/management/commands/import_users_original.py @@ -1,17 +1,11 @@ import os from django.conf import settings -from django.contrib.auth.models import Group, UserDataUsage +from django.contrib.auth.models import Group from django.contrib.auth import get_user_model -from django.core.management.base import BaseCommand, CommandError -from django.core.exceptions import ObjectDoesNotExist +from django.core.management.base import BaseCommand base_dir = settings.BASE_DIR -for user in get_user_model().objects.all(): - user.set_password('test1234') - print(user.username) - user.save() - class Command(BaseCommand): def handle(self, *args, **options): @@ -19,7 +13,6 @@ def handle(self, *args, **options): file_path = os.path.join(base_dir, 'local_data', 'users.tsv') print("file path is:",file_path) - with open(file_path, 'r') as fp: for line in fp: if line.startswith('#'): @@ -35,38 +28,31 @@ def handle(self, *args, **options): groups = '' # duplicated user - try: - user = get_user_model().objects.get(username=username) + user_obj, created = get_user_model().objects.get_or_create( + username=username, + defaults={ + 'username': username, + 'first_name': first_name, + 'last_name': last_name, + 'email': email, + 'is_active': is_active, + 'is_staff': is_staff, + 'is_superuser': is_superuser, + } + ) + print(username, first_name, last_name, email, is_active, is_staff, is_superuser, groups) + if not created: print(username, "already exist") - print(username, first_name, last_name, email, is_active, is_staff, is_superuser, groups) - continue - except ObjectDoesNotExist: - print("adding new object") - print(username, first_name, last_name, email, is_active, is_staff, is_superuser, groups) - - group_objs = [] for group in groups.split(','): group_obj, _ = Group.objects.get_or_create(name=group.strip()) group_objs.append(group_obj) - user_obj = get_user_model().objects.create( - username=username, - first_name=first_name, - last_name=last_name, - email=email, - is_active=is_active, - is_staff=is_staff, - is_superuser=is_superuser, - ) - print(group_objs) - - if group_objs: print("group_objs exist") user_obj.groups.add(*group_objs) diff --git a/coldfront/core/utils/management/commands/load_allocationuserdata.py b/coldfront/core/utils/management/commands/load_allocationuserdata.py index 316d811b2..9d07500de 100644 --- a/coldfront/core/utils/management/commands/load_allocationuserdata.py +++ b/coldfront/core/utils/management/commands/load_allocationuserdata.py @@ -101,10 +101,6 @@ def handle(self, *args, **options): admin_user.is_staff = True admin_user.save() - for user in get_user_model().objects.all(): - user.set_password('test1234') - user.save() - for resource in resources: resource_type, parent_resource, name, description, is_available, is_public, is_allocatable = resource diff --git a/coldfront/core/utils/management/commands/load_test_data.py b/coldfront/core/utils/management/commands/load_test_data.py index e3e073d6f..485b23d67 100644 --- a/coldfront/core/utils/management/commands/load_test_data.py +++ b/coldfront/core/utils/management/commands/load_test_data.py @@ -1,10 +1,8 @@ import datetime -import os from dateutil.relativedelta import relativedelta from django.conf import settings from django.contrib.auth import get_user_model -from django.core.management import call_command from django.core.management.base import BaseCommand from coldfront.core.allocation.models import (Allocation, AllocationAttribute, @@ -23,7 +21,6 @@ from coldfront.core.resource.models import (Resource, ResourceAttribute, ResourceAttributeType, ResourceType) -from coldfront.core.user.models import UserProfile base_dir = settings.BASE_DIR @@ -154,11 +151,13 @@ def handle(self, *args, **options): email=email.strip() ) - admin_user, _ = get_user_model().objects.get_or_create(username='admin') - admin_user.is_superuser = True - admin_user.is_staff = True - admin_user.save() - + admin_user, _ = get_user_model().objects.get_or_create( + username='admin', + defaults={ + 'is_superuser': True, + 'is_staff': True + } + ) for user in get_user_model().objects.all(): user.set_password('test1234') user.save() @@ -237,8 +236,7 @@ def handle(self, *args, **options): publication_source = PublicationSource.objects.get(name='doi') - - project_user_obj, _ = ProjectUser.objects.get_or_create( + ProjectUser.objects.get_or_create( user=pi1, project=project_obj, role=ProjectUserRoleChoice.objects.get(name='Manager'), @@ -276,7 +274,7 @@ def handle(self, *args, **options): allocation=allocation_obj, value='Fairshare=parent') - allocation_user_obj = AllocationUser.objects.create( + AllocationUser.objects.create( allocation=allocation_obj, user=pi1, status=AllocationUserStatusChoice.objects.get(name='Active') @@ -330,7 +328,7 @@ def handle(self, *args, **options): allocation=allocation_obj, value='2022-01-01') - allocation_user_obj = AllocationUser.objects.create( + AllocationUser.objects.create( allocation=allocation_obj, user=pi1, status=AllocationUserStatusChoice.objects.get(name='Active') @@ -560,33 +558,49 @@ def handle(self, *args, **options): name='quantity_label'), resource=Resource.objects.get(name='University Cloud'), value='Enter number of compute allocations to purchase') ResourceAttribute.objects.get_or_create(resource_attribute_type=ResourceAttributeType.objects.get( name='quantity_label'), resource=Resource.objects.get(name='ProjectStorage'), value='Enter storage in 1TB increments') - ResourceAttribute.objects.get_or_create(resource_attribute_type=ResourceAttributeType.objects.get( - name='quantity_label'), resource=Resource.objects.get(name='Budgetstorage'), value='Enter storage in 10TB increments (minimum purchase is 10TB)') - - ResourceAttribute.objects.get_or_create(resource_attribute_type=ResourceAttributeType.objects.get( - name='slurm_cluster'), resource=Resource.objects.get(name='Chemistry'), value='chemistry') - ResourceAttribute.objects.get_or_create(resource_attribute_type=ResourceAttributeType.objects.get( - name='slurm_cluster'), resource=Resource.objects.get(name='Physics'), value='physics') - ResourceAttribute.objects.get_or_create(resource_attribute_type=ResourceAttributeType.objects.get( - name='slurm_cluster'), resource=Resource.objects.get(name='Industry'), value='industry') - ResourceAttribute.objects.get_or_create(resource_attribute_type=ResourceAttributeType.objects.get( - name='slurm_cluster'), resource=Resource.objects.get(name='University HPC'), value='university-hpc') - ResourceAttribute.objects.get_or_create(resource_attribute_type=ResourceAttributeType.objects.get( - name='slurm_cluster'), resource=Resource.objects.get(name='University Metered HPC'), + ResourceAttribute.objects.get_or_create( + resource_attribute_type=ResourceAttributeType.objects.get(name='quantity_label'), + resource=Resource.objects.get(name='Budgetstorage'), + value='Enter storage in 10TB increments (minimum purchase is 10TB)') + + ResourceAttribute.objects.get_or_create( + resource_attribute_type=ResourceAttributeType.objects.get(name='slurm_cluster'), + resource=Resource.objects.get(name='Chemistry'), + value='chemistry') + ResourceAttribute.objects.get_or_create( + resource_attribute_type=ResourceAttributeType.objects.get(name='slurm_cluster'), + resource=Resource.objects.get(name='Physics'), value='physics') + ResourceAttribute.objects.get_or_create( + resource_attribute_type=ResourceAttributeType.objects.get(name='slurm_cluster'), + resource=Resource.objects.get(name='Industry'), value='industry') + ResourceAttribute.objects.get_or_create( + resource_attribute_type=ResourceAttributeType.objects.get(name='slurm_cluster'), + resource=Resource.objects.get(name='University HPC'), value='university-hpc') + ResourceAttribute.objects.get_or_create( + resource_attribute_type=ResourceAttributeType.objects.get(name='slurm_cluster'), + resource=Resource.objects.get(name='University Metered HPC'), value='metered-hpc') ResourceAttribute.objects.get_or_create(resource_attribute_type=ResourceAttributeType.objects.get( name='slurm_specs'), resource=Resource.objects.get(name='Chemistry-scavenger'), value='QOS+=scavenger:Fairshare=100') - ResourceAttribute.objects.get_or_create(resource_attribute_type=ResourceAttributeType.objects.get( - name='slurm_specs'), resource=Resource.objects.get(name='Physics-scavenger'), value='QOS+=scavenger:Fairshare=100') - ResourceAttribute.objects.get_or_create(resource_attribute_type=ResourceAttributeType.objects.get( - name='slurm_specs'), resource=Resource.objects.get(name='Industry-scavenger'), value='QOS+=scavenger:Fairshare=100') - ResourceAttribute.objects.get_or_create(resource_attribute_type=ResourceAttributeType.objects.get( - name='slurm_specs'), resource=Resource.objects.get(name='Chemistry-cgray'), value='QOS+=cgray:Fairshare=100') - ResourceAttribute.objects.get_or_create(resource_attribute_type=ResourceAttributeType.objects.get( - name='slurm_specs'), resource=Resource.objects.get(name='Physics-sfoster'), value='QOS+=sfoster:Fairshare=100') - ResourceAttribute.objects.get_or_create(resource_attribute_type=ResourceAttributeType.objects.get( - name='slurm_specs'), resource=Resource.objects.get(name='University Metered HPC'), + ResourceAttribute.objects.get_or_create( + resource_attribute_type=ResourceAttributeType.objects.get(name='slurm_specs'), + resource=Resource.objects.get(name='Physics-scavenger'), + value='QOS+=scavenger:Fairshare=100') + ResourceAttribute.objects.get_or_create( + resource_attribute_type=ResourceAttributeType.objects.get(name='slurm_specs'), + resource=Resource.objects.get(name='Industry-scavenger'), + value='QOS+=scavenger:Fairshare=100') + ResourceAttribute.objects.get_or_create( + resource_attribute_type=ResourceAttributeType.objects.get(name='slurm_specs'), + resource=Resource.objects.get(name='Chemistry-cgray'), + value='QOS+=cgray:Fairshare=100') + ResourceAttribute.objects.get_or_create( + resource_attribute_type=ResourceAttributeType.objects.get(name='slurm_specs'), + resource=Resource.objects.get(name='Physics-sfoster'), value='QOS+=sfoster:Fairshare=100') + ResourceAttribute.objects.get_or_create( + resource_attribute_type=ResourceAttributeType.objects.get(name='slurm_specs'), + resource=Resource.objects.get(name='University Metered HPC'), value='GrpTRESMins=cpu={cpumin}') #slurm_specs_attrib_list for University Metered HPC @@ -598,7 +612,7 @@ def handle(self, *args, **options): 'cpumin *= 60' ] ResourceAttribute.objects.get_or_create(resource_attribute_type=ResourceAttributeType.objects.get( - name='slurm_specs_attriblist'), resource=Resource.objects.get(name='University Metered HPC'), + name='slurm_specs_attriblist'), resource=Resource.objects.get(name='University Metered HPC'), value="\n".join(attriblist_list)) # call_command('loaddata', 'test_data.json') diff --git a/coldfront/core/utils/management/commands/show_users_in_project_but_not_in_allocation.py b/coldfront/core/utils/management/commands/show_users_in_project_but_not_in_allocation.py index 573a181f2..2524a0752 100644 --- a/coldfront/core/utils/management/commands/show_users_in_project_but_not_in_allocation.py +++ b/coldfront/core/utils/management/commands/show_users_in_project_but_not_in_allocation.py @@ -1,7 +1,4 @@ -import os - from django.conf import settings -from django.core.management import call_command from django.core.management.base import BaseCommand from coldfront.core.project.models import Project @@ -17,8 +14,8 @@ def handle(self, *args, **options): status__name='Active').values_list('user__username', flat=True)) users_in_allocation = [] for allocation in project.allocation_set.filter(status__name__in=('Active', - 'New', 'Paid', 'Payment Pending', - 'Payment Requested', 'Renewal Requested')): + 'New', 'Paid', 'Payment Pending', + 'Payment Requested', 'Renewal Requested')): users_in_allocation.extend(allocation.allocationuser_set.filter( status__name='Active').values_list('user__username', flat=True)) diff --git a/coldfront/plugins/slurm/management/commands/slurm_sync.py b/coldfront/plugins/slurm/management/commands/slurm_sync.py index 633e5d84f..b2c9dc28e 100644 --- a/coldfront/plugins/slurm/management/commands/slurm_sync.py +++ b/coldfront/plugins/slurm/management/commands/slurm_sync.py @@ -34,8 +34,9 @@ def _cluster_from_dump(self, cluster, file=None): else: with tempfile.TemporaryDirectory() as tmpdir: fname = os.path.join(tmpdir, 'cluster.cfg') + cluster_name = cluster.get_attribute("slurm_cluster") try: - slurm_dump_cluster(cluster, fname) + slurm_dump_cluster(cluster_name, fname) with open(fname) as fh: slurm_cluster = SlurmCluster.new_from_stream(fh) except SlurmError as e: @@ -46,7 +47,9 @@ def _cluster_from_dump(self, cluster, file=None): def handle(self, *args, **options): # make new SlurmCluster obj containing the dump from the cluster file = options['file'] - cluster_resources = Resource.objects.filter(resource_type__name='Cluster') + cluster_resources = Resource.objects.filter( + resource_type__name='Cluster', is_available=True + ) slurm_clusters = {r: self._cluster_from_dump(r, file=file) for r in cluster_resources} slurm_clusters = { r:c for r, c in slurm_clusters.items() if r.get_attribute('slurm_cluster') == c.name diff --git a/coldfront/plugins/slurm/utils.py b/coldfront/plugins/slurm/utils.py index b4f2ae39d..ee1befd25 100644 --- a/coldfront/plugins/slurm/utils.py +++ b/coldfront/plugins/slurm/utils.py @@ -162,7 +162,9 @@ def slurm_collect_fairshares(cluster=None, output_file=None): output_str = f' > {output_file}' if output_file else '' cmd = SLURM_CMD_PULL_FAIRSHARE + cluster_str + output_str - fairshare_data = _run_slurm_cmd(cmd) + fairshare_data = _run_slurm_cmd(cmd, noop=False) fairshare_data = fairshare_data.decode('utf-8').split('\n') + if "-----" not in fairshare_data[1]: + fairshare_data = fairshare_data[1:] fairshare_data = slurm_fixed_width_lines_to_dict(fairshare_data) return fairshare_data diff --git a/coldfront/plugins/xdmod/tasks.py b/coldfront/plugins/xdmod/tasks.py new file mode 100644 index 000000000..2be6df758 --- /dev/null +++ b/coldfront/plugins/xdmod/tasks.py @@ -0,0 +1,6 @@ +from django.core.management import call_command + +def xdmod_usage(): + """Add xdmod usage data + """ + call_command('xdmod_usage', sync=True) diff --git a/coldfront/plugins/xdmod/utils.py b/coldfront/plugins/xdmod/utils.py index 2f0a1d9c3..8f2785fe0 100644 --- a/coldfront/plugins/xdmod/utils.py +++ b/coldfront/plugins/xdmod/utils.py @@ -116,7 +116,6 @@ def xdmod_fetch(self, account, statistic, realm, group_by='total'): payload['group_by'] = self.group_by[group_by] payload['statistic'] = statistic payload['realm'] = realm - print("PAYLOAD:", payload) if group_by == 'total': core_hours = self.fetch_value(payload, search_item=account) elif group_by == 'per-user':