From ce9af231c906d76c5549b083a8ac0efeb47be0e8 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Tue, 27 Aug 2024 13:20:54 -0400 Subject: [PATCH 1/8] add boslfs02 and holylabs to add_resource_defaults --- .../commands/add_resource_defaults.py | 59 +++++++++++-------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/coldfront/core/resource/management/commands/add_resource_defaults.py b/coldfront/core/resource/management/commands/add_resource_defaults.py index 16e772014..ce87ca4ee 100644 --- a/coldfront/core/resource/management/commands/add_resource_defaults.py +++ b/coldfront/core/resource/management/commands/add_resource_defaults.py @@ -27,6 +27,7 @@ def handle(self, *args, **options): ('used_tb', 'Float'), ('file_count', 'Int'), ('allocated_tb', 'Float'), + ('ChangeableAllocations', 'Yes/No'), # ('Core Count', 'Int'), # ('expiry_time', 'Int'), # ('fee_applies', 'Yes/No'), @@ -75,33 +76,39 @@ def handle(self, *args, **options): default_value_type = ResourceAttributeType.objects.get(name='quantity_default_value') label_type = ResourceAttributeType.objects.get(name='quantity_label') - for name, desc, is_public, rtype, parent_name, default_value in ( - ('Tier 0', 'Bulk - Lustre', True, storage_tier, None, 1), - ('Tier 1', 'Enterprise - Isilon', True, storage_tier, None, 1), - ('Tier 2', 'CEPH storage', True, storage_tier, None, 1), - ('Tier 3', 'Attic Storage - Tape', True, storage_tier, None, 20), - ('holylfs04/tier0', 'Holyoke data center lustre storage', True, storage, 'Tier 0', 1), - ('holylfs05/tier0', 'Holyoke data center lustre storage', True, storage, 'Tier 0', 1), - ('nesetape/tier3', 'Cold storage for past projects', True, storage, 'Tier 3', 20), - ('holy-isilon/tier1', 'Tier1 storage with snapshots and disaster recovery copy', True, storage, 'Tier 1', 1), - ('bos-isilon/tier1', 'Tier1 storage for on-campus storage mounting', True, storage, 'Tier 1', 1), - ('holystore01/tier0', 'Luster storage under Tier0', True, storage, 'Tier 0', 1), - ('b-nfs02-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1), - ('b-nfs03-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1), - ('b-nfs04-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1), - ('b-nfs05-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1), - ('b-nfs06-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1), - ('b-nfs07-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1), - ('b-nfs08-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1), - ('b-nfs09-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1), - ('h-nfs16-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1), - ('h-nfs17-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1), - ('h-nfs18-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1), - ('h-nfs19-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1), + for name, desc, is_public, rtype, parent_name, default_value, reqspayment, is_allocatable in ( + ('Tier 0', 'Bulk - Lustre', True, storage_tier, None, 1, True, True), + ('Tier 1', 'Enterprise - Isilon', True, storage_tier, None, 1, True, True), + ('Tier 2', 'CEPH storage', True, storage_tier, None, 1, True, True), + ('Tier 3', 'Attic Storage - Tape', True, storage_tier, None, 20, True, True), + ('holylfs04/tier0', 'Holyoke data center lustre storage', True, storage, 'Tier 0', 1, True, True), + ('holylfs05/tier0', 'Holyoke data center lustre storage', True, storage, 'Tier 0', 1, True, True), + ('nesetape/tier3', 'Cold storage for past projects', True, storage, 'Tier 3', 20, True, True), + ('holy-isilon/tier1', 'Tier1 storage with snapshots and disaster recovery copy', True, storage, 'Tier 1', 1, True, True), + ('bos-isilon/tier1', 'Tier1 storage for on-campus storage mounting', True, storage, 'Tier 1', 1, True, True), + ('holystore01/tier0', 'Luster storage under Tier0', True, storage, 'Tier 0', 1, True, True), + ('b-nfs02-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('b-nfs03-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('b-nfs04-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('b-nfs05-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('b-nfs06-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('b-nfs07-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('b-nfs08-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('b-nfs09-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('h-nfs16-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('h-nfs17-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('h-nfs18-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('h-nfs19-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('boslfs02', 'complimentary lab storage', True, storage, None, 1, False, False), + ('holylabs', 'complimentary lab storage', True, storage, None, 1, False, False), ): resource_defaults = { - 'description':desc, 'is_public':is_public, 'resource_type':rtype, + 'description': desc, + 'is_public': is_public, + 'resource_type': rtype, + 'requires_payment': reqspayment, + 'is_allocatable': is_allocatable, } if parent_name: resource_defaults['parent_resource'] = Resource.objects.get(name=parent_name) @@ -113,6 +120,10 @@ def handle(self, *args, **options): resource_attribute_type=default_value_type, defaults={'value': default_value} ) + resource_obj.resourceattribute_set.update_or_create( + resource_attribute_type=default_value_type, + defaults={'value': default_value} + ) quantity_label = "TB" if default_value == 20: From c9b6b31a49a9580847f0d3a428767b62cee2ff2f Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Tue, 27 Aug 2024 17:05:55 -0400 Subject: [PATCH 2/8] change allocation requirespayment and is_changeable attr setting routines --- coldfront/core/allocation/views.py | 6 +++++ .../test_data/test_fixtures/ifx.json | 20 +++++++++++--- .../commands/import_add_allocations.py | 2 +- .../commands/import_allocationquotas.py | 8 ------ .../commands/id_import_new_allocations.py | 27 ++++++++++++------- coldfront/plugins/fasrc/utils.py | 6 ----- coldfront/plugins/fasrc_monitoring/views.py | 3 ++- 7 files changed, 43 insertions(+), 29 deletions(-) diff --git a/coldfront/core/allocation/views.py b/coldfront/core/allocation/views.py index f78a11608..4eccbcd5e 100644 --- a/coldfront/core/allocation/views.py +++ b/coldfront/core/allocation/views.py @@ -269,6 +269,7 @@ def post(self, request, *args, **kwargs): err = 'You do not have permission to update the allocation' messages.error(request, err) return HttpResponseRedirect(reverse('allocation-detail', kwargs={'pk': pk})) + initial_data = { 'status': allocation_obj.status, 'end_date': allocation_obj.end_date, @@ -356,6 +357,11 @@ def post(self, request, *args, **kwargs): allocation_obj.resources.add(resource) allocation_obj.status = AllocationStatusChoice.objects.get(name='Active') + AllocationAttribute.objects.get_or_create( + allocation=allocation_obj, + allocation_attribute_type=AllocationAttributeType.objects.get(name='RequiresPayment'), + defaults={'value': resource.requires_payment} + ) elif action == 'deny': allocation_obj.status = AllocationStatusChoice.objects.get(name='Denied') diff --git a/coldfront/core/test_helpers/test_data/test_fixtures/ifx.json b/coldfront/core/test_helpers/test_data/test_fixtures/ifx.json index 86b8501a7..8bbd187c7 100644 --- a/coldfront/core/test_helpers/test_data/test_fixtures/ifx.json +++ b/coldfront/core/test_helpers/test_data/test_fixtures/ifx.json @@ -82,7 +82,7 @@ "fields": { "product_name": "holylfs10/tier1", "product_number": "IFXNANANANANANANANA2", - "product_description": "testresource/tier1", + "product_description": "holylfs10/tier1", "billing_calculator": "coldfront.plugins.ifx.calculator.ColdfrontBillingCalculator", "reporting_group": "null", "facility_id": 1, @@ -93,9 +93,9 @@ "model": "ifxbilling.product", "pk": 2, "fields": { - "product_name": "testresource/tier1", + "product_name": "holylfs09/tier1", "product_number": "IFXNANANANANANANANA", - "product_description": "testresource/tier1", + "product_description": "holylfs09/tier1", "billing_calculator": "coldfront.plugins.ifx.calculator.ColdfrontBillingCalculator", "reporting_group": "null", "facility_id": 1, @@ -143,6 +143,20 @@ { "model": "ifxbilling.rate", "pk": 2, + "fields": { + "name": "Uni Rate", + "price": 100, + "units": "TB", + "is_active": true, + "product_id": 2, + "created": "2022-02-22T16:26:49.635Z", + "updated": "2022-02-22T16:26:49.635Z", + "max_qty": null + } + }, + { + "model": "ifxbilling.rate", + "pk": 3, "fields": { "name": "Uni Rate", "price": 100, diff --git a/coldfront/core/utils/management/commands/import_add_allocations.py b/coldfront/core/utils/management/commands/import_add_allocations.py index 1d6631746..b4be48079 100644 --- a/coldfront/core/utils/management/commands/import_add_allocations.py +++ b/coldfront/core/utils/management/commands/import_add_allocations.py @@ -72,7 +72,7 @@ def handle(self, *args, **options): defaults={ 'status': AllocationStatusChoice.objects.get(name='Active'), 'start_date': datetime.datetime.now(), - 'is_changeable': True, + 'is_changeable': resource.is_allocatable, 'justification': f'Allocation Information for {lab_name}', } ) diff --git a/coldfront/core/utils/management/commands/import_allocationquotas.py b/coldfront/core/utils/management/commands/import_allocationquotas.py index 02a60341c..8c45e8b10 100644 --- a/coldfront/core/utils/management/commands/import_allocationquotas.py +++ b/coldfront/core/utils/management/commands/import_allocationquotas.py @@ -93,14 +93,6 @@ def handle(self, *args, **options): allocation_attribute_obj.allocationattributeusage.value = lab_usage allocation_attribute_obj.allocationattributeusage.save() - allocation_attribute_type_payment = AllocationAttributeType.objects.get( - name='RequiresPayment') - allocation_attribute_payment, _ = AllocationAttribute.objects.get_or_create( - allocation_attribute_type=allocation_attribute_type_payment, - allocation=allocation, - defaults={'value': True} - ) - allocation_attribute_payment.save() except Project.DoesNotExist: print("Project not found: " + lab_name) tocsv = [lab_name,resource_name,"Project"] diff --git a/coldfront/plugins/fasrc/management/commands/id_import_new_allocations.py b/coldfront/plugins/fasrc/management/commands/id_import_new_allocations.py index e261ddfdf..d88320088 100644 --- a/coldfront/plugins/fasrc/management/commands/id_import_new_allocations.py +++ b/coldfront/plugins/fasrc/management/commands/id_import_new_allocations.py @@ -21,11 +21,7 @@ ) from coldfront.core.utils.fasrc import update_csv, select_one_project_allocation, save_json from coldfront.core.resource.models import Resource -if ENV.bool('PLUGIN_SFTOCF', default=False): - from coldfront.plugins.sftocf.utils import ( - StarFishRedash, - RedashDataPipeline, - ) +from coldfront.plugins.sftocf.utils import StarFishRedash, RedashDataPipeline from coldfront.plugins.fasrc.utils import ( AllTheThingsConn, match_entries_with_projects, @@ -60,11 +56,18 @@ def handle(self, *args, **options): redash_api = StarFishRedash() allocation_usages = redash_api.return_query_results(query='subdirectory') + + # make item calls subdir_type = AllocationAttributeType.objects.get(name='Subdirectory') + projectstatuschoice_active = ProjectStatusChoice.objects.get(name='Active') + allocationstatuschoice_active = AllocationStatusChoice.objects.get(name='Active') + allocationuserstatuschoice_active = AllocationUserStatusChoice.objects.get(name='Active') + allocationattrtype_payment = AllocationAttributeType.objects.get( + name='RequiresPayment') for project in proj_models: if project.status.name == 'New': - project.status = ProjectStatusChoice.objects.get(name='Active') + project.status = projectstatuschoice_active project.save() for lab, allocations in result_cleaned.items(): @@ -90,9 +93,9 @@ def handle(self, *args, **options): resources__name=resource, allocationattribute__value=lab_path, defaults={ - 'status': AllocationStatusChoice.objects.get(name='Active'), + 'status': allocationstatuschoice_active, 'start_date': datetime.now(), - 'is_changeable': True, + 'is_changeable': resource.is_allocatable, 'justification': f'Allocation Information for {lab_name}', } ) @@ -105,6 +108,11 @@ def handle(self, *args, **options): allocation_attribute_type_id=subdir_type.pk, value=lab_path ) + AllocationAttribute.objects.create( + allocation=allocation, + allocation_attribute_type_id=allocationattrtype_payment.pk, + value=resource.requires_payment + ) print(f'allocation created: {allocation_str}') logger.info('allocation created: %s', allocation_str) allocation.save() @@ -124,8 +132,7 @@ def handle(self, *args, **options): _, created = AllocationUser.objects.get_or_create( allocation=allocation, user=pi_obj, - defaults={ - 'status': AllocationUserStatusChoice.objects.get(name='Active')} + defaults={'status': allocationuserstatuschoice_active} ) except ValidationError: logger.warning('adding PI %s to allocation %s failed', diff --git a/coldfront/plugins/fasrc/utils.py b/coldfront/plugins/fasrc/utils.py index b7e07185a..7cb9acba7 100644 --- a/coldfront/plugins/fasrc/utils.py +++ b/coldfront/plugins/fasrc/utils.py @@ -286,7 +286,6 @@ def push_quota_data(result_file): # collect commonly used database objects here proj_models = proj_models.prefetch_related('allocation_set') allocation_attribute_types = AllocationAttributeType.objects.all() - allocation_attribute_type_payment = allocation_attribute_types.get(name='RequiresPayment') for lab, quota_dicts in result_json_cleaned.items(): logger.info('PROJECT: %s ====================================', lab) @@ -319,11 +318,6 @@ def push_quota_data(result_file): alloc_attr_obj.allocationattributeusage.value = v[1] alloc_attr_obj.allocationattributeusage.save() - # 5. AllocationAttribute - allocation.allocationattribute_set.update_or_create( - allocation_attribute_type=allocation_attribute_type_payment, - defaults={'value': True} - ) counts['complete'] += 1 except Exception as exc: allocation_name = f"{data_dict['lab']}/{data_dict['server']}" diff --git a/coldfront/plugins/fasrc_monitoring/views.py b/coldfront/plugins/fasrc_monitoring/views.py index b2ac85bef..46e8fca72 100644 --- a/coldfront/plugins/fasrc_monitoring/views.py +++ b/coldfront/plugins/fasrc_monitoring/views.py @@ -56,7 +56,8 @@ def get_context_data(self, **kwargs): projects = Project.objects.all() pi_not_projectuser = [p for p in projects if p.pi_id not in p.projectuser_set.values_list('user_id', flat=True)] allocation_not_changeable = Allocation.objects.filter( - status__name__in=PENDING_ACTIVE_ALLOCATION_STATUSES, is_changeable=False + status__name__in=PENDING_ACTIVE_ALLOCATION_STATUSES, is_changeable=False, + resources__is_allocatable=True ) multiple_allocation_resources = Allocation.objects.annotate( num_vols=Count('resources') From b70bded12fcfac9f5a6d6176a7b3f21dea38c1d4 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Wed, 28 Aug 2024 11:37:08 -0400 Subject: [PATCH 3/8] add allocationdetail creation test --- coldfront/core/allocation/tests/test_views.py | 39 ++++++++++++++++++- coldfront/core/test_helpers/factories.py | 2 + 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/coldfront/core/allocation/tests/test_views.py b/coldfront/core/allocation/tests/test_views.py index 72dee0aa6..a669a39bf 100644 --- a/coldfront/core/allocation/tests/test_views.py +++ b/coldfront/core/allocation/tests/test_views.py @@ -9,8 +9,11 @@ Allocation, AllocationUserNote, AllocationAttribute, + AllocationAttributeType, + AllocationStatusChoice, AllocationChangeRequest, ) +from coldfront.core.resource.models import Resource from coldfront.core.test_helpers.factories import ( setup_models, UserFactory, @@ -74,7 +77,7 @@ def setUpTestData(cls): AllocationFactory() for i in list(range(50)) ] for allocation in cls.additional_allocations: - allocation.resources.add(ResourceFactory(name='holylfs09/tier1', id=2)) + allocation.resources.add(Resource.objects.get(name='holylfs09/tier1')) cls.nonproj_nonallocation_user = UserFactory(username='rdrake') def test_allocation_list_access_admin(self): @@ -351,6 +354,40 @@ def test_allocationuser_button_visibility(self): ) +class AllocationDetailViewPostTest(AllocationViewBaseTest): + def setUp(self): + self.new_allocation = AllocationFactory( + project=self.project, quantity=20, justification='test new allocation', + status=AllocationStatusChoice.objects.get(name='New') + ) + self.new_allocation.resources.add(Resource.objects.get(name='holylfs09/tier1')) + quotatb = AllocationAttributeType.objects.get(name='Storage Quota (TB)') + self.new_allocation.allocationattribute_set.create( + value=20, allocation_attribute_type=quotatb + ) + self.url = f'/allocation/{self.new_allocation.pk}/' + + def test_allocationdetail_approval_post(self): + """test approval of new allocation""" + self.client.force_login(self.admin_user) + form_data = { + 'status': AllocationStatusChoice.objects.get(name='Active').pk, + #'start_date': self.new_allocation.start_date, + #'end_date': self.new_allocation.end_date, + 'description': "description test", + 'is_locked': False, + 'is_changeable': True, + 'resource': self.new_allocation.resources.first().pk, + 'action': 'approve', + 'auto_create_opts': '1', + } + response = self.client.post(self.url, data=form_data, follow=True) + # confirm that messages in response contains "Allocation Activated" + self.assertContains(response, 'Allocation Activated') + self.new_allocation.refresh_from_db() + self.assertEqual(self.new_allocation.status.name, 'Active') + + class AllocationCreateViewTest(AllocationViewBaseTest): """Tests for the AllocationCreateView""" diff --git a/coldfront/core/test_helpers/factories.py b/coldfront/core/test_helpers/factories.py index 2c892b131..a02ab4d2b 100644 --- a/coldfront/core/test_helpers/factories.py +++ b/coldfront/core/test_helpers/factories.py @@ -348,6 +348,7 @@ def setup_models(test_case): ResourceTypeFactory(name=resource_type) for resource, r_id, rtype in [ ('holylfs10/tier1', 1, 'Storage'), + ('holylfs09/tier1', 2, 'Storage'), ('Test Cluster', 3, 'Cluster') ]: ResourceFactory(name=resource, id=r_id, resource_type__name=rtype) @@ -361,6 +362,7 @@ def setup_models(test_case): ('High Security', 'Yes/No', False, False), ('DUA', 'Yes/No', False, False), ('External Sharing', 'Yes/No', False, False), + ('RequiresPayment', 'Yes/No', False, False), ): AllocationAttributeTypeFactory( name=name, From 36435db758b96a261e2b95c0142aef93e7526f3a Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Wed, 28 Aug 2024 12:29:26 -0400 Subject: [PATCH 4/8] only show cost information for allocations that require payment --- coldfront/core/allocation/models.py | 6 +++++ .../allocation/allocation_detail.html | 24 +++++++++++-------- coldfront/core/allocation/views.py | 5 ++++ coldfront/core/department/views.py | 5 +++- .../templates/project/project_detail.html | 2 +- coldfront/core/project/views.py | 2 +- 6 files changed, 31 insertions(+), 13 deletions(-) diff --git a/coldfront/core/allocation/models.py b/coldfront/core/allocation/models.py index ee7b45737..db91e7140 100644 --- a/coldfront/core/allocation/models.py +++ b/coldfront/core/allocation/models.py @@ -138,6 +138,12 @@ def save(self, *args, **kwargs): def offer_letter_code(self): return self.get_attribute('Offer Letter Code') + @property + def requires_payment(self): + requires_payment = self.get_attribute('Requires Payment') + if requires_payment == None: + return self.get_parent_resource.requires_payment + @property def fairshare(self): return self.get_attribute('FairShare') diff --git a/coldfront/core/allocation/templates/allocation/allocation_detail.html b/coldfront/core/allocation/templates/allocation/allocation_detail.html index b79348917..f8880b2b4 100644 --- a/coldfront/core/allocation/templates/allocation/allocation_detail.html +++ b/coldfront/core/allocation/templates/allocation/allocation_detail.html @@ -129,7 +129,7 @@

Allocation Information

{% endif %} - {% if "Storage" in allocation.get_parent_resource.resource_type.name %} + {% if invoice %} FIINE Expense Codes: @@ -164,13 +164,15 @@

Allocation Information

Quota (TB): {{ allocation.size|floatformat:2 }} - - Total Amount Due: - {% cost_tb allocation.size as cost %} - {% if cost %} - {{ cost }} - {% endif %} - + {% if invoice %} + + Total Amount Due: + {% cost_tb allocation.size as cost %} + {% if cost %} + {{ cost }} + {% endif %} + + {% endif %} {% endif %} Start Date: @@ -489,7 +491,9 @@

Users in Al {% if "Storage" in allocation.get_parent_resource.resource_type.name %} Logical Usage Percent Usage - Cost Per User (TB/month) + {% if invoice %} + Cost Per User (TB/month) + {% endif %} {% else %} CPU Hours Percent Usage @@ -546,7 +550,7 @@

Users in Al {{userusage|div:allocationusage|mul:100|floatformat:2 }}% {% endif %} - {% if "Storage" in allocation.get_parent_resource.resource_type.name %} + {% if invoice %} {% cost_bytes userusage as cost %} {% if cost %} {{ cost }} diff --git a/coldfront/core/allocation/views.py b/coldfront/core/allocation/views.py index 4eccbcd5e..21ca8f951 100644 --- a/coldfront/core/allocation/views.py +++ b/coldfront/core/allocation/views.py @@ -206,6 +206,11 @@ def get_context_data(self, **kwargs): context['expense_codes'] = expense_codes offer_letter_code_type = AllocationAttributeType.objects.get(name="Expense Code") + context['invoice'] = ( + allocation_obj.requires_payment + and allocation_obj.status.name in ACTIVE_ALLOCATION_STATUSES + and "Storage" in allocation_obj.get_parent_resource.resource_type.name + ) context['expense_code'] = allocation_obj.allocationattribute_set.filter( allocation_attribute_type=offer_letter_code_type ) diff --git a/coldfront/core/department/views.py b/coldfront/core/department/views.py index c6dd4b2ee..841fed795 100644 --- a/coldfront/core/department/views.py +++ b/coldfront/core/department/views.py @@ -203,7 +203,10 @@ def get_context_data(self, **kwargs): status__name='Active', ) p.storage_allocs = p.allocs.filter( - resources__resource_type__name='Storage') + resources__resource_type__name='Storage', + allocationattribute__allocation_attribute_type__name='RequiresPayment', + allocationattribute__value='True' + ) p.compute_allocs = p.allocs.filter( resources__resource_type__name='Cluster') storage_pi_dict[p.pi].extend(list(p.storage_allocs)) diff --git a/coldfront/core/project/templates/project/project_detail.html b/coldfront/core/project/templates/project/project_detail.html index ac803fe95..081418c3d 100644 --- a/coldfront/core/project/templates/project/project_detail.html +++ b/coldfront/core/project/templates/project/project_detail.html @@ -121,7 +121,7 @@

Storage   {{ allocation.allocationuser_set.count }} {{ allocation.size|floatformat:1 }} {{ allocation.get_parent_resource.unit }} {{ allocation.usage|floatformat:1 }} - ${{allocation.cost|floatformat:2 }} + {% if allocation_obj.requires_payment %}${{allocation.cost|floatformat:2 }}{% endif %} diff --git a/coldfront/core/project/views.py b/coldfront/core/project/views.py index 42c51bbd2..c95b996cd 100644 --- a/coldfront/core/project/views.py +++ b/coldfront/core/project/views.py @@ -206,7 +206,7 @@ def get_context_data(self, **kwargs): compute_allocations = allocations.filter(resources__resource_type__name='Cluster') allocation_total = {'allocation_user_count': 0, 'size': 0, 'cost': 0, 'usage':0} for allocation in storage_allocations: - if allocation.cost: + if allocation.cost and allocation.requires_payment: allocation_total['cost'] += allocation.cost if allocation.size: allocation_total['size'] += allocation.size From 580f4c5fd08914f02454e87cd37ff4b2bcc9b36a Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Wed, 28 Aug 2024 13:17:06 -0400 Subject: [PATCH 5/8] update ATT pipeline and allocation request approval form to accommodate holylabs and boslfs02 --- coldfront/core/allocation/forms.py | 3 +- .../commands/add_resource_defaults.py | 33 +++++++++++-------- coldfront/plugins/fasrc/utils.py | 6 ++-- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/coldfront/core/allocation/forms.py b/coldfront/core/allocation/forms.py index e8de4b9c0..fc3b63a5a 100644 --- a/coldfront/core/allocation/forms.py +++ b/coldfront/core/allocation/forms.py @@ -246,7 +246,8 @@ def __init__(self, *args, **kwargs): else: if allo_resource.resource_type.name == 'Storage Tier': self.fields['resource'].queryset = Resource.objects.filter( - parent_resource=allo_resource + parent_resource=allo_resource, + is_allocatable=True ) else: self.fields['resource'].required = False diff --git a/coldfront/core/resource/management/commands/add_resource_defaults.py b/coldfront/core/resource/management/commands/add_resource_defaults.py index ce87ca4ee..ee0cf09af 100644 --- a/coldfront/core/resource/management/commands/add_resource_defaults.py +++ b/coldfront/core/resource/management/commands/add_resource_defaults.py @@ -87,20 +87,25 @@ def handle(self, *args, **options): ('holy-isilon/tier1', 'Tier1 storage with snapshots and disaster recovery copy', True, storage, 'Tier 1', 1, True, True), ('bos-isilon/tier1', 'Tier1 storage for on-campus storage mounting', True, storage, 'Tier 1', 1, True, True), ('holystore01/tier0', 'Luster storage under Tier0', True, storage, 'Tier 0', 1, True, True), - ('b-nfs02-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), - ('b-nfs03-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), - ('b-nfs04-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), - ('b-nfs05-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), - ('b-nfs06-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), - ('b-nfs07-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), - ('b-nfs08-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), - ('b-nfs09-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), - ('h-nfs16-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), - ('h-nfs17-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), - ('h-nfs18-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), - ('h-nfs19-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), - ('boslfs02', 'complimentary lab storage', True, storage, None, 1, False, False), - ('holylabs', 'complimentary lab storage', True, storage, None, 1, False, False), + ('b-nfs02-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), + ('b-nfs03-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), + ('b-nfs04-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), + ('b-nfs05-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), + ('b-nfs06-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), + ('b-nfs07-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), + ('b-nfs08-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), + ('b-nfs09-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), + ('h-nfs11-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), + ('h-nfs12-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), + ('h-nfs13-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), + ('h-nfs14-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), + ('h-nfs15-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), + ('h-nfs16-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), + ('h-nfs17-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), + ('h-nfs18-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), + ('h-nfs19-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), + ('boslfs02', 'complimentary lab storage', True, storage, 'Tier 0', 1, False, False), + ('holylabs', 'complimentary lab storage', True, storage, 'Tier 0', 1, False, False), ): resource_defaults = { diff --git a/coldfront/plugins/fasrc/utils.py b/coldfront/plugins/fasrc/utils.py index 7cb9acba7..f25e82f47 100644 --- a/coldfront/plugins/fasrc/utils.py +++ b/coldfront/plugins/fasrc/utils.py @@ -28,7 +28,7 @@ def produce_query_statement(self, vol_type, volumes=None): query_dict = { 'quota': { - 'volumes': '|'.join(r.name.split('/')[0] for r in Resource.objects.filter(name__contains='tier0')), + 'volumes': '|'.join(r.name.split('/')[0] for r in Resource.objects.filter(parent_resource__name='Tier 0')), 'relation': 'HasQuota', 'match': "(e:Quota) MATCH (d:ConfigValue {Name: 'Quota.Invocation'})", 'server': 'filesystem', @@ -47,7 +47,7 @@ def produce_query_statement(self, vol_type, volumes=None): 'unique':'datetime(e.DotsLFSUpdateDate) as begin_date' }, 'isilon': { - 'volumes': '|'.join(r.name.split('/')[0] for r in Resource.objects.filter(name__contains='tier1')), + 'volumes': '|'.join(r.name.split('/')[0] for r in Resource.objects.filter(parent_resource__name='Tier 1')), 'relation': 'Owns', 'match': "(e:IsilonPath) MATCH (d:ConfigValue {Name: 'IsilonPath.Invocation'})", 'server': 'Isilon', @@ -67,7 +67,7 @@ def produce_query_statement(self, vol_type, volumes=None): 'unique': 'datetime(e.DotsUpdateDate) as begin_date' }, 'volume': { - 'volumes': '|'.join(r.name.split('/')[0] for r in Resource.objects.filter(name__contains='tier2')), + 'volumes': '|'.join(r.name.split('/')[0] for r in Resource.objects.filter(parent_resource__name='Tier 2')), 'relation': 'Owns', 'match': '(e:Volume)', 'server': 'Hostname', From 90e5dd9d6d953a6244df4525c4437bfdd32cae10 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Wed, 28 Aug 2024 13:47:51 -0400 Subject: [PATCH 6/8] view and template fixes --- coldfront/core/allocation/models.py | 2 ++ .../templates/allocation/allocation_detail.html | 13 +++++++------ coldfront/core/resource/admin.py | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/coldfront/core/allocation/models.py b/coldfront/core/allocation/models.py index db91e7140..157bc89f4 100644 --- a/coldfront/core/allocation/models.py +++ b/coldfront/core/allocation/models.py @@ -276,6 +276,8 @@ def cost(self): return None except TypeError: return None + except ObjectDoesNotExist: + return None size_attr_name = self._return_size_attr_name() if not size_attr_name: return None diff --git a/coldfront/core/allocation/templates/allocation/allocation_detail.html b/coldfront/core/allocation/templates/allocation/allocation_detail.html index f8880b2b4..b1e02e144 100644 --- a/coldfront/core/allocation/templates/allocation/allocation_detail.html +++ b/coldfront/core/allocation/templates/allocation/allocation_detail.html @@ -78,7 +78,7 @@

Allocation Information

- Resource{{ allocation.resources.all|pluralize }} in allocation: + Allocated Resource{{ allocation.resources.all|pluralize }}: {% if allocation.get_resources_as_list %} {% for resource in allocation.get_resources_as_list %} @@ -140,7 +140,7 @@

Allocation Information

{% endif %} - Total Users in Your Bill: + Total Users with Usage: {{ allocation_users.count }} @@ -555,10 +555,11 @@

Users in Al {% if cost %} {{ cost }} {% endif %} - {% else %} - {{ user.effectvusage }} - {{ user.normshares }} - {{ user.fairshare }} + {% endif %} + {% if 'Cluster' in allocation.get_parent_resource.resource_type.name %} + {{ user.effectvusage }} + {{ user.normshares }} + {{ user.fairshare }} {% endif %} {% endfor %} diff --git a/coldfront/core/resource/admin.py b/coldfront/core/resource/admin.py index 11a1f3cf8..6af1ccefa 100644 --- a/coldfront/core/resource/admin.py +++ b/coldfront/core/resource/admin.py @@ -47,7 +47,7 @@ class ResourceAdmin(SimpleHistoryAdmin): # readonly_fields_change = ('resource_type', ) fields_change = ('resource_type', 'parent_resource', 'is_allocatable', 'name', 'description', 'is_available', 'is_public', 'requires_payment', 'allowed_groups', 'allowed_users', 'linked_resources') - list_display = ('pk', 'name', 'description', 'parent_resource', 'is_allocatable', 'resource_type_name', + list_display = ('pk', 'name', 'description', 'parent_resource', 'is_allocatable', 'requires_payment', 'resource_type_name', 'is_available', 'is_public', 'created', 'modified', ) search_fields = ('name', 'description', 'resource_type__name') list_filter = ('resource_type__name', 'is_allocatable', 'is_available', 'is_public', 'requires_payment' ) From 53683b5a61797b47096ca603df883f18c8df2023 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Wed, 28 Aug 2024 13:53:58 -0400 Subject: [PATCH 7/8] add CPU Hours label to department compute usage table --- .../core/department/templates/department/department_detail.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coldfront/core/department/templates/department/department_detail.html b/coldfront/core/department/templates/department/department_detail.html index 7b3452d62..52b5b9c83 100644 --- a/coldfront/core/department/templates/department/department_detail.html +++ b/coldfront/core/department/templates/department/department_detail.html @@ -141,7 +141,7 @@

Quarterly Cl Project Allocation Users - Usage + Usage (CPU Hours) From a0a7223437ff4d6205c33eb45209f292cccb8e50 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Thu, 29 Aug 2024 10:22:12 -0400 Subject: [PATCH 8/8] update att query to remove false duplicates --- coldfront/plugins/fasrc/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/coldfront/plugins/fasrc/utils.py b/coldfront/plugins/fasrc/utils.py index f25e82f47..650c0e8a9 100644 --- a/coldfront/plugins/fasrc/utils.py +++ b/coldfront/plugins/fasrc/utils.py @@ -88,7 +88,8 @@ def produce_query_statement(self, vol_type, volumes=None): statement = { 'statement': f"MATCH p=(g:Group)-[r:{d['relation']}]-{d['match']}\ - WHERE (e.{d['server']} =~ '.*({d['volumes']}).*') AND {d['validation_query']}\ + WHERE g.ATTStaleData = false AND (e.{d['server']} =~ '.*({d['volumes']}).*')\ + AND {d['validation_query']}\ AND NOT (g.ADSamAccountName =~ '.*(disabled|rc_admin).*')\ RETURN\ {d['unique']},\