Merge branch 'cp_slurm' of https://github.com/fasrc/coldfront into cp_slurm

Author: claire-peters

coldfront/config/email.py

@@ -22,3 +22,4 @@
 EMAIL_ALLOCATION_EXPIRING_NOTIFICATION_DAYS = ENV.list('EMAIL_ALLOCATION_EXPIRING_NOTIFICATION_DAYS', cast=int, default=[7, 14, 30])
 EMAIL_SIGNATURE = ENV.str('EMAIL_SIGNATURE', default='', multiline=True)
 EMAIL_ADMINS_ON_ALLOCATION_EXPIRE = ENV.bool('EMAIL_ADMINS_ON_ALLOCATION_EXPIRE', default=False)
+ADMIN_REMINDER_EMAIL = ENV.str('ADMIN_REMINDER_EMAIL', default='')

coldfront/core/allocation/forms.py

@@ -150,18 +150,7 @@ def clean(self):
                 [d[:3], d[3:8], d[8:12], d[12:18], d[18:24], d[24:28], d[28:33]]
             )
             cleaned_expensecode = insert_dashes(replace_productcode(digits_only(expense_code)))
-            if 'ifxbilling' in settings.INSTALLED_APPS:
-                try:
-                    matched_fiineaccts = FiineAPI.listAccounts(code=cleaned_expensecode)
-                    if not matched_fiineaccts:
-                        self.add_error(
-                            "expense_code",
-                            "expense code not found in system - please check the code or get in touch with a system administrator."
-                        )
-                except Exception:
-                    #Not authorized to use accounts_list
-                    pass
-                cleaned_data['expense_code'] = cleaned_expensecode
+            cleaned_data['expense_code'] = cleaned_expensecode
         elif existing_expense_codes and existing_expense_codes != '------':
             cleaned_data['expense_code'] = existing_expense_codes
         return cleaned_data

coldfront/core/allocation/models.py

@@ -598,20 +598,29 @@ def clean(self):
 
         expected_value_type = self.allocation_attribute_type.attribute_type.name.strip()
         error = None
-        if expected_value_type == 'Float' and not isinstance(literal_eval(self.value), (float,int)):
-            error = 'Value must be a float.'
-        elif expected_value_type == 'Int' and not isinstance(literal_eval(self.value), int):
-            error = 'Value must be an integer.'
-        elif expected_value_type == 'Yes/No' and self.value not in ['Yes', 'No']:
+        if expected_value_type in ['Float', 'Int']:
+            try:
+                literal_val = literal_eval(self.value)
+            except SyntaxError as exc:
+                error = 'Value must be entirely numeric. Please remove any non-numeric characters.'
+                raise ValidationError(
+                    f'Invalid Value "{self.value}" for "{self.allocation_attribute_type.name}". {error}'
+                ) from exc
+            if expected_value_type == 'Float' and not isinstance(literal_val, (float,int)):
+                error = 'Value must be a float.'
+            elif expected_value_type == 'Int' and not isinstance(literal_val, int):
+                error = 'Value must be an integer.'
+        elif expected_value_type == "Yes/No" and self.value not in ["Yes", "No"]:
             error = 'Allowed inputs are "Yes" or "No".'
-        elif expected_value_type == 'Date':
+        elif expected_value_type == "Date":
             try:
                 datetime.datetime.strptime(self.value.strip(), '%Y-%m-%d')
             except ValueError:
                 error = 'Date must be in format YYYY-MM-DD'
         if error:
             raise ValidationError(
-                'Invalid Value "%s" for "%s". %s' % (self.value, self.allocation_attribute_type.name, error))
+                f'Invalid Value "{self.value}" for "{self.allocation_attribute_type.name}". {error}'
+            )
 
     def __str__(self):
         return str(self.allocation_attribute_type.name)

coldfront/core/allocation/tasks.py

@@ -31,6 +31,7 @@
 
 EMAIL_ADMINS_ON_ALLOCATION_EXPIRE = import_from_settings('EMAIL_ADMINS_ON_ALLOCATION_EXPIRE')
 EMAIL_ADMIN_LIST = import_from_settings('EMAIL_ADMIN_LIST')
+ADMIN_REMINDER_EMAIL = import_from_settings('ADMIN_REMINDER_EMAIL')
 
 def update_statuses():
 
@@ -69,12 +70,14 @@ def send_request_reminder_emails():
             'signature': EMAIL_SIGNATURE,
             'url_base': f'{CENTER_BASE_URL.strip("/")}/allocation/change-request/'
         }
-        send_admin_email_template(
-            'Pending Allocation Changes',
-            'email/pending_allocation_changes.txt',
-            allocation_change_template_context,
-        )
 
+        send_email_template(
+            subject='Pending Allocation Changes',
+            template_name='email/pending_allocation_changes.txt',
+            template_context=allocation_change_template_context,
+            sender=EMAIL_SENDER,
+            receiver_list=[ADMIN_REMINDER_EMAIL,],
+        )
     # Allocation Requests are allocations marked as "new"
     pending_allocations = Allocation.objects.filter(
         status__name = 'New', created__lte=req_alert_date
@@ -87,11 +90,15 @@ def send_request_reminder_emails():
             'signature': EMAIL_SIGNATURE,
             'url_base': f'{CENTER_BASE_URL.strip("/")}/allocation/'
         }
-        send_admin_email_template(
-            'Pending Allocations',
-            'email/pending_allocations.txt',
-            new_allocation_template_context,
+
+        send_email_template(
+            subject='Pending Allocations',
+            template_name='email/pending_allocations.txt',
+            template_context=new_allocation_template_context,
+            sender=EMAIL_SENDER,
+            receiver_list=[ADMIN_REMINDER_EMAIL,],
         )
+
     # return statement for testing
     return (pending_changerequests, pending_allocations)
 

coldfront/core/allocation/templates/allocation/allocation_detail.html

@@ -106,11 +106,11 @@ <h3><i class="fas fa-list" aria-hidden="true"></i> Allocation Information</h3>
               <th scope="row" class="text-nowrap">Service Period:</th>
               <td>1 Month</td>
           </tr>
-          {% if offer_letter_code %}
+          {% if expense_code %}
             <tr>
                 <th scope="row" class="text-nowrap">Requested Expense Code:</th>
                 <td>
-                  {% for code in offer_letter_code %}
+                  {% for code in expense_code %}
                     {{ code.value }}<br>
                   {% endfor %}
                 </td>

coldfront/core/allocation/tests/test_models.py

@@ -1,20 +1,22 @@
 """Unit tests for the allocation models"""
 
 from django.test import TestCase
+from django.core.exceptions import ValidationError
 
 from coldfront.core.test_helpers.factories import setup_models, AllocationFactory
 
 UTIL_FIXTURES = [
         "coldfront/core/test_helpers/test_data/test_fixtures/ifx.json",
 ]
 
+
 class AllocationModelTests(TestCase):
     """tests for Allocation model"""
     fixtures = UTIL_FIXTURES
 
     @classmethod
     def setUpTestData(cls):
-        """Set up project to test model properties and methods"""
+        """Set up allocation to test model properties and methods"""
         setup_models(cls)
 
     def test_allocation_str(self):
@@ -23,9 +25,9 @@ def test_allocation_str(self):
             self.proj_allocation.get_parent_resource.name,
             self.proj_allocation.project.pi
         )
-
         self.assertEqual(str(self.proj_allocation), allocation_str)
 
+
     def test_allocation_usage_property(self):
         """Test that allocation usage property displays correctly"""
         self.assertEqual(self.proj_allocation.usage, 10)
@@ -34,3 +36,27 @@ def test_allocation_usage_property_na(self):
         """Create allocation with no usage. Usage property should return None"""
         allocation = AllocationFactory()
         self.assertIsNone(allocation.usage)
+
+class AllocationAttributeModelTests(TestCase):
+    """Tests for allocationattribute models"""
+    fixtures = UTIL_FIXTURES
+
+    @classmethod
+    def setUpTestData(cls):
+        """Set up allocationattribute to test model properties and methods"""
+        setup_models(cls)
+        cls.allocationattribute = cls.proj_allocation.allocationattribute_set.get(
+            allocation_attribute_type__name='Storage Quota (TB)'
+        )
+
+    def test_allocationattribute_clean_no_error(self):
+        """cleaning a numeric value for an int or float AllocationAttributeType produces no error"""
+        self.allocationattribute.value = "1000"
+        self.allocationattribute.clean()
+
+    def test_allocationattribute_clean_nonnumeric_error(self):
+        """cleaning a non-numeric value for int or float AllocationAttributeTypes returns an informative error message"""
+
+        self.allocationattribute.value = "1000TB"
+        with self.assertRaisesMessage(ValidationError, 'Value must be entirely numeric. Please remove any non-numeric characters.'):
+            self.allocationattribute.clean()

coldfront/core/allocation/views.py

@@ -68,6 +68,7 @@
 
 
 if 'ifxbilling' in settings.INSTALLED_APPS:
+    from fiine.client import API as FiineAPI
     from ifxbilling.models import Account, UserProductAccount
 if 'django_q' in settings.INSTALLED_APPS:
     from django_q.tasks import Task
@@ -560,6 +561,10 @@ def form_valid(self, form):
         quantity = form_data.get('quantity', 1)
         allocation_account = form_data.get('allocation_account', None)
 
+        if resource_obj.name == "Tier 3" and quantity % 20 != 0:
+            form.add_error("quantity", format_html("Tier 3 quantity must be a multiple of 20."))
+            return self.form_invalid(form)
+
         # A resource is selected that requires an account name selection but user has no account names
         if (
             ALLOCATION_ACCOUNT_ENABLED
@@ -659,7 +664,19 @@ def form_valid(self, form):
             'quantity':quantity,
             'nese': nese,
             'used_percentage': used_percentage,
+            'expense_code': expense_code,
+            'unmatched_code': False,
         }
+
+        if 'ifxbilling' in settings.INSTALLED_APPS:
+            try:
+                matched_fiineaccts = FiineAPI.listAccounts(code=expense_code)
+                if not matched_fiineaccts:
+                    other_vars['unmatched_code'] = True
+            except Exception:
+                #Not authorized to use accounts_list
+                pass
+
         send_allocation_admin_email(
             allocation_obj,
             'New Allocation Request',
@@ -1800,7 +1817,6 @@ def post(self, request, *args, **kwargs):
                 if new_value != attribute_change.new_value:
                     attribute_change.new_value = new_value
                     attribute_change.save()
-
         if action == 'update':
             message = 'Allocation change request updated!'
         if action == 'approve':
@@ -1986,11 +2002,21 @@ def post(self, request, *args, **kwargs):
 
         if form_data.get('end_date_extension') != 0:
             change_requested = True
+
+        # if requested resource is on NESE, add to vars
+        nese = bool(allocation_obj.resources.filter(name__contains="nesetape"))
+
         if attrs_to_change:
             for entry in formset:
                 formset_data = entry.cleaned_data
 
                 new_value = formset_data.get('new_value')
+                # require nese shares to be divisible by 20
+                tbs = int(new_value) if formset_data['name'] == 'Storage Quota (TB)' else False
+                if nese and tbs and tbs % 20 != 0:
+                    messages.error(request, "Tier 3 quantity must be a multiple of 20.")
+                    return HttpResponseRedirect(reverse('allocation-change', kwargs={'pk': pk}))
+
                 if new_value != '':
                     change_requested = True
                     allocation_attribute = AllocationAttribute.objects.get(
@@ -2027,17 +2053,19 @@ def post(self, request, *args, **kwargs):
             for a in attribute_changes_to_make
             if a[0].allocation_attribute_type.name == 'Storage Quota (TB)'
         ]
-        # if requested resource is on NESE, add to vars
-        nese = bool(allocation_obj.resources.filter(name__contains="nesetape"))
 
         email_vars = {'justification': justification}
         if quantity:
             quantity_num = int(float(quantity[0][1]))
             difference = quantity_num - int(float(allocation_obj.size))
             used_percentage = allocation_obj.get_parent_resource.used_percentage
+            current_size = allocation_obj.size
+            if nese:
+                current_size = round(current_size, -1)
+                difference = round(difference, -1)
             email_vars['quantity'] = quantity_num
             email_vars['nese'] = nese
-            email_vars['current_size'] = allocation_obj.size
+            email_vars['current_size'] = current_size
             email_vars['difference'] = difference
             email_vars['used_percentage'] = used_percentage
 

coldfront/core/portal/views.py

@@ -34,7 +34,7 @@ def home(request):
                     & Q(projectuser__status__name='Active')
                 )
             )
-        ).distinct().order_by('-created')[:5]
+        ).distinct().order_by('-created')
 
         allocation_list = Allocation.objects.filter(
             Q(status__name__in=['Active', 'New', 'Renewal Requested', ]) &
@@ -44,7 +44,7 @@ def home(request):
                 (Q(project__projectuser__role__name='Manager') |
                 Q(allocationuser__user=request.user) &
                 Q(allocationuser__status__name='Active'))
-        ).distinct().order_by('-created')[:5]
+        ).distinct().order_by('-created')
 
         managed_allocations = Allocation.objects.filter(
             Q(status__name__in=['Active', 'New', 'Renewal Requested', ])

coldfront/core/project/templates/project/project_detail.html

@@ -601,6 +601,8 @@ <h3 class="d-inline" id="research_outputs"><i class="far fa-newspaper" aria-hidd
 <!-- End Admin Messages -->
 
 <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/v/dt/jszip-2.5.0/dt-1.10.24/af-2.3.5/b-1.7.0/b-colvis-1.7.0/b-html5-1.7.0/b-print-1.7.0/cr-1.5.3/date-1.0.2/fc-3.3.2/kt-2.6.1/r-2.2.7/rg-1.1.2/rr-1.2.7/sl-1.3.2/datatables.min.css"/>
+<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/pdfmake/0.1.36/pdfmake.min.js"></script>
+<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/pdfmake/0.1.36/vfs_fonts.js"></script>
 <script type="text/javascript" src="https://cdn.datatables.net/v/dt/jszip-2.5.0/dt-1.10.24/af-2.3.5/b-1.7.0/b-colvis-1.7.0/b-html5-1.7.0/b-print-1.7.0/cr-1.5.3/date-1.0.2/fc-3.3.2/kt-2.6.1/r-2.2.7/rg-1.1.2/rr-1.2.7/sl-1.3.2/datatables.min.js"></script>
 
 
@@ -677,34 +679,14 @@ <h3 class="d-inline" id="research_outputs"><i class="far fa-newspaper" aria-hidd
        }],
        dom: 'B<"clear">lfrtip',
        order: [[ 3, "desc" ]],
-       buttons: [
-        {
-          name: 'primary',
-          extend: 'collection',
-          background: false,
-          autoClose: true,
-          text: 'Export',
-          buttons: [ 'csv', 'excel', 'pdf' ]
-        }
-        // {
-    //     name: 'toggleusers',
-    //     text: function() {
-    //         return $('#projectuser_table').attr('filter') == "on" ? 'Show All Users' : "Show Active Users"
-    //       },
-    //     action: function(e, dt, node, config) {
-    //       var table = $('#projectuser_table');
-    //       var filter = table.attr('filter') === "on" ? 'off' : "on";
-    //       document.querySelector('#projectuser_table').setAttribute('filter', filter);
-    //       if (filter == 'on') {
-    //         $.fn.dataTable.ext.search.push(
-    //           function(settings, data, dataIndex) {
-    //               return $(dt.row(dataIndex).node()).attr('status') == "Active";
-    //             });
-    //       } else {$.fn.dataTable.ext.search.pop();}
-    //       table.DataTable().draw();
-    //       this.text(filter == 'on' ? 'Show All Users' : "Show Active Users")
-    //     }
-      ]
+       buttons: [{
+         name: 'primary',
+         extend: 'collection',
+         background: false,
+         autoClose: true,
+         text: 'Export',
+         buttons: [ 'csv', 'excel', 'pdf' ]
+       }]
     });
 
     $('#allocation_history').DataTable({

coldfront/core/resource/management/commands/add_resource_defaults.py

@@ -76,13 +76,26 @@ def handle(self, *args, **options):
         for name, desc, is_public, rtype, parent_name, default_value in (
             ('Tier 0', 'Bulk - Lustre', True, storage_tier, None, 1),
             ('Tier 1', 'Enterprise - Isilon', True, storage_tier, None, 1),
+            ('Tier 2', 'CEPH storage', True, storage_tier, None, 1),
             ('Tier 3', 'Attic Storage - Tape', True, storage_tier, None, 20),
             ('holylfs04/tier0', 'Holyoke data center lustre storage', True, storage, 'Tier 0', 1),
             ('holylfs05/tier0', 'Holyoke data center lustre storage', True, storage, 'Tier 0', 1),
             ('nesetape/tier3', 'Cold storage for past projects', True, storage, 'Tier 3', 20),
             ('holy-isilon/tier1', 'Tier1 storage with snapshots and disaster recovery copy', True, storage, 'Tier 1', 1),
             ('bos-isilon/tier1', 'Tier1 storage for on-campus storage mounting', True, storage, 'Tier 1', 1),
             ('holystore01/tier0', 'Luster storage under Tier0', True, storage, 'Tier 0', 1),
+            ('b-nfs02-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
+            ('b-nfs03-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
+            ('b-nfs04-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
+            ('b-nfs05-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
+            ('b-nfs06-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
+            ('b-nfs07-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
+            ('b-nfs08-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
+            ('b-nfs09-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
+            ('h-nfs16-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
+            ('h-nfs17-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
+            ('h-nfs18-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
+            ('h-nfs19-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
         ):
 
             resource_defaults = {