diff --git a/coldfront/config/email.py b/coldfront/config/email.py
index aee4d5f0d..6d9db2978 100644
--- a/coldfront/config/email.py
+++ b/coldfront/config/email.py
@@ -22,3 +22,4 @@
EMAIL_ALLOCATION_EXPIRING_NOTIFICATION_DAYS = ENV.list('EMAIL_ALLOCATION_EXPIRING_NOTIFICATION_DAYS', cast=int, default=[7, 14, 30])
EMAIL_SIGNATURE = ENV.str('EMAIL_SIGNATURE', default='', multiline=True)
EMAIL_ADMINS_ON_ALLOCATION_EXPIRE = ENV.bool('EMAIL_ADMINS_ON_ALLOCATION_EXPIRE', default=False)
+ADMIN_REMINDER_EMAIL = ENV.str('ADMIN_REMINDER_EMAIL', default='')
diff --git a/coldfront/core/allocation/forms.py b/coldfront/core/allocation/forms.py
index dd0d1875f..90b1ef48b 100644
--- a/coldfront/core/allocation/forms.py
+++ b/coldfront/core/allocation/forms.py
@@ -150,18 +150,7 @@ def clean(self):
[d[:3], d[3:8], d[8:12], d[12:18], d[18:24], d[24:28], d[28:33]]
)
cleaned_expensecode = insert_dashes(replace_productcode(digits_only(expense_code)))
- if 'ifxbilling' in settings.INSTALLED_APPS:
- try:
- matched_fiineaccts = FiineAPI.listAccounts(code=cleaned_expensecode)
- if not matched_fiineaccts:
- self.add_error(
- "expense_code",
- "expense code not found in system - please check the code or get in touch with a system administrator."
- )
- except Exception:
- #Not authorized to use accounts_list
- pass
- cleaned_data['expense_code'] = cleaned_expensecode
+ cleaned_data['expense_code'] = cleaned_expensecode
elif existing_expense_codes and existing_expense_codes != '------':
cleaned_data['expense_code'] = existing_expense_codes
return cleaned_data
diff --git a/coldfront/core/allocation/models.py b/coldfront/core/allocation/models.py
index 805394bf0..53f5b21de 100644
--- a/coldfront/core/allocation/models.py
+++ b/coldfront/core/allocation/models.py
@@ -598,20 +598,29 @@ def clean(self):
expected_value_type = self.allocation_attribute_type.attribute_type.name.strip()
error = None
- if expected_value_type == 'Float' and not isinstance(literal_eval(self.value), (float,int)):
- error = 'Value must be a float.'
- elif expected_value_type == 'Int' and not isinstance(literal_eval(self.value), int):
- error = 'Value must be an integer.'
- elif expected_value_type == 'Yes/No' and self.value not in ['Yes', 'No']:
+ if expected_value_type in ['Float', 'Int']:
+ try:
+ literal_val = literal_eval(self.value)
+ except SyntaxError as exc:
+ error = 'Value must be entirely numeric. Please remove any non-numeric characters.'
+ raise ValidationError(
+ f'Invalid Value "{self.value}" for "{self.allocation_attribute_type.name}". {error}'
+ ) from exc
+ if expected_value_type == 'Float' and not isinstance(literal_val, (float,int)):
+ error = 'Value must be a float.'
+ elif expected_value_type == 'Int' and not isinstance(literal_val, int):
+ error = 'Value must be an integer.'
+ elif expected_value_type == "Yes/No" and self.value not in ["Yes", "No"]:
error = 'Allowed inputs are "Yes" or "No".'
- elif expected_value_type == 'Date':
+ elif expected_value_type == "Date":
try:
datetime.datetime.strptime(self.value.strip(), '%Y-%m-%d')
except ValueError:
error = 'Date must be in format YYYY-MM-DD'
if error:
raise ValidationError(
- 'Invalid Value "%s" for "%s". %s' % (self.value, self.allocation_attribute_type.name, error))
+ f'Invalid Value "{self.value}" for "{self.allocation_attribute_type.name}". {error}'
+ )
def __str__(self):
return str(self.allocation_attribute_type.name)
diff --git a/coldfront/core/allocation/tasks.py b/coldfront/core/allocation/tasks.py
index ac9face92..b83f49c2b 100644
--- a/coldfront/core/allocation/tasks.py
+++ b/coldfront/core/allocation/tasks.py
@@ -31,6 +31,7 @@
EMAIL_ADMINS_ON_ALLOCATION_EXPIRE = import_from_settings('EMAIL_ADMINS_ON_ALLOCATION_EXPIRE')
EMAIL_ADMIN_LIST = import_from_settings('EMAIL_ADMIN_LIST')
+ADMIN_REMINDER_EMAIL = import_from_settings('ADMIN_REMINDER_EMAIL')
def update_statuses():
@@ -69,12 +70,14 @@ def send_request_reminder_emails():
'signature': EMAIL_SIGNATURE,
'url_base': f'{CENTER_BASE_URL.strip("/")}/allocation/change-request/'
}
- send_admin_email_template(
- 'Pending Allocation Changes',
- 'email/pending_allocation_changes.txt',
- allocation_change_template_context,
- )
+ send_email_template(
+ subject='Pending Allocation Changes',
+ template_name='email/pending_allocation_changes.txt',
+ template_context=allocation_change_template_context,
+ sender=EMAIL_SENDER,
+ receiver_list=[ADMIN_REMINDER_EMAIL,],
+ )
# Allocation Requests are allocations marked as "new"
pending_allocations = Allocation.objects.filter(
status__name = 'New', created__lte=req_alert_date
@@ -87,11 +90,15 @@ def send_request_reminder_emails():
'signature': EMAIL_SIGNATURE,
'url_base': f'{CENTER_BASE_URL.strip("/")}/allocation/'
}
- send_admin_email_template(
- 'Pending Allocations',
- 'email/pending_allocations.txt',
- new_allocation_template_context,
+
+ send_email_template(
+ subject='Pending Allocations',
+ template_name='email/pending_allocations.txt',
+ template_context=new_allocation_template_context,
+ sender=EMAIL_SENDER,
+ receiver_list=[ADMIN_REMINDER_EMAIL,],
)
+
# return statement for testing
return (pending_changerequests, pending_allocations)
diff --git a/coldfront/core/allocation/templates/allocation/allocation_detail.html b/coldfront/core/allocation/templates/allocation/allocation_detail.html
index fbf634206..e95d9634b 100644
--- a/coldfront/core/allocation/templates/allocation/allocation_detail.html
+++ b/coldfront/core/allocation/templates/allocation/allocation_detail.html
@@ -106,11 +106,11 @@ Allocation Information
Service Period: |
1 Month |
- {% if offer_letter_code %}
+ {% if expense_code %}
| Requested Expense Code: |
- {% for code in offer_letter_code %}
+ {% for code in expense_code %}
{{ code.value }}
{% endfor %}
|
diff --git a/coldfront/core/allocation/tests/test_models.py b/coldfront/core/allocation/tests/test_models.py
index 940e2cb2b..ff7cf2ded 100644
--- a/coldfront/core/allocation/tests/test_models.py
+++ b/coldfront/core/allocation/tests/test_models.py
@@ -1,6 +1,7 @@
"""Unit tests for the allocation models"""
from django.test import TestCase
+from django.core.exceptions import ValidationError
from coldfront.core.test_helpers.factories import setup_models, AllocationFactory
@@ -8,13 +9,14 @@
"coldfront/core/test_helpers/test_data/test_fixtures/ifx.json",
]
+
class AllocationModelTests(TestCase):
"""tests for Allocation model"""
fixtures = UTIL_FIXTURES
@classmethod
def setUpTestData(cls):
- """Set up project to test model properties and methods"""
+ """Set up allocation to test model properties and methods"""
setup_models(cls)
def test_allocation_str(self):
@@ -23,9 +25,9 @@ def test_allocation_str(self):
self.proj_allocation.get_parent_resource.name,
self.proj_allocation.project.pi
)
-
self.assertEqual(str(self.proj_allocation), allocation_str)
+
def test_allocation_usage_property(self):
"""Test that allocation usage property displays correctly"""
self.assertEqual(self.proj_allocation.usage, 10)
@@ -34,3 +36,27 @@ def test_allocation_usage_property_na(self):
"""Create allocation with no usage. Usage property should return None"""
allocation = AllocationFactory()
self.assertIsNone(allocation.usage)
+
+class AllocationAttributeModelTests(TestCase):
+ """Tests for allocationattribute models"""
+ fixtures = UTIL_FIXTURES
+
+ @classmethod
+ def setUpTestData(cls):
+ """Set up allocationattribute to test model properties and methods"""
+ setup_models(cls)
+ cls.allocationattribute = cls.proj_allocation.allocationattribute_set.get(
+ allocation_attribute_type__name='Storage Quota (TB)'
+ )
+
+ def test_allocationattribute_clean_no_error(self):
+ """cleaning a numeric value for an int or float AllocationAttributeType produces no error"""
+ self.allocationattribute.value = "1000"
+ self.allocationattribute.clean()
+
+ def test_allocationattribute_clean_nonnumeric_error(self):
+ """cleaning a non-numeric value for int or float AllocationAttributeTypes returns an informative error message"""
+
+ self.allocationattribute.value = "1000TB"
+ with self.assertRaisesMessage(ValidationError, 'Value must be entirely numeric. Please remove any non-numeric characters.'):
+ self.allocationattribute.clean()
diff --git a/coldfront/core/allocation/views.py b/coldfront/core/allocation/views.py
index e4b7379f7..b76baaeee 100644
--- a/coldfront/core/allocation/views.py
+++ b/coldfront/core/allocation/views.py
@@ -68,6 +68,7 @@
if 'ifxbilling' in settings.INSTALLED_APPS:
+ from fiine.client import API as FiineAPI
from ifxbilling.models import Account, UserProductAccount
if 'django_q' in settings.INSTALLED_APPS:
from django_q.tasks import Task
@@ -560,6 +561,10 @@ def form_valid(self, form):
quantity = form_data.get('quantity', 1)
allocation_account = form_data.get('allocation_account', None)
+ if resource_obj.name == "Tier 3" and quantity % 20 != 0:
+ form.add_error("quantity", format_html("Tier 3 quantity must be a multiple of 20."))
+ return self.form_invalid(form)
+
# A resource is selected that requires an account name selection but user has no account names
if (
ALLOCATION_ACCOUNT_ENABLED
@@ -659,7 +664,19 @@ def form_valid(self, form):
'quantity':quantity,
'nese': nese,
'used_percentage': used_percentage,
+ 'expense_code': expense_code,
+ 'unmatched_code': False,
}
+
+ if 'ifxbilling' in settings.INSTALLED_APPS:
+ try:
+ matched_fiineaccts = FiineAPI.listAccounts(code=expense_code)
+ if not matched_fiineaccts:
+ other_vars['unmatched_code'] = True
+ except Exception:
+ #Not authorized to use accounts_list
+ pass
+
send_allocation_admin_email(
allocation_obj,
'New Allocation Request',
@@ -1800,7 +1817,6 @@ def post(self, request, *args, **kwargs):
if new_value != attribute_change.new_value:
attribute_change.new_value = new_value
attribute_change.save()
-
if action == 'update':
message = 'Allocation change request updated!'
if action == 'approve':
@@ -1986,11 +2002,21 @@ def post(self, request, *args, **kwargs):
if form_data.get('end_date_extension') != 0:
change_requested = True
+
+ # if requested resource is on NESE, add to vars
+ nese = bool(allocation_obj.resources.filter(name__contains="nesetape"))
+
if attrs_to_change:
for entry in formset:
formset_data = entry.cleaned_data
new_value = formset_data.get('new_value')
+ # require nese shares to be divisible by 20
+ tbs = int(new_value) if formset_data['name'] == 'Storage Quota (TB)' else False
+ if nese and tbs and tbs % 20 != 0:
+ messages.error(request, "Tier 3 quantity must be a multiple of 20.")
+ return HttpResponseRedirect(reverse('allocation-change', kwargs={'pk': pk}))
+
if new_value != '':
change_requested = True
allocation_attribute = AllocationAttribute.objects.get(
@@ -2027,17 +2053,19 @@ def post(self, request, *args, **kwargs):
for a in attribute_changes_to_make
if a[0].allocation_attribute_type.name == 'Storage Quota (TB)'
]
- # if requested resource is on NESE, add to vars
- nese = bool(allocation_obj.resources.filter(name__contains="nesetape"))
email_vars = {'justification': justification}
if quantity:
quantity_num = int(float(quantity[0][1]))
difference = quantity_num - int(float(allocation_obj.size))
used_percentage = allocation_obj.get_parent_resource.used_percentage
+ current_size = allocation_obj.size
+ if nese:
+ current_size = round(current_size, -1)
+ difference = round(difference, -1)
email_vars['quantity'] = quantity_num
email_vars['nese'] = nese
- email_vars['current_size'] = allocation_obj.size
+ email_vars['current_size'] = current_size
email_vars['difference'] = difference
email_vars['used_percentage'] = used_percentage
diff --git a/coldfront/core/portal/views.py b/coldfront/core/portal/views.py
index 0db6dac9f..855e7ed80 100644
--- a/coldfront/core/portal/views.py
+++ b/coldfront/core/portal/views.py
@@ -34,7 +34,7 @@ def home(request):
& Q(projectuser__status__name='Active')
)
)
- ).distinct().order_by('-created')[:5]
+ ).distinct().order_by('-created')
allocation_list = Allocation.objects.filter(
Q(status__name__in=['Active', 'New', 'Renewal Requested', ]) &
@@ -44,7 +44,7 @@ def home(request):
(Q(project__projectuser__role__name='Manager') |
Q(allocationuser__user=request.user) &
Q(allocationuser__status__name='Active'))
- ).distinct().order_by('-created')[:5]
+ ).distinct().order_by('-created')
managed_allocations = Allocation.objects.filter(
Q(status__name__in=['Active', 'New', 'Renewal Requested', ])
diff --git a/coldfront/core/project/templates/project/project_detail.html b/coldfront/core/project/templates/project/project_detail.html
index e5ff96484..59849672a 100644
--- a/coldfront/core/project/templates/project/project_detail.html
+++ b/coldfront/core/project/templates/project/project_detail.html
@@ -601,6 +601,8 @@
+
+
@@ -677,34 +679,14 @@ lfrtip',
order: [[ 3, "desc" ]],
- buttons: [
- {
- name: 'primary',
- extend: 'collection',
- background: false,
- autoClose: true,
- text: 'Export',
- buttons: [ 'csv', 'excel', 'pdf' ]
- }
- // {
- // name: 'toggleusers',
- // text: function() {
- // return $('#projectuser_table').attr('filter') == "on" ? 'Show All Users' : "Show Active Users"
- // },
- // action: function(e, dt, node, config) {
- // var table = $('#projectuser_table');
- // var filter = table.attr('filter') === "on" ? 'off' : "on";
- // document.querySelector('#projectuser_table').setAttribute('filter', filter);
- // if (filter == 'on') {
- // $.fn.dataTable.ext.search.push(
- // function(settings, data, dataIndex) {
- // return $(dt.row(dataIndex).node()).attr('status') == "Active";
- // });
- // } else {$.fn.dataTable.ext.search.pop();}
- // table.DataTable().draw();
- // this.text(filter == 'on' ? 'Show All Users' : "Show Active Users")
- // }
- ]
+ buttons: [{
+ name: 'primary',
+ extend: 'collection',
+ background: false,
+ autoClose: true,
+ text: 'Export',
+ buttons: [ 'csv', 'excel', 'pdf' ]
+ }]
});
$('#allocation_history').DataTable({
diff --git a/coldfront/core/resource/management/commands/add_resource_defaults.py b/coldfront/core/resource/management/commands/add_resource_defaults.py
index 8da072dbe..119bbbad9 100644
--- a/coldfront/core/resource/management/commands/add_resource_defaults.py
+++ b/coldfront/core/resource/management/commands/add_resource_defaults.py
@@ -76,6 +76,7 @@ def handle(self, *args, **options):
for name, desc, is_public, rtype, parent_name, default_value in (
('Tier 0', 'Bulk - Lustre', True, storage_tier, None, 1),
('Tier 1', 'Enterprise - Isilon', True, storage_tier, None, 1),
+ ('Tier 2', 'CEPH storage', True, storage_tier, None, 1),
('Tier 3', 'Attic Storage - Tape', True, storage_tier, None, 20),
('holylfs04/tier0', 'Holyoke data center lustre storage', True, storage, 'Tier 0', 1),
('holylfs05/tier0', 'Holyoke data center lustre storage', True, storage, 'Tier 0', 1),
@@ -83,6 +84,18 @@ def handle(self, *args, **options):
('holy-isilon/tier1', 'Tier1 storage with snapshots and disaster recovery copy', True, storage, 'Tier 1', 1),
('bos-isilon/tier1', 'Tier1 storage for on-campus storage mounting', True, storage, 'Tier 1', 1),
('holystore01/tier0', 'Luster storage under Tier0', True, storage, 'Tier 0', 1),
+ ('b-nfs02-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
+ ('b-nfs03-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
+ ('b-nfs04-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
+ ('b-nfs05-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
+ ('b-nfs06-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
+ ('b-nfs07-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
+ ('b-nfs08-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
+ ('b-nfs09-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
+ ('h-nfs16-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
+ ('h-nfs17-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
+ ('h-nfs18-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
+ ('h-nfs19-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1),
):
resource_defaults = {
diff --git a/coldfront/core/utils/fasrc.py b/coldfront/core/utils/fasrc.py
index 4ac3aec62..faf4c1b40 100644
--- a/coldfront/core/utils/fasrc.py
+++ b/coldfront/core/utils/fasrc.py
@@ -59,8 +59,10 @@ def select_one_project_allocation(project_obj, resource_obj, dirpath=None):
project_obj
resource_obj
"""
- allocation_query = project_obj.allocation_set.filter(
- resources__id=resource_obj.id)
+ filter_vals = {'resources__id': resource_obj.id}
+ # if dirpath:
+ # filter_vals['allocationattribute__value'] = dirpath
+ allocation_query = project_obj.allocation_set.filter(**filter_vals)
if allocation_query.count() == 1:
allocation_obj = allocation_query.first()
elif allocation_query.count() < 1:
diff --git a/coldfront/core/utils/mail.py b/coldfront/core/utils/mail.py
index 5a653f7d9..977ac188b 100644
--- a/coldfront/core/utils/mail.py
+++ b/coldfront/core/utils/mail.py
@@ -106,11 +106,12 @@ def send_allocation_admin_email(
url_path = reverse('allocation-request-list')
url = build_link(url_path, domain_url=domain_url)
- pi_name = f'{allocation_obj.project.pi.first_name} {allocation_obj.project.pi.last_name} ({allocation_obj.project.pi.username})'
+ pi_name = f'{allocation_obj.project.pi.first_name} {allocation_obj.project.pi.last_name}'
resource_name = allocation_obj.get_parent_resource
ctx = email_template_context()
- ctx['pi'] = pi_name
+ ctx['pi_name'] = pi_name
+ ctx['pi_username'] = f'{allocation_obj.project.pi.username}'
ctx['resource'] = resource_name
ctx['url'] = url
if other_vars:
diff --git a/coldfront/plugins/fasrc/management/commands/id_import_new_allocations.py b/coldfront/plugins/fasrc/management/commands/id_import_new_allocations.py
index c1703a90a..737898bd9 100644
--- a/coldfront/plugins/fasrc/management/commands/id_import_new_allocations.py
+++ b/coldfront/plugins/fasrc/management/commands/id_import_new_allocations.py
@@ -11,18 +11,25 @@
from django.core.management.base import BaseCommand
from coldfront.core.project.models import ProjectStatusChoice
-from coldfront.core.allocation.models import (AllocationUser,
- AllocationAttribute,
- AllocationAttributeType,
- AllocationStatusChoice,
- AllocationUserStatusChoice)
+from coldfront.core.allocation.models import (
+ AllocationUser,
+ AllocationAttribute,
+ AllocationStatusChoice,
+ AllocationAttributeType,
+ AllocationUserStatusChoice,
+)
from coldfront.core.utils.fasrc import update_csv, select_one_project_allocation, save_json
from coldfront.core.resource.models import Resource
-from coldfront.plugins.sftocf.utils import (StarFishRedash,
- STARFISH_SERVER,
- pull_sf_push_cf_redash)
-from coldfront.plugins.fasrc.utils import (AllTheThingsConn,
- match_entries_with_projects, push_quota_data)
+from coldfront.plugins.sftocf.utils import (
+ StarFishRedash,
+ STARFISH_SERVER,
+ pull_sf_push_cf_redash
+)
+from coldfront.plugins.fasrc.utils import (
+ AllTheThingsConn,
+ match_entries_with_projects,
+ push_quota_data
+)
logger = logging.getLogger(__name__)
@@ -52,24 +59,28 @@ def handle(self, *args, **options):
redash_api = StarFishRedash(STARFISH_SERVER)
allocation_usages = redash_api.return_query_results(query='subdirectory')
- subdir_type = AllocationAttributeType.objects.get(name="Subdirectory")
+ subdir_type = AllocationAttributeType.objects.get(name='Subdirectory')
- for lab, allocations in result_json_cleaned.items():
- project = proj_models.get(title=lab)
+ for project in proj_models:
if project.status.name == 'New':
project.status = ProjectStatusChoice.objects.get(name='Active')
project.save()
+
+ for lab, allocations in result_json_cleaned.items():
+ project = proj_models.get(title=lab)
for entry in allocations:
lab_name = entry['lab']
lab_server = entry['server']
lab_path = entry['fs_path'].replace(f'/n/{entry["server"]}/', '')
- resource = Resource.objects.get(name__contains=entry["server"])
- alloc_obj = select_one_project_allocation(project, resource, dirpath=entry['fs_path'])
+ resource = Resource.objects.get(name__contains=entry['server'])
+ alloc_obj = select_one_project_allocation(project, resource, dirpath=lab_path)
if alloc_obj is not None:
continue
- lab_usage_entries = [i for i in allocation_usages if i['vol_name'] == lab_server
- and lab_path in i['path'] and i['group_name'] == lab_name]
+ lab_usage_entries = [
+ i for i in allocation_usages if i['vol_name'] == lab_server
+ and lab_path in i['path'] and i['group_name'] == lab_name
+ ]
if not lab_usage_entries:
continue
@@ -81,27 +92,30 @@ def handle(self, *args, **options):
'start_date': datetime.now(),
'is_changeable': True,
'justification': f'Allocation Information for {lab_name}',
- }
- )
+ }
+ )
# do not modify status of inactive allocations
+ allocation_str = f'{lab_name} {lab_server} {lab_path}'
if created:
allocation.resources.add(resource)
AllocationAttribute.objects.create(
allocation=allocation,
allocation_attribute_type_id=subdir_type.pk,
value=lab_path
- )
- print(f'allocation created: {lab_name}')
+ )
+ print(f'allocation created: {allocation_str}')
allocation.save()
- command_report['allocations_added'].append(f'{lab_name} {lab_server} {lab_path}')
- row = {'project_title': lab_name,
- 'server': lab_server,
- 'path': lab_path,
- 'date': datetime.now()}
+ command_report['allocations_added'].append(allocation_str)
+ row = {
+ 'project_title': lab_name,
+ 'server': lab_server,
+ 'path': lab_path,
+ 'date': datetime.now()
+ }
added_allocations_df = added_allocations_df.append(row, ignore_index=True)
else:
- command_report['allocations_existing'].append(f'{lab_name} {lab_server} {lab_path}')
+ command_report['allocations_existing'].append(allocation_str)
continue
pi_obj = project.pi
try:
@@ -112,7 +126,8 @@ def handle(self, *args, **options):
'status': AllocationUserStatusChoice.objects.get(name='Active')}
)
except ValidationError:
- logger.warning('adding PI %s to allocation %s failed', pi_obj.pi.username, allocation.pk)
+ logger.warning('adding PI %s to allocation %s failed',
+ pi_obj.pi.username, allocation.pk)
created = None
if created:
print('PI added: ' + project.pi.username)
diff --git a/coldfront/plugins/fasrc/utils.py b/coldfront/plugins/fasrc/utils.py
index 28ca4ca7b..1f8666657 100644
--- a/coldfront/plugins/fasrc/utils.py
+++ b/coldfront/plugins/fasrc/utils.py
@@ -27,50 +27,69 @@ def produce_query_statement(self, vol_type, volumes=None):
query_dict = {
'quota': {
- 'match': '[r:HasQuota]-(e:Quota) MATCH (d:ConfigValue {Name: \'Quota.Invocation\'})',
- 'validation_query': 'NOT ((e.SizeGB IS null) OR (e.usedBytes = 0 AND e.SizeGB = 1024)) AND NOT (e.Path IS null)',
+ 'relation': 'HasQuota',
+ 'match': "(e:Quota) MATCH (d:ConfigValue {Name: 'Quota.Invocation'})",
+ 'server': 'filesystem',
+ 'validation_query':
+ "NOT ((e.SizeGB IS null) OR (e.usedBytes = 0 AND e.SizeGB = 1024)) \
+ AND (datetime() - duration('P31D') <= datetime(r.DotsLFSUpdateDate)) \
+ AND NOT (e.Path IS null)",
'r_updated': 'DotsLFSUpdateDate',
- 'storage_type': '\'Quota\'',
+ 'storage_type': 'Quota',
'usedgb': 'usedGB',
'sizebytes': 'limitBytes',
'usedbytes': 'usedBytes',
'fs_path': 'Path',
- 'server': 'filesystem',
'server_replace': '/n/',
'path_replace': '/n//',
'unique':'datetime(e.DotsLFSUpdateDate) as begin_date'
},
'isilon': {
- 'match': '[r:Owns]-(e:IsilonPath) MATCH (d:ConfigValue {Name: \'IsilonPath.Invocation\'})',
- 'validation_query':"r.DotsUpdateDate = d.DotsUpdateDate \
- AND NOT (e.Path =~ '.*/rc_admin/.*')\
- AND (e.Path =~ '.*labs.*')\
- AND NOT (e.SizeGB = 0)",
- 'r_updated': 'DotsUpdateDate',
- 'storage_type':'\'Isilon\'',
+ 'relation': 'Owns',
+ 'match': "(e:IsilonPath) MATCH (d:ConfigValue {Name: 'IsilonPath.Invocation'})",
+ 'server': 'Isilon',
+ 'validation_query': "r.DotsUpdateDate = d.DotsUpdateDate \
+ AND NOT (e.Path =~ '.*/rc_admin/.*')\
+ AND (e.Path =~ '.*labs.*')\
+ AND (datetime() - duration('P31D') <= datetime(r.DotsUpdateDate)) \
+ AND NOT (e.SizeGB = 0)",
'fs_path':'Path',
- 'server':'Isilon',
+ 'r_updated': 'DotsUpdateDate',
+ 'storage_type': 'Isilon',
'usedgb': 'UsedGB',
'sizebytes': 'SizeBytes',
'usedbytes': 'UsedBytes',
'server_replace': '01.rc.fas.harvard.edu',
'path_replace': '/ifs/',
- 'unique':'datetime(e.DotsUpdateDate) as begin_date'
- }
+ 'unique': 'datetime(e.DotsUpdateDate) as begin_date'
+ },
+ 'volume': {
+ 'relation': 'Owns',
+ 'match': '(e:Volume)',
+ 'server': 'Hostname',
+ 'validation_query': 'NOT (e.SizeGB = 0)',
+ 'r_updated': 'DotsLVSUpdateDate',
+ 'storage_type': 'Volume',
+ 'fs_path': 'LogicalVolume',
+ 'path_replace': '/dev/data/',
+ 'usedgb': 'UsedGB',
+ 'sizebytes': 'SizeGB * 1073741824',
+ 'usedbytes': 'UsedGB * 1073741824',
+ 'server_replace': '.rc.fas.harvard.edu',
+ 'unique': 'datetime(e.DotsLVSUpdateDate) as update_date, \
+ datetime(e.DotsLVDisplayUpdateDate) as display_date'
+ },
}
d = query_dict[vol_type]
- if volumes:
- volumes = '|'.join(volumes)
- else:
- volumes = '|'.join([r.name.split('/')[0] for r in Resource.objects.all()])
- where = f"(e.{d['server']} =~ \'.*({volumes}).*\')"
-
+ if not volumes:
+ volumes = [r.name.split('/')[0] for r in Resource.objects.filter(resource_type__name='Storage')]
+ volumes = '|'.join(volumes)
+ where = f"(e.{d['server']} =~ '.*({volumes}).*')"
statement = {
- 'statement': f"MATCH p=(g:Group)-{d['match']} \
+ 'statement': f"MATCH p=(g:Group)-[r:{d['relation']}]-{d['match']} \
WHERE {where} AND {d['validation_query']}\
AND NOT (g.ADSamAccountName =~ '.*(disabled|rc_admin).*')\
- AND (datetime() - duration('P31D') <= datetime(r.{d['r_updated']})) \
RETURN \
{d['unique']}, \
g.ADSamAccountName as lab,\
@@ -79,7 +98,7 @@ def produce_query_statement(self, vol_type, volumes=None):
e.{d['usedbytes']} as byte_usage,\
(e.{d['usedgb']} / 1024.0) as tb_usage,\
replace(e.{d['fs_path']}, '{d['path_replace']}', '') as fs_path, \
- {d['storage_type']} as storage_type, \
+ '{d['storage_type']}' as storage_type, \
datetime(r.{d['r_updated']}) as rel_updated, \
replace(e.{d['server']}, '{d['server_replace']}', '') as server"
}
@@ -89,28 +108,32 @@ def produce_query_statement(self, vol_type, volumes=None):
class QuotaDataPuller:
"""pull and standardize quota data"""
- def pull(self, format):
- standardizer = self.get_standardizer(format)
+ def __init__(self, volumes=None):
+ self.volumes = volumes
+
+ def pull(self, standard):
+ standardizer = self.get_standardizer(standard)
return standardizer()
- def get_standardizer(self, format):
- if format == 'ATTQuery':
+ def get_standardizer(self, standard):
+ if standard == 'ATTQuery':
return self._standardize_attquery
- if format == 'NESEfile':
+ if standard == 'NESEfile':
return self._standardize_nesefile
- raise ValueError(format)
+ raise ValueError(standard)
def _standardize_attquery(self):
- attconn = AllTheThingsConn()
+ attconn = AllTheThingsConn(volumes=self.volumes)
resp_json = attconn.pull_quota_data()
return attconn.format_query_results(resp_json)
def _standardize_nesefile(self):
datafile = 'nese_data/pools'
header_file = 'nese_data/pools.header'
- translator = dict((
- kv.split('=') for kv in (l.strip('\n') for l in open('nese_data/groupkey'))
- ))
+ with open('nese_data/groupkey') as groupkey_file:
+ translator = dict((
+ kv.split('=') for kv in (l.strip('\n') for l in groupkey_file)
+ ))
headers_df = pd.read_csv(header_file, header=0, delim_whitespace=True)
headers = headers_df.columns.values.tolist()
data = pd.read_csv(datafile, names=headers, delim_whitespace=True)
@@ -133,50 +156,44 @@ def _standardize_nesefile(self):
return nesedict
-
class AllTheThingsConn:
- def __init__(self):
+ def __init__(self, volumes=None):
self.url = 'https://allthethings.rc.fas.harvard.edu:7473/db/data/transaction/commit'
self.token = import_from_settings('NEO4JP', '')
self.headers = generate_headers(self.token)
+ self.volumes = volumes
def post_query(self, query):
- resp = requests.post(self.url, headers=self.headers, data=json.dumps(query), verify=False)
+ resp = requests.post(self.url, headers=self.headers, data=json.dumps(query))
return json.loads(resp.text)
def format_query_results(self, resp_json):
result_dicts = list(resp_json['results'])
- return [dict(zip(rdict['columns'],entrydict['row'])) \
- for rdict in result_dicts for entrydict in rdict['data'] ]
+ return [dict(zip(d['columns'], ed['row'])) for d in result_dicts for ed in d['data']]
def stage_user_member_query(self, groupsearch, pi=False):
- match_statement = f'MATCH (u:User)-[r:MemberOf|ManagedBy]-(g:Group) \
- WHERE (g.ADSamAccountName =~ \'{groupsearch}\')'
- return_statement = 'type(r) AS relationship,\
- g.ADManaged_By AS group_manager'
+ match_vars = '(u:User)-[r:MemberOf|ManagedBy]-(g:Group) WHERE'
+ return_vars = 'type(r) AS relationship, g.ADManaged_By AS group_manager'
if pi:
- match_statement = f"MATCH (g:Group) WITH g\
- MATCH (u:User)\
- WHERE (g.ADSamAccountName =~ \'({groupsearch})\') \
- AND u.ADSamAccountName = g.ADManaged_By"
- return_statement = 'u.ADParentCanonicalName AS path, \
- u.ADDepartment AS department, '
+ match_vars = '(g:Group) WITH g MATCH (u:User)\
+ WHERE u.ADSamAccountName = g.ADManaged_By AND'
+ return_vars = 'u.ADParentCanonicalName AS path, u.ADDepartment AS department'
query = {'statements': [{
- 'statement': f'{match_statement} \
- RETURN \
- u.ADgivenName AS first_name, \
- u.ADsurname AS last_name, \
- u.ADSamAccountName AS user_name, \
- u.ADenabled AS user_enabled, \
- g.ADSamAccountName AS group_name,\
- {return_statement} \
- g.ADManaged_By AS group_manager, \
- u.ADgidNumber AS user_gid_number, \
- u.ADTitle AS title, \
- u.ADCompany AS company, \
- g.ADgidNumber AS group_gid_number'
- }]}
+ 'statement': f"MATCH {match_vars} (g.ADSamAccountName =~ '({groupsearch})')\
+ RETURN \
+ u.ADgivenName AS first_name, \
+ u.ADsurname AS last_name, \
+ u.ADSamAccountName AS user_name, \
+ u.ADenabled AS user_enabled, \
+ g.ADSamAccountName AS group_name,\
+ {return_vars},\
+ g.ADManaged_By AS group_manager, \
+ u.ADgidNumber AS user_gid_number, \
+ u.ADTitle AS title, \
+ u.ADCompany AS company, \
+ g.ADgidNumber AS group_gid_number"
+ }]}
resp_json = self.post_query(query)
resp_json_formatted = self.format_query_results(resp_json)
return resp_json_formatted
@@ -188,14 +205,13 @@ def collect_group_membership(self, groupsearch):
resp_json_formatted = self.stage_user_member_query(groupsearch)
return resp_json_formatted
-
def collect_pi_data(self, grouplist):
"""collect information on pis for a given list of groups
"""
resp_json_formatted = self.stage_user_member_query(grouplist, pi=True)
return resp_json_formatted
- def pull_quota_data(self, volumes=None):
+ def pull_quota_data(self):
"""Produce JSON file of quota data for LFS and Isilon from AlltheThings.
Parameters
----------
@@ -203,12 +219,9 @@ def pull_quota_data(self, volumes=None):
"""
logger = logging.getLogger('import_quotas')
query = ATTAllocationQuery()
- if volumes:
- volumes = '|'.join(volumes)
- else:
- volumes = '|'.join([r.name.split('/')[0] for r in Resource.objects.all()])
- query.produce_query_statement('isilon')
- query.produce_query_statement('quota')
+ query.produce_query_statement('isilon', volumes=self.volumes)
+ query.produce_query_statement('quota', volumes=self.volumes)
+ query.produce_query_statement('volume', volumes=self.volumes)
resp_json = self.post_query(query.queries)
logger.debug(resp_json)
return resp_json
@@ -223,6 +236,7 @@ def matched_dict_processing(allocation, data_dicts, paired_allocs, log_message):
logger.warning('too many matches for allocation %s: %s', allocation, data_dicts)
return paired_allocs
+
def pair_allocations_data(project, quota_dicts):
"""pair allocations with usage dicts"""
logger = logging.getLogger('import_quotas')
@@ -253,7 +267,9 @@ def pair_allocations_data(project, quota_dicts):
]
unpaired_dicts = [d for d in unpaired_dicts if d not in paired_allocs.values()]
if unpaired_dicts or unpaired_allocs:
- logger.warning("WARNING: unpaired allocation data: %s %s", unpaired_allocs, unpaired_dicts)
+ logger.warning(
+ "WARNING: unpaired allocation data: %s %s", unpaired_allocs, unpaired_dicts
+ )
return paired_allocs
@@ -311,9 +327,9 @@ def push_quota_data(result_file):
defaults={'value':True}
)
counts['complete'] += 1
- except Exception as e:
+ except Exception as exc:
allocation_name = f"{data_dict['lab']}/{data_dict['server']}"
- errored_allocations[allocation_name] = e
+ errored_allocations[allocation_name] = exc
log_missing('allocation', missing_allocations)
logger.warning('error counts: %s', counts)
logger.warning('errored_allocations:\n%s', errored_allocations)
@@ -330,10 +346,11 @@ def match_entries_with_projects(result_json):
[result_json.pop(t) for t in missing_proj_titles]
return result_json, proj_models
+
def pull_push_quota_data(volumes=None):
logger = logging.getLogger('import_quotas')
- att_data = QuotaDataPuller().pull('ATTQuery')
- nese_data = QuotaDataPuller().pull('NESEfile')
+ att_data = QuotaDataPuller(volumes=volumes).pull('ATTQuery')
+ nese_data = QuotaDataPuller(volumes=volumes).pull('NESEfile')
combined_data = att_data + nese_data
resp_json_by_lab = {entry['lab']:[] for entry in combined_data}
[resp_json_by_lab[e['lab']].append(e) for e in combined_data]
@@ -347,8 +364,5 @@ def pull_push_quota_data(volumes=None):
def generate_headers(token):
"""Generate 'headers' attribute by using the 'token' attribute.
"""
- headers = {
- 'accept': 'application/json',
- 'Authorization': f'Bearer {token}',
- }
+ headers = {'accept': 'application/json', 'Authorization': f'Bearer {token}'}
return headers
diff --git a/coldfront/plugins/ifx/management/commands/processIfxappsMessages.py b/coldfront/plugins/ifx/management/commands/processIfxappsMessages.py
index 2033c6976..3e5737c12 100644
--- a/coldfront/plugins/ifx/management/commands/processIfxappsMessages.py
+++ b/coldfront/plugins/ifx/management/commands/processIfxappsMessages.py
@@ -5,6 +5,7 @@
'''
import logging
from time import sleep
+from django.db import connection
from django.conf import settings
from django.core.management.base import BaseCommand
from ifxbilling.fiine import handle_fiine_ifxapps_messages
@@ -80,3 +81,4 @@ def handle(self, *args, **kwargs):
logger.error(e)
sleep(sleep_seconds)
+ connection.close()
diff --git a/coldfront/plugins/ifx/models.py b/coldfront/plugins/ifx/models.py
index 6da48207a..1c0fd6d82 100644
--- a/coldfront/plugins/ifx/models.py
+++ b/coldfront/plugins/ifx/models.py
@@ -136,12 +136,13 @@ def resource_post_save(sender, instance, **kwargs):
except ProductResource.DoesNotExist:
# Need to create a Product and ProductResource for this Resource
products = FiineAPI.listProducts(product_name=instance.name)
+ facility = Facility.objects.get(name='Research Computing Storage')
if not products:
- facility = Facility.objects.get(name='Research Computing Storage')
product = create_new_product(product_name=instance.name, product_description=instance.name, facility=facility)
else:
fiine_product = products[0].to_dict()
- fiine_product.pop('facility')
+ fiine_product.pop('object_code_category')
+ fiine_product['facility'] = facility
fiine_product['billing_calculator'] = 'coldfront.plugins.ifx.calculator.NewColdfrontBillingCalculator'
(product, created) = Product.objects.get_or_create(**fiine_product)
product_resource = ProductResource.objects.create(product=product, resource=instance)
diff --git a/coldfront/templates/email/allocation_renewed.txt b/coldfront/templates/email/allocation_renewed.txt
index c08a1cf0c..4671c63fa 100644
--- a/coldfront/templates/email/allocation_renewed.txt
+++ b/coldfront/templates/email/allocation_renewed.txt
@@ -1,2 +1,3 @@
-A allocation has been renewed for {{pi}} - {{resource}}. Please activate the allocation:
+A allocation has been renewed for {{pi_name}} ({{pi_username}}) - {{resource}}.
+Please activate the allocation:
{{url}}
diff --git a/coldfront/templates/email/new_allocation_change_request.txt b/coldfront/templates/email/new_allocation_change_request.txt
index 2a2fc1003..2feed9f94 100644
--- a/coldfront/templates/email/new_allocation_change_request.txt
+++ b/coldfront/templates/email/new_allocation_change_request.txt
@@ -1,5 +1,5 @@
{% load mathfilters %}
-An allocation change request for has been made for {{pi}} - {{resource}}.
+An allocation change request for has been made for {{pi_name}} ({{pi_username}}) - {{resource}}.
{% if quantity %}
Requested total size: {{quantity}} TB.
@@ -11,14 +11,14 @@ This will require adding {{difference|floatformat}} TB to the current size of {{
{% endif %}
{% if nese %}
-Here is a draft ticket to send to NESE:
+Here is a draft ticket to send to NESE. Please carefully check the values to ensure their accuracy.
To: help@nese.mghpcc.org
Subject: TB Globus tape setup for HU
-Service (access type) = S3 vs Globus:
-If Globus, short description for Globus Share:
-Tennant name (8 char max - Ex. acohen): {{pi.username}}
+Service (access type) = S3 vs Globus: Globus
+If Globus, short description for Globus Share: {{pi_name}}
+Tennant name (11 char max - Ex. acohen): {{pi_username}}
Size in 20TB allotments: {{quantity|div:19.48|floatformat:"0"}} (This is a difference of {{difference|div:19.48|floatformat:"0"}} 20TB units from the current size of {{current_size|div:19.48|floatformat:"0"}} 20TB units)
Any additional details:
Any additional administrators (if applicable):
diff --git a/coldfront/templates/email/new_allocation_request.txt b/coldfront/templates/email/new_allocation_request.txt
index 1b280e49f..4697a00fe 100644
--- a/coldfront/templates/email/new_allocation_request.txt
+++ b/coldfront/templates/email/new_allocation_request.txt
@@ -1,5 +1,5 @@
{% load mathfilters %}
-A new allocation has been requested for {{pi}} - {{resource}}.
+A new allocation has been requested for {{pi_name}} ({{pi_username}}) - {{resource}}.
Requested size: {{quantity}} TB.
Justification:
{{justification}}
@@ -8,15 +8,21 @@ Justification:
{{resource}} was last recorded as {{used_percentage}}% allocated.
{% endif %}
+{% if unmatched_code %}
+The expense code entered does not match any in the FIINE database.
+Please check it for accuracy and ensure that it is added to FIINE:
+{{expense_code}}
+{% endif %}
+
{% if nese %}
Here is a draft ticket to send to NESE:
To: help@nese.mghpcc.org
Subject: TB Globus tape setup for HU
-Service (access type) = S3 vs Globus:
-If Globus, short description for Globus Share:
-Tennant name (8 char max - Ex. acohen): {{pi.username}}
+Service (access type) = S3 vs Globus: Globus
+If Globus, short description for Globus Share: {{pi_name}}
+Tennant name (8 char max - Ex. acohen): {{pi_username}}
Size in 20TB allotments: {{quantity|div:19.48|floatformat:"0"}}
Any additional details:
Any additional administrators (if applicable):
diff --git a/container_startup.sh b/container_startup.sh
index 6e2f289de..78668d236 100644
--- a/container_startup.sh
+++ b/container_startup.sh
@@ -9,7 +9,7 @@
service redis-server start
python ./manage.py qcluster &
python ./manage.py add_scheduled_tasks
-python ./manage.py collectstatic
+python ./manage.py collectstatic --noinput
# initial_setup does not appear to work as requested.
python ./manage.py initial_setup &