fix merge conflict

Dockerfile

@@ -3,7 +3,7 @@
 # to build for a development environment, run the following command:
 # docker build --build-arg build_env=dev -t coldfront --ssh default . --network=host
 
-FROM python:3.8
+FROM python:3.10
 
 ARG build_env=production
 ENV BUILD_ENV=$build_env
@@ -29,9 +29,9 @@ RUN pip install --upgrade pip && \
 
 COPY . .
 
-RUN if [ "${BUILD_ENV}" = "dev" ]; then pip install django-redis reportlab==3.6.6 django-debug-toolbar; fi
+RUN if [ "${BUILD_ENV}" = "dev" ]; then pip install django-redis django-debug-toolbar; fi
 
-RUN pip install ldap3 django_auth_ldap django-author==1.0.2 django-prometheus gunicorn
+RUN pip install django-prometheus gunicorn
 
 ENV PYTHONPATH /usr/src/app:/usr/src/app/ifxreport:/usr/src/app/ifxbilling:/usr/src/app/fiine.client:/usr/src/app/ifxurls:/usr/src/app/nanites.client:/usr/src/app/ifxuser:/usr/src/app/ifxmail.client:/usr/src/app/ifxec
 

coldfront/config/base.py

@@ -59,6 +59,7 @@
 sys.modules['fontawesome_free'] = __import__('fontawesome-free')
 INSTALLED_APPS += [
     'crispy_forms',
+    'crispy_bootstrap4',
     'sslserver',
     'django_q',
     'simple_history',

coldfront/config/core.py

@@ -30,7 +30,7 @@
 ALLOCATION_DEFAULT_ALLOCATION_LENGTH = ENV.int('ALLOCATION_DEFAULT_ALLOCATION_LENGTH', default=365)
 
 # Categorization of allocation statuses
-PENDING_ALLOCATION_STATUSES = ['New', 'In Progress', 'On Hold']
+PENDING_ALLOCATION_STATUSES = ['New', 'In Progress', 'On Hold', 'Pending Activation']
 ACTIVE_ALLOCATION_STATUSES = ['Active']
 PENDING_ACTIVE_ALLOCATION_STATUSES = PENDING_ALLOCATION_STATUSES + ACTIVE_ALLOCATION_STATUSES
 INACTIVE_ALLOCATION_STATUSES = ['Denied', 'Expired', 'Inactive', 'Pending Deactivation']

coldfront/core/allocation/management/commands/add_allocation_defaults.py

@@ -28,6 +28,7 @@ def handle(self, *args, **options):
             ('Pending Deactivation', 'Allocation is slated for deactivation'),
             ('In Progress', 'Allocation request is being processed'),
             ('On Hold', 'Allocation request is on hold'),
+            ('Pending Activation', 'Allocation is in the process of being set up and not yet ready for use/billing'),
             # UBCCR Defaults
             # 'Paid', 'Payment Pending', 'Payment Requested',
             # 'Payment Declined', 'Revoked', 'Renewal Requested', 'Unpaid',

coldfront/core/allocation/templates/allocation/allocation_detail.html

@@ -82,7 +82,7 @@ <h3><i class="fas fa-list" aria-hidden="true"></i> Allocation Information</h3>
             <td>
               {% if allocation.get_resources_as_list %}
                 {% for resource in allocation.get_resources_as_list %}
-                  {% if request.user.is_superuser and allocation.status.name == "New" and "Tier" in allocation.get_resources_as_string %}
+                  {% if request.user.is_superuser and allocation.status.name in "New,In Progress,On Hold" and "Tier" in allocation.get_resources_as_string %}
                     {{ form.resource }} <a color="red">Please choose a resource.</a>
                   {% else %}
                     <a href="{% url 'resource-detail' resource.pk %}">{{ resource }}</a>
@@ -296,7 +296,7 @@ <h3><i class="fas fa-list" aria-hidden="true"></i> Allocation Information</h3>
       </div>
 
       <!-- New Allocation Action Form -->
-      {% if request.user.is_superuser or request.user.is_staff and allocation.status.name == 'New' %}
+      {% if request.user.is_superuser or request.user.is_staff and allocation.status.name in 'New,In Progress,On Hold' %}
         <div class="container mb-3 mt-3">
           <p class="text-justify"><i>
             To approve the request, select a resource volume in the field above and fill out the form below before clicking "Approve". To deny the request, simply click "Deny".

coldfront/core/project/templates/project/project_detail.html

@@ -234,20 +234,24 @@ <h3 class="d-inline"><i class="fas fa-list" aria-hidden="true"></i> Allocation H
         <tbody>
           {% for allocation, record in allocation_history_records %}
           <tr style="background-color:#FFFFFF">
-            <td>{{ allocation.get_parent_resource.name }}</td>
+            <td>
+              <a href="{% url 'allocation-detail' allocation.pk %}">
+		    {{ allocation.get_parent_resource.name }}
+              </a>
+	    </td>
             <td>{{ allocation.get_parent_resource.resource_type.name }}</td>
             <td>{{ allocation.path }}</td>
             <td>{{ allocation.allocationuser_set.count }}</td>
             <td>{{ allocation.size|floatformat:1 }}</td>
 
             {% if allocation.status.name in 'New,In Progress,On Hold' %}
-              <td class="text-info">{{ Requested }}</td>
+              <td class="text-success">Requested</td>
             {% elif allocation.status.name == 'Active' %}
               <td class="text-success">{{ allocation.status.name }}</td>
             {% elif allocation.status.name in 'Pending Deactivation,Denied,Inactive' %}
               <td class="text-danger">{{ allocation.status.name }}</td>
             {% else %}
-              <td class="text-info">{{ allocation.status.name }}</td>
+              <td class="text-success">{{ allocation.status.name }}</td>
             {% endif %}
             <td>
               {% if allocation.status.name not in 'New,Denied' %}
@@ -297,8 +301,10 @@ <h3 class="d-inline" id="users"><i class="fas fa-users" aria-hidden="true"></i>
     <div class="float-right">
       {% if project.status.name != 'Archived' and is_allowed_to_update_project %}
         <a class="btn btn-primary" href="{{mailto}}" role="button"><i class="far fa-envelope" aria-hidden="true"></i> Email Project Users</a>
-        <a class="btn btn-success" href="{% url 'project-add-users-search' project.id %}" role="button"><i class="fas fa-user-plus" aria-hidden="true"></i> Add Users</a>
-        <a class="btn btn-danger" href="{% url 'project-remove-users' project.id %}" role="button"><i class="fas fa-user-times" aria-hidden="true"></i> Remove Users</a>
+        {% if request.user.is_superuser or request.user == project.pi %}
+          <a class="btn btn-success" href="{% url 'project-add-users-search' project.id %}" role="button"><i class="fas fa-user-plus" aria-hidden="true"></i> Add Users</a>
+          <a class="btn btn-danger" href="{% url 'project-remove-users' project.id %}" role="button"><i class="fas fa-user-times" aria-hidden="true"></i> Remove Users</a>
+	{% endif %}
       {% endif %}
     </div>
   </div>

coldfront/core/project/test_views.py

@@ -144,7 +144,6 @@ def test_projectdetail_addnotification_button_visibility(self):
         utils.page_does_not_contain_for_user(self, self.project_user, self.url, 'Add Notification') # non-manager user cannot see add notification button
 
 
-
 class ProjectCreateTest(ProjectViewTestBase):
     """Tests for project create view"""
 
@@ -164,7 +163,6 @@ def test_project_access(self):
         utils.test_user_cannot_access(self, self.nonproject_user, self.url)
 
 
-
 class ProjectAttributeCreateTest(ProjectViewTestBase):
     """Tests for project attribute create view"""
 
@@ -214,7 +212,6 @@ def test_project_attribute_create_post_required_values(self):
         })
         self.assertFormError(response, 'form', 'value', 'This field is required.')
 
-
     def test_project_attribute_create_value_type_match(self):
         """ProjectAttributeCreate correctly flags value-type mismatch"""
 
@@ -226,8 +223,7 @@ def test_project_attribute_create_value_type_match(self):
             'value': True,
             'project': self.project.pk
         })
-        self.assertFormError(response, 'form', '', 'Invalid Value True. Value must be an int.')
-
+        self.assertContains(response, 'Invalid Value True. Value must be an int.')
 
 
 class ProjectAttributeUpdateTest(ProjectViewTestBase):
@@ -241,7 +237,6 @@ def setUpTestData(cls):
         )
         cls.url = f'/project/{cls.project.pk}/project-attribute-update/{cls.projectattribute.pk}'
 
-
     def test_project_attribute_update_access(self):
         """Test access to project attribute update page"""
         self.project_access_tstbase(self.url)
@@ -251,7 +246,6 @@ def test_project_attribute_update_access(self):
         utils.test_user_cannot_access(self, self.nonproject_user, self.url)
 
 
-
 class ProjectAttributeDeleteTest(ProjectViewTestBase):
     """Tests for ProjectAttributeDeleteView"""
 
@@ -262,7 +256,6 @@ def setUpTestData(cls):
         cls.projectattribute = ProjectAttributeFactory(value=36238, proj_attr_type=cls.projectattributetype, project=cls.project)
         cls.url = f'/project/{cls.project.pk}/project-attribute-delete/'
 
-
     def test_project_attribute_delete_access(self):
         """test access to project attribute delete page"""
         # logged-out user gets redirected, admin can access delete page
@@ -274,7 +267,6 @@ def test_project_attribute_delete_access(self):
         utils.test_user_cannot_access(self, self.nonproject_user, self.url)
 
 
-
 class ProjectListViewTest(ProjectViewTestBase):
     """Tests for projectlist view"""
 
@@ -337,7 +329,6 @@ def test_project_list_search(self):
         response = utils.login_and_get_page(self.client, self.admin_user, url)
         self.assertEqual(len(response.context['object_list']), 1)
 
-
     def test_project_list_search_pagination(self):
         """confirm that navigation to next page of search works as expected"""
         url = self.url + '?show_all_projects=on'
@@ -357,7 +348,6 @@ def test_projectremoveusersview_access(self):
         self.project_access_tstbase(self.url)
 
 
-
 class ProjectUpdateViewTest(ProjectViewTestBase):
     """Tests for ProjectUpdateView"""
     def setUp(self):
@@ -402,7 +392,6 @@ def test_projectnotecreateview_access(self):
         self.project_access_tstbase(self.url)
 
 
-
 class ProjectAddUsersSearchView(ProjectViewTestBase):
     """Tests for ProjectAddUsersSearchView"""
     def setUp(self):
@@ -414,7 +403,6 @@ def test_projectadduserssearchview_access(self):
         self.project_access_tstbase(self.url)
 
 
-
 class ProjectUserDetailViewTest(ProjectViewTestBase):
     """Tests for ProjectUserDetailView"""
     def setUp(self):

coldfront/core/project/views.py

@@ -460,7 +460,8 @@ class ProjectAddUsersSearchView(LoginRequiredMixin, UserPassesTestMixin, Templat
     def test_func(self):
         """UserPassesTestMixin Tests"""
         project_obj = get_object_or_404(Project, pk=self.kwargs.get('pk'))
-        if project_obj.has_perm(self.request.user, ProjectPermission.UPDATE):
+        # if project_obj.has_perm(self.request.user, ProjectPermission.UPDATE):
+        if self.request.user.is_superuser or self.request.user == project.pi:
             return True
         return False
 
@@ -488,7 +489,8 @@ class ProjectAddUsersSearchResultsView(
     def test_func(self):
         """UserPassesTestMixin Tests"""
         project_obj = get_object_or_404(Project, pk=self.kwargs.get('pk'))
-        if project_obj.has_perm(self.request.user, ProjectPermission.UPDATE):
+        # if project_obj.has_perm(self.request.user, ProjectPermission.UPDATE):
+        if self.request.user.is_superuser or self.request.user == project.pi:
             return True
         return False
 
@@ -560,7 +562,8 @@ class ProjectAddUsersView(LoginRequiredMixin, UserPassesTestMixin, View):
     def test_func(self):
         """UserPassesTestMixin Tests"""
         project_obj = get_object_or_404(Project, pk=self.kwargs.get('pk'))
-        if project_obj.has_perm(self.request.user, ProjectPermission.UPDATE):
+        # if project_obj.has_perm(self.request.user, ProjectPermission.UPDATE):
+        if self.request.user.is_superuser or self.request.user == project.pi:
             return True
         err = 'You do not have permission to add users to the project.'
         messages.error(self.request, err)
@@ -616,6 +619,8 @@ def post(self, request, *args, **kwargs):
             allocation_form_data = allocation_form.cleaned_data['allocation']
             if '__select_all__' in allocation_form_data:
                 allocation_form_data.remove('__select_all__')
+            errors = []
+            successes = []
             for form in formset:
                 user_form_data = form.cleaned_data
                 if user_form_data['selected']:
@@ -636,7 +641,7 @@ def post(self, request, *args, **kwargs):
                     except Exception as e:
                         error = f"Could not locate user for {user_username}: {e}"
                         logger.error('P665: %s', error)
-                        messages.error(request, error)
+                        errors.append(error)
                         continue
 
                     role_choice = user_form_data.get('role')
@@ -653,13 +658,14 @@ def post(self, request, *args, **kwargs):
                                 project_obj.title,
                             )
                         except Exception as e:
-                            error = f"Could not add user {user_obj} to AD Group for {project_obj.title}: {e}"
+                            error = f"Could not add user {user_obj} to AD Group for {project_obj.title}: {e}\nPlease contact Coldfront administration for further assistance."
                             logger.error(
                                 "P685: user %s could not add AD user of %s to AD Group of %s: %s",
                                 self.request.user, user_obj, project_obj.title, e
                             )
-                            messages.error(request, error)
+                            errors.append(error)
                             continue
+                        successes.append(f"User {user_obj} added to AD Group for {project_obj.title}")
 
                     # Is the user already in the project?
                     project_obj.projectuser_set.update_or_create(
@@ -690,6 +696,9 @@ def post(self, request, *args, **kwargs):
                             sender=self.__class__,
                             allocation_user_pk=allocation_user_obj.pk,
                         )
+            if errors:
+                for error in errors:
+                    messages.error(request, error)
 
             messages.success(request, f'Added {added_users_count} users to project.')
         else:
@@ -710,7 +719,8 @@ class ProjectRemoveUsersView(LoginRequiredMixin, UserPassesTestMixin, TemplateVi
     def test_func(self):
         """UserPassesTestMixin Tests"""
         project_obj = get_object_or_404(Project, pk=self.kwargs.get('pk'))
-        if project_obj.has_perm(self.request.user, ProjectPermission.UPDATE):
+        # if project_obj.has_perm(self.request.user, ProjectPermission.UPDATE):
+        if self.request.user.is_superuser or self.request.user == project.pi:
             return True
         return False
 

coldfront/core/utils/fasrc.py

@@ -69,11 +69,13 @@ def select_one_project_allocation(project_obj, resource_obj, dirpath=None):
     allocation_query = project_obj.allocation_set.filter(**filter_vals)
     if allocation_query.count() == 1:
         allocation_obj = allocation_query.first()
+        if allocation_obj.path and dirpath and allocation_obj.path not in dirpath and dirpath not in allocation_obj.path:
+            return None
     elif allocation_query.count() < 1:
         allocation_obj = None
     elif allocation_query.count() > 1:
         allocation_obj = next((a for a in allocation_query if a.path.lower() in dirpath.lower()),
-                                'MultiAllocationError')
+                                None)
     return allocation_obj
 
 
@@ -119,7 +121,6 @@ def get_resource_rate(resource):
     price = convert_size_fmt(rate_obj.price, 'TB', source_unit=rate_obj.units)
     return round(price/100, 2)
 
-
 def id_present_missing_resources(resourceserver_list):
     """
     Collect all Resource entries with resources in param resourceserver_list;

coldfront/plugins/fasrc/management/commands/id_import_new_allocations.py

@@ -83,6 +83,7 @@ def handle(self, *args, **options):
                     and lab_path in i['path'] and i['group_name'] == lab_name
                 ]
                 if not lab_usage_entries:
+                    logger.info("No starfish usage data found for", lab_name, lab_server, lab_path)
                     continue
 
                 allocation, created = project.allocation_set.get_or_create(

coldfront/plugins/fasrc/utils.py

@@ -73,7 +73,7 @@ def produce_query_statement(self, vol_type, volumes=None):
                 'fs_path': 'LogicalVolume',
                 'path_replace': '/dev/data/',
                 'usedgb': 'UsedGB',
-                'sizebytes': 'SizeGB * 1073741824 * .931',
+                'sizebytes': 'SizeGB * 1073741824',
                 'usedbytes': 'UsedGB * 1073741824',
                 'server_replace': '.rc.fas.harvard.edu',
                 'unique': 'datetime(e.DotsLVSUpdateDate) as update_date, \
@@ -132,17 +132,15 @@ def _standardize_attquery(self):
     def _standardize_nesefile(self):
         datafile = 'nese_data/pools'
         header_file = 'nese_data/pools.header'
-        with open('nese_data/groupkey') as groupkey_file:
+        with open('nese_data/local_groupkey') as groupkey_file:
             translator = dict((
                 kv.split('=') for kv in (l.strip('\n') for l in groupkey_file)
             ))
         headers_df = pd.read_csv(header_file, header=0, delim_whitespace=True)
         headers = headers_df.columns.values.tolist()
         data = pd.read_csv(datafile, names=headers, delim_whitespace=True)
         data = data.loc[data['pool'].str.contains('1')]
-        for k, v in translator.items():
-            data['pool'] = data['pool'].str.replace(k, v)
-        data['lab'] = data['pool'].str.replace('1', '')
+        data['lab'] = data['pool'].str.replace('1', '').str.replace('hugl', '').str.replace('hus3', '')
         data['server'] = 'nesetape'
         data['storage_type'] = 'tape'
         data['byte_allocation'] = data['mib_capacity'] * 1048576
@@ -155,6 +153,11 @@ def _standardize_nesefile(self):
             'byte_usage', 'tb_allocation', 'tb_usage', 'fs_path',
         ]]
         nesedict = data.to_dict(orient='records')
+        for d in nesedict:
+            if translator.get(d['lab']):
+                d['lab'] = translator[d['lab']]
+            else:
+                d['lab'] = d['lab']+'_lab'
         return nesedict
 
 
@@ -244,7 +247,7 @@ def pair_allocations_data(project, quota_dicts):
     """pair allocations with usage dicts"""
     logger = logging.getLogger('coldfront.import_quotas')
     unpaired_allocs = project.allocation_set.filter(
-        status__name='Active',
+        status__name__in=['Active','Pending Deactivation'],
         resources__resource_type__name='Storage'
     )
     paired_allocs = {}
@@ -270,7 +273,7 @@ def pair_allocations_data(project, quota_dicts):
             log_message = f'Resource-based match: {allocation}, {dicts[0]}'
             paired_allocs = matched_dict_processing(allocation, dicts, paired_allocs, log_message)
     unpaired_allocs = [
-        a for a in unpaired_allocs if a not in paired_allocs
+        a for a in unpaired_allocs if a not in paired_allocs and a.status.name == 'Active'
     ]
     unpaired_dicts = [d for d in unpaired_dicts if d not in paired_allocs.values()]
     if unpaired_dicts or unpaired_allocs:

coldfront/plugins/ldap/management/commands/add_projects.py

@@ -21,7 +21,7 @@ def handle(self, *args, **kwargs):
         project_csv = 'local_data/ready_to_add/add_projects.csv'
         projects_list = pd.read_csv(project_csv)
         added_projects, errs = import_projects_projectusers(projects_list.title.to_list())
-        add_later = errs['no_pi'] + errs['not_found'] + errs['no_fos']
+        add_later = errs['no_pi'] + [errs['not_found']] + errs['no_fos']
         projects_to_add = projects_list.loc[projects_list['title'].isin(add_later)].copy()
         try:
             projects_to_add['first_attempt'] = projects_to_add['first_attempt'].fillna(datetime.now())