From 99c3098acf42eb0c43055420eb9ef220883a056f Mon Sep 17 00:00:00 2001 From: Oliver Stolpe Date: Tue, 17 Dec 2024 17:15:05 +0100 Subject: [PATCH] WIP; removed state-dump-v2 command and api endpoint --- adminsec/tasks.py | 6 +- adminsec/urls.py | 5 - adminsec/views_api.py | 16 --- hpcaccess/users/models.py | 1 + usersec/serializers.py | 94 +---------------- usersec/tests/test_serializers.py | 73 -------------- utils/cli/hpc_access_cli/main.py | 15 --- utils/cli/hpc_access_cli/models.py | 84 +--------------- utils/cli/hpc_access_cli/states.py | 156 +---------------------------- 9 files changed, 13 insertions(+), 437 deletions(-) diff --git a/adminsec/tasks.py b/adminsec/tasks.py index cde5211..8374866 100644 --- a/adminsec/tasks.py +++ b/adminsec/tasks.py @@ -58,7 +58,8 @@ def _sync_ldap(write=False, verbose=False, ldapcon=None): first_name = userinfo.givenName last_name = userinfo.sn mail = userinfo.mail - name = userinfo.displayName + name = userinfo.cn + display_name = userinfo.displayName disabled = True if userAccountControl: @@ -82,6 +83,9 @@ def _sync_ldap(write=False, verbose=False, ldapcon=None): if name: user.name = name[0] + if display_name: + user.display_name = display_name[0] + user.is_active = not disabled if user.hpcuser_user.exists(): diff --git a/adminsec/urls.py b/adminsec/urls.py index ae3f276..61164ca 100644 --- a/adminsec/urls.py +++ b/adminsec/urls.py @@ -318,11 +318,6 @@ view=views_api.HpcProjectCreateRequestRetrieveUpdateApiView.as_view(), name="api-hpcprojectcreaterequest-retrieveupdate", ), - path( - "api/hpcaccess-status/", - view=views_api.HpcAccessStatusApiView.as_view(), - name="api-hpcaccess-status", - ), ] urlpatterns = urlpatterns_ui + urlpatterns_api diff --git a/adminsec/views_api.py b/adminsec/views_api.py index 0b73122..f931d8d 100644 --- a/adminsec/views_api.py +++ b/adminsec/views_api.py @@ -26,7 +26,6 @@ HpcUser, ) from usersec.serializers import ( - HpcAccessStatusSerializer, HpcGroupCreateRequestSerializer, HpcGroupSerializer, HpcProjectCreateRequestSerializer, @@ -218,18 +217,3 @@ def perform_update(self, serializer): raise ValidationError(errors) super().perform_update(serializer) - - -class HpcAccessStatusApiView(RetrieveAPIView): - """API view for listing all users.""" - - serializer_class = HpcAccessStatusSerializer - permission_classes = [IsAdminUser | IsHpcAdminUser] - - def get_object(self): - """Return the object to be used in the view.""" - return HpcAccessStatus( - hpc_users=HpcUser.objects.all(), - hpc_groups=HpcGroup.objects.all(), - hpc_projects=HpcProject.objects.all(), - ) diff --git a/hpcaccess/users/models.py b/hpcaccess/users/models.py index 17169b6..590c7ef 100644 --- a/hpcaccess/users/models.py +++ b/hpcaccess/users/models.py @@ -21,6 +21,7 @@ class User(AbstractUser): phone = CharField(_("Phone number of User"), blank=True, null=True, max_length=32) email = EmailField(_("email address"), blank=True, null=True) uid = IntegerField(_("UID of User"), blank=True, null=True) + display_name = CharField(_("Display Name"), blank=True, null=True, max_length=255) consented_to_terms = BooleanField( _("Terms consent status"), default=False, diff --git a/usersec/serializers.py b/usersec/serializers.py index 74bd344..711ffc6 100644 --- a/usersec/serializers.py +++ b/usersec/serializers.py @@ -45,6 +45,7 @@ class HpcUserAbstractSerializer(HpcObjectAbstractSerializer): full_name = serializers.SerializerMethodField() first_name = serializers.SerializerMethodField() last_name = serializers.SerializerMethodField() + display_name = serializers.SerializerMethodField() phone_number = serializers.SerializerMethodField() home_directory = serializers.CharField() login_shell = serializers.CharField() @@ -65,12 +66,15 @@ def get_first_name(self, obj) -> Optional[str]: def get_phone_number(self, obj) -> Optional[str]: return obj.user.phone + def get_display_name(self, obj) -> Optional[str]: + return obj.user.display_name class Meta: fields = HpcObjectAbstractSerializer.Meta.fields + [ "email", "full_name", "first_name", "last_name", + "display_name", "phone_number", "primary_group", "resources_requested", @@ -98,36 +102,6 @@ class Meta: ] -class HpcUserStatusSerializer(HpcUserAbstractSerializer, serializers.ModelSerializer): - """Serializer for HpcUser model.""" - - primary_group = serializers.SerializerMethodField() - - def get_primary_group(self, obj): - if obj.primary_group is None: - return HPC_ALUMNI_GROUP - return obj.primary_group.name - - class Meta: - model = HpcUser - fields = [ - "uid", - "email", - "full_name", - "first_name", - "last_name", - "phone_number", - "primary_group", - "resources_requested", - "status", - "description", - "username", - "expiration", - "home_directory", - "login_shell", - ] - - class HpcUserVersionSerializer(HpcUserAbstractSerializer, serializers.ModelSerializer): """Serializer for HpcUserVersion model.""" @@ -195,26 +169,6 @@ class Meta: ] -class HpcGroupStatusSerializer(HpcGroupAbstractSerializer, serializers.ModelSerializer): - """Serializer for HpcGroup model.""" - - owner = serializers.SlugRelatedField(slug_field="username", read_only=True) - - class Meta: - model = HpcUser - fields = [ - "owner", - "delegate", - "resources_requested", - "status", - "description", - "name", - "folders", - "expiration", - "gid", - ] - - class HpcProjectAbstractSerializer(HpcObjectAbstractSerializer): """Common base class for HPC project serializers.""" @@ -271,29 +225,6 @@ class Meta: ] -class HpcProjectStatusSerializer(HpcProjectAbstractSerializer, serializers.ModelSerializer): - """Serializer for HpcProject model.""" - - group = serializers.SlugRelatedField(slug_field="name", read_only=True) - delegate = serializers.SlugRelatedField(slug_field="username", read_only=True) - members = serializers.SlugRelatedField(slug_field="username", many=True, read_only=True) - - class Meta: - model = HpcUser - fields = [ - "gid", - "group", - "delegate", - "resources_requested", - "status", - "description", - "name", - "folders", - "expiration", - "members", - ] - - class HpcRequestAbstractSerializer(HpcObjectAbstractSerializer): """Common base class for HPC request serializers.""" @@ -451,20 +382,3 @@ class Meta: "primary_group", "full_name", ] - - -class HpcAccessStatusSerializer(serializers.Serializer): - """Serializer for HpcAccessStatus model.""" - - hpc_users = serializers.SerializerMethodField() - hpc_groups = serializers.SerializerMethodField() - hpc_projects = serializers.SerializerMethodField() - - def get_hpc_users(self, obj): - return HpcUserStatusSerializer(obj.hpc_users, many=True).data - - def get_hpc_groups(self, obj): - return HpcGroupStatusSerializer(obj.hpc_groups, many=True).data - - def get_hpc_projects(self, obj): - return HpcProjectStatusSerializer(obj.hpc_projects, many=True).data diff --git a/usersec/tests/test_serializers.py b/usersec/tests/test_serializers.py index 7a325c8..eec44e1 100644 --- a/usersec/tests/test_serializers.py +++ b/usersec/tests/test_serializers.py @@ -6,12 +6,9 @@ from usersec.serializers import ( HpcGroupCreateRequestSerializer, HpcGroupSerializer, - HpcGroupStatusSerializer, HpcProjectCreateRequestSerializer, HpcProjectSerializer, - HpcProjectStatusSerializer, HpcUserSerializer, - HpcUserStatusSerializer, ) from usersec.tests.factories import ( HpcGroupCreateRequestFactory, @@ -106,73 +103,3 @@ def testSerializerExisting(self): result["group"] = "group_uuid_placeholder" result["name_requested"] = "name_requested_placeholder" self.assertMatchSnapshot(result) - - -@freeze_time(FROZEN_TIME) -class TestHpcUserStatusSerializer(ResetSequenceMixin, TestCaseSnap, TestCasePlus): - def setUp(self): - super().setUp() - self.hpc_user = HpcUserFactory() - - def testSerializerExisting(self): - serializer = HpcUserStatusSerializer(self.hpc_user) - result = dict(serializer.data) - result["email"] = "email_placeholder" - result["primary_group"] = "primary_group_name_placeholder" - result["phone_number"] = "phone_number_placeholder" - result["full_name"] = "name_placeholder" - result["first_name"] = "first_name_placeholder" - result["last_name"] = "last_name_placeholder" - self.assertMatchSnapshot(result) - - -@freeze_time(FROZEN_TIME) -class TestHpcGroupStatusSerializer(ResetSequenceMixin, TestCaseSnap, TestCasePlus): - def setUp(self): - super().setUp() - self.hpc_group = HpcGroupFactory() - - def testSerializerExisting(self): - serializer = HpcGroupStatusSerializer(self.hpc_group) - result = dict(serializer.data) - self.assertMatchSnapshot(result) - - -@freeze_time(FROZEN_TIME) -class TestHpcProjectStatusSerializer(ResetSequenceMixin, TestCaseSnap, TestCasePlus): - def setUp(self): - super().setUp() - self.hpc_project = HpcProjectFactory() - - def testSerializerExisting(self): - serializer = HpcProjectStatusSerializer(self.hpc_project) - result = dict(serializer.data) - result["group"] = "group_name_placeholder" - self.assertMatchSnapshot(result) - - -# TODO somehow the tests fail, but the serializer and API actually work -# @freeze_time(FROZEN_TIME) -# class TestHcpaccessStatus(ResetSequenceMixin, TestCaseSnap, TestCasePlus): -# def setUp(self): -# super().setUp() -# hpc_group = HpcGroupFactory() -# HpcProjectFactory(group=hpc_group) -# HpcUserFactory(primary_group=hpc_group) -# self.hpc_access_status = HpcAccessStatus( -# hpc_users=HpcUser.objects.all(), -# hpc_groups=HpcGroup.objects.all(), -# hpc_projects=HpcProject.objects.all(), -# ) - -# def testSerializerExisting(self): -# serializer = HpcAccessStatusSerializer(self.hpc_access_status) -# result = dict(serializer.data) -# result["hpc_users"][0]["email"] = "email_placeholder" -# result["hpc_users"][0]["primary_group"] = "primary_group_name_placeholder" -# result["hpc_users"][0]["phone_number"] = "phone_number_placeholder" -# result["hpc_users"][0]["full_name"] = "name_placeholder" -# result["hpc_users"][0]["first_name"] = "first_name_placeholder" -# result["hpc_users"][0]["last_name"] = "last_name_placeholder" -# result["hpc_users"][0]["uid"] = 2000 -# self.assertMatchSnapshot(result) diff --git a/utils/cli/hpc_access_cli/main.py b/utils/cli/hpc_access_cli/main.py index a26cbf1..4097a72 100644 --- a/utils/cli/hpc_access_cli/main.py +++ b/utils/cli/hpc_access_cli/main.py @@ -16,7 +16,6 @@ TargetStateBuilder, TargetStateComparison, convert_to_hpcaccess_state, - convert_to_hpcaccess_state_v2, deploy_hpcaccess_state, fs_validation, gather_hpcaccess_state, @@ -80,20 +79,6 @@ def dump_data( console_out.print_json(data=hpcaccess_state.model_dump(mode="json")) -@app.command("state-dump-v2") -def dump_data_v2( - config_path: Annotated[ - str, typer.Option(..., help="path to configuration file") - ] = "/etc/hpc-access-cli/config.json", -): - """dump system state as hpc-access state""" - settings = load_settings(config_path) - console_err.print_json(data=settings.model_dump(mode="json")) - system_state = gather_system_state(settings) - hpcaccess_state = convert_to_hpcaccess_state_v2(system_state) - console_out.print_json(data=hpcaccess_state.model_dump(mode="json")) - - @app.command("state-sync") def sync_data( config_path: Annotated[ diff --git a/utils/cli/hpc_access_cli/models.py b/utils/cli/hpc_access_cli/models.py index 0b44003..b12cf40 100644 --- a/utils/cli/hpc_access_cli/models.py +++ b/utils/cli/hpc_access_cli/models.py @@ -255,6 +255,8 @@ class HpcUser(BaseModel): first_name: Optional[str] #: The last name of the user. last_name: Optional[str] + #: The name to display. + display_name: Optional[str] #: The office phone number of the user. phone_number: Optional[str] #: The requested resources. @@ -277,37 +279,6 @@ class HpcUser(BaseModel): current_version: int -class HpcUserV2(BaseModel): - """A user as read from the hpc-access API.""" - - #: The UUID of the primary ``HpcGroup``. - primary_group: Optional[str] - #: Description of the record. - description: Optional[str] - #: The user's email address. - email: Optional[str] - #: The full name of the user. - full_name: str - #: The first name fo the user. - first_name: Optional[str] - #: The last name of the user. - last_name: Optional[str] - #: The office phone number of the user. - phone_number: Optional[str] - #: The requested resources. - resources_requested: Optional[ResourceDataUser] - #: The status of the record. - status: Status - #: The POSIX UID of the user. - uid: int - #: The username of the record. - username: str - #: The home directory. - home_directory: str - #: The login shell - login_shell: str - - class HpcGroup(BaseModel): """A group as read from the hpc-access API.""" @@ -337,26 +308,6 @@ class HpcGroup(BaseModel): current_version: int -class HpcGroupV2(BaseModel): - """A group as read from the hpc-access API.""" - - #: The owning ``HpcUser``. - owner: str - #: Description of the record. - description: Optional[str] - #: The delegate. - delegate: Optional[str] - #: The requested resources. - resources_requested: Optional[ResourceData] - #: The status of the record. - status: Status - #: The POSIX GID of the corresponding Unix group. - gid: Optional[int] - #: The name of the record. - name: str - #: The folders of the group. - folders: GroupFolders - class HpcProject(BaseModel): """A project as read from the hpc-access API.""" @@ -389,29 +340,6 @@ class HpcProject(BaseModel): members: List[UUID] -class HpcProjectV2(BaseModel): - """A project as read from the hpc-access API.""" - - #: The owning ``HpcGroup``, owner of group is owner of project. - group: Optional[str] - #: Description of the record. - description: Optional[str] - #: The delegate for the project. - delegate: Optional[str] - #: The requested resources. - resources_requested: Optional[ResourceData] - #: The status of the record. - status: Status - #: The POSIX GID of the corresponding Unix group. - gid: Optional[int] - #: The name of the record. - name: str - #: The folders of the group. - folders: GroupFolders - #: The project's member user UUIDs. - members: List[str] - - class SystemState(BaseModel): """System state retrieved from LDAP and file system.""" @@ -431,14 +359,6 @@ class HpcaccessState(BaseModel): hpc_projects: Dict[UUID, HpcProject] -class HpcaccessStateV2(BaseModel): - """State as loaded from hpc-access.""" - - hpc_users: List[HpcUserV2] - hpc_groups: List[HpcGroupV2] - hpc_projects: List[HpcProjectV2] - - @enum.unique class StateOperation(enum.Enum): """Operation to perform on the state.""" diff --git a/utils/cli/hpc_access_cli/states.py b/utils/cli/hpc_access_cli/states.py index 80a84cb..322834a 100644 --- a/utils/cli/hpc_access_cli/states.py +++ b/utils/cli/hpc_access_cli/states.py @@ -40,13 +40,9 @@ # Gecos, GroupFolders, HpcaccessState, - HpcaccessStateV2, HpcGroup, - HpcGroupV2, HpcProject, - HpcProjectV2, HpcUser, - HpcUserV2, LdapGroup, LdapGroupOp, LdapUser, @@ -514,7 +510,7 @@ def build_hpcuser(u: LdapUser, quotas: Dict[str, str]) -> HpcUser: uuid=user_uuids[u.uid], primary_group=primary_group, description=None, - full_name=u.cn, + full_name=u.display_name, first_name=u.given_name, last_name=u.sn, email=u.mail, @@ -632,156 +628,6 @@ def build_hpcproject(p: LdapGroup, quotas: Dict[str, str]) -> Optional[HpcProjec ) -def convert_to_hpcaccess_state_v2(system_state: SystemState) -> HpcaccessStateV2: - """Convert hpc-access to system state. - - Note that this will make up the UUIDs. - """ - # create UUID mapping from user/groupnames - user_by_uid = {u.uid: u for u in system_state.ldap_users.values()} - user_by_dn = {u.dn: u for u in system_state.ldap_users.values()} - group_by_name = {strip_prefix(g.cn): g for g in system_state.ldap_groups.values()} - group_by_gid_number = {g.gid_number: g for g in system_state.ldap_groups.values()} - group_by_owner_dn: Dict[str, LdapGroup] = {} - for g in system_state.ldap_groups.values(): - if g.owner_dn: - group_by_owner_dn[user_by_dn[g.owner_dn].dn] = g - user_quotas: Dict[str, ResourceDataUser] = {} - group_quotas: Dict[str, ResourceData] = {} - for fs_data in system_state.fs_directories.values(): - try: - entity, name, resource = fs_validation(fs_data) - except ValueError as e: - console_err.log(f"WARNING: {e}") - continue - - quota_bytes = fs_data.quota_bytes if fs_data.quota_bytes is not None else 0 - - if entity == ENTITY_USERS: - if name not in user_by_uid: - console_err.log(f"WARNING: user {name} not found") - continue - if name not in user_quotas: - user_quotas[name] = {} - user_quotas[name][resource] = quota_bytes / 1024**3 - elif entity in (ENTITY_GROUPS, ENTITY_PROJECTS): - if name not in group_by_name: - console_err.log(f"WARNING: group {name} not found") - continue - if name not in group_quotas: - group_quotas[name] = {} - group_quotas[name][resource] = quota_bytes / 1024**4 - - def build_hpcuser(u: LdapUser, quotas: Dict[str, str]) -> HpcUserV2: - if u.login_shell != LOGIN_SHELL_DISABLED: - status = Status.ACTIVE - else: - status = Status.EXPIRED - if u.gid_number and u.gid_number in group_by_gid_number: - primary_group = group_by_gid_number[u.gid_number].cn - else: - primary_group = None - if not primary_group.startswith(POSIX_AG_PREFIX) and not primary_group == "hpc-alumnis": - console_err.log(f"User belongs to group that is not a group ({primary_group}, {u.uid})") - return HpcUserV2( - primary_group=strip_prefix(primary_group, prefix=POSIX_AG_PREFIX), - description=None, - full_name=u.cn, - first_name=u.given_name, - last_name=u.sn, - email=u.mail, - phone_number=u.telephone_number, - resources_requested=ResourceDataUser(**quotas), - status=status, - uid=u.uid_number, - username=u.uid, - home_directory=u.home_directory, - login_shell=u.login_shell, - ) - - def build_hpcgroup(g: LdapGroup, quotas: Dict[str, str]) -> Optional[HpcGroupV2]: - name = strip_prefix(g.cn, POSIX_AG_PREFIX) - if not g.owner_dn: - console_err.log(f"no owner DN for {g.cn}, skipping") - return - return HpcGroupV2( - name=name, - description=g.description, - owner=user_by_dn[g.owner_dn].uid, - delegate=user_by_dn[g.delegate_dns[0]].uid if g.delegate_dns else None, - resources_requested=ResourceData(**quotas), - status=Status.ACTIVE, - gid=g.gid_number, - folders=GroupFolders( - tier1_work=f"{BASE_PATH_TIER1}/work/groups/{name}", - tier1_scratch=f"{BASE_PATH_TIER1}/scratch/groups/{name}", - tier2_mirrored=f"{BASE_PATH_TIER2}/mirrored/groups/{name}", - tier2_unmirrored=f"{BASE_PATH_TIER2}/unmirrored/groups/{name}", - ), - ) - - def build_hpcproject(p: LdapGroup, quotas: Dict[str, str]) -> Optional[HpcProjectV2]: - name = strip_prefix(p.cn, POSIX_PROJECT_PREFIX) - if not p.owner_dn: - console_err.log(f"no owner DN for {p.cn}, skipping") - return - members = [] - for uid in p.member_uids: - uid = uid.strip() - user = user_by_uid[uid] - members.append(user.uid) - gid_number = user_by_dn[p.owner_dn].gid_number - if not gid_number: - group = None - else: - group = group_by_gid_number[gid_number].cn - return HpcProjectV2( - name=name, - description=g.description, - group=group, - delegate=user_by_dn[p.delegate_dns[0]].uid if p.delegate_dns else None, - resources_requested=ResourceData(**quotas), - status=Status.ACTIVE, - gid=p.gid_number, - folders=GroupFolders( - tier1_work=f"{BASE_PATH_TIER1}/work/projects/{name}", - tier1_scratch=f"{BASE_PATH_TIER1}/scratch/projects/{name}", - tier2_mirrored=f"{BASE_PATH_TIER2}/mirrored/projects/{name}", - tier2_unmirrored=f"{BASE_PATH_TIER2}/unmirrored/projects/{name}", - ), - members=members, - ) - - # construct the resulting state - hpc_users = [] - hpc_groups = [] - hpc_projects = [] - - for u in system_state.ldap_users.values(): - hpc_user = build_hpcuser(u, user_quotas.get(u.uid, {})) - hpc_users.append(hpc_user) - - for g in system_state.ldap_groups.values(): - if g.cn.startswith(POSIX_AG_PREFIX): - hpc_group = build_hpcgroup( - g, group_quotas.get(strip_prefix(g.cn, prefix=POSIX_AG_PREFIX), {}) - ) - if hpc_group: - hpc_groups.append(hpc_group) - elif g.cn.startswith(POSIX_PROJECT_PREFIX): - hpc_project = build_hpcproject( - g, group_quotas.get(strip_prefix(g.cn, prefix=POSIX_PROJECT_PREFIX), {}) - ) - if hpc_project: - hpc_projects.append(hpc_project) - - return HpcaccessStateV2( - hpc_users=hpc_users, - hpc_groups=hpc_groups, - hpc_projects=hpc_projects, - ) - - class TargetStateComparison: """Helper class that compares two system states.