eduid_user_cleaner skatteverket worker

.vscode/settings.json

@@ -18,11 +18,13 @@
     "cSpell.words": [
         "accr",
         "amapi",
+        "amdb",
         "authlib",
         "authns",
         "baar",
         "backchannel",
         "backupdir",
+        "bson",
         "checkpw",
         "chpass",
         "csrft",
@@ -33,36 +35,48 @@
         "eppn",
         "fastapi",
         "Freja",
+        "gnap",
         "hashname",
         "hashpw",
         "httpx",
         "hubba",
+        "IDNIN",
         "jsonify",
         "jwks",
         "klass",
+        "ladok",
         "lundberg",
         "masv",
         "mischttp",
         "nameid",
         "navet",
+        "nins",
         "NORDUnet",
         "oidc",
+        "orcid",
+        "prid",
         "proofings",
         "proquint",
         "PWAUTH",
+        "pydantic",
         "pysaml2",
         "reauthn",
         "REFEDS",
         "replicaset",
+        "respx",
         "satosa",
         "scim",
         "SCIMAPI",
         "sess",
+        "SIGTERM",
         "skatteverket",
         "Svenska",
         "svipe",
         "svipeid",
         "swamid",
+        "swedenconnect",
+        "TELE",
+        "teleadress",
         "Testsson",
         "timeutil",
         "Unmarshal",

requirements/test_requirements.txt

@@ -2085,7 +2085,6 @@ zxcvbn==4.4.28 \
     # via
     #   -c main.txt
     #   -r main.in
-
 # The following packages are considered to be unsafe in a requirements file:
 pip==23.2.1 \
     --hash=sha256:7ccf472345f20d35bdc9d1841ff5f313260c2c33fe417f48c30ac46cccabf5be \

src/eduid/common/clients/amapi_client/amapi_client.py

@@ -20,15 +20,15 @@
 
 
 class AMAPIClient(GNAPClient):
-    def __init__(self, amapi_url: str, app, auth_data=GNAPClientAuthData, **kwargs):
-        super().__init__(auth_data=auth_data, app=app, **kwargs)
+    def __init__(self, amapi_url: str, auth_data=GNAPClientAuthData, verify_tls: bool = True, **kwargs):
+        super().__init__(auth_data=auth_data, verify=verify_tls, **kwargs)
         self.amapi_url = amapi_url
 
     def _users_base_url(self) -> str:
         return urlappend(self.amapi_url, "users")
 
     def _put(self, base_path: str, user: str, endpoint: str, body: Any) -> httpx.Response:
-        return self.put(urlappend(base_path, f"{user}/{endpoint}"), json=body.json())
+        return self.put(url=urlappend(base_path, f"{user}/{endpoint}"), content=body.json())
 
     def update_user_email(self, user: str, body: UserUpdateEmailRequest) -> UserUpdateResponse:
         ret = self._put(base_path=self._users_base_url(), user=user, endpoint="email", body=body)

src/eduid/common/clients/amapi_client/testing.py

@@ -1,19 +1,92 @@
-from typing import Optional
+from typing import Mapping, Optional
 
 import respx
+from httpx import Request, Response
 
 from eduid.common.clients.gnap_client.testing import MockedSyncAuthAPIMixin
+from eduid.common.misc.timeutil import utc_now
+from eduid.common.models.amapi_user import (
+    UserUpdateEmailRequest,
+    UserUpdateLanguageRequest,
+    UserUpdateNameRequest,
+    UserUpdatePhoneRequest,
+)
+from eduid.userdb.mail import MailAddressList
+from eduid.userdb.phone import PhoneNumberList
+from eduid.userdb.user import User
+from eduid.userdb.userdb import AmDB
 
 
 class MockedAMAPIMixin(MockedSyncAuthAPIMixin):
-    def start_mock_amapi(self, access_token_value: Optional[str] = None):
+    def start_mock_amapi(self, central_user_db: Optional[AmDB] = None, access_token_value: Optional[str] = None):
         self.start_mock_auth_api(access_token_value=access_token_value)
+        self.central_user_db = central_user_db
+        mocked_users = respx.mock(base_url="http://localhost/amapi", assert_all_called=False)
 
-        self.mocked_users = respx.mock(base_url="http://localhost", assert_all_called=False)
-        put_users_name_route = self.mocked_users.put(
-            url="/users/hubba-bubba/name",
-            name="put_users_name_request",
+        mocked_users.put(url="/users/hubba-bubba/name", name="200_put_name").mock(
+            side_effect=self._side_effect_update_name
         )
-        put_users_name_route.pass_through()
-        self.mocked_users.start()
-        self.addCleanup(self.mocked_users.stop)  # type: ignore
+        mocked_users.put(url="/users/hubba-bubba/email", name="200_put_email").mock(
+            side_effect=self._side_effect_update_email
+        )
+        mocked_users.put(url="/users/hubba-bubba/language", name="200_put_language").mock(
+            side_effect=self._side_effect_update_language
+        )
+
+        mocked_users.start()
+        self.addCleanup(mocked_users.stop)  # type: ignore
+
+    def _side_effect_update(self, func_name: str) -> User:
+        if self.central_user_db is None:
+            raise ValueError(f"side affect '{func_name}' was called but self.amdb is None")
+        return self.central_user_db.get_user_by_eppn("hubba-bubba")
+
+    def _side_effect_update_name(self, request: Request) -> Response:
+        db_user = self._side_effect_update(func_name="_side_effect_update_name")
+        assert self.central_user_db is not None
+        mock_request = UserUpdateNameRequest.parse_raw(request.content)
+
+        db_user.given_name = mock_request.given_name
+        db_user.surname = mock_request.surname
+        db_user.display_name = mock_request.display_name
+        self.central_user_db.save(user=db_user)
+        return Response(200, text='{"status": "true"}')
+
+    def _side_effect_update_email(self, request: Request) -> Response:
+        db_user = self._side_effect_update(func_name="_side_effect_update_email")
+        assert self.central_user_db is not None
+        mock_request = UserUpdateEmailRequest.parse_raw(request.content)
+
+        mail_addresses = [mail.to_dict() for mail in mock_request.mail_addresses]
+        db_user = self.central_user_db.get_user_by_eppn("hubba-bubba")
+        self.central_user_db.unverify_mail_aliases(user_id=db_user.user_id, mail_aliases=mail_addresses)
+
+        db_user.mail_addresses = MailAddressList(elements=mock_request.mail_addresses)
+        self.central_user_db.save(user=db_user)
+        return Response(200, text='{"status": "true"}')
+
+    def _side_effect_update_language(self, request: Request) -> Response:
+        db_user = self._side_effect_update(func_name="_side_effect_update_language")
+        assert self.central_user_db is not None
+        mock_request = UserUpdateLanguageRequest.parse_raw(request.content)
+
+        db_user.language = mock_request.language
+        self.central_user_db.save(user=db_user)
+        return Response(200, text='{"status": "true"}')
+
+    def _side_effect_update_phone(self, request: Request) -> Response:
+        db_user = self._side_effect_update(func_name="_side_effect_update_phone")
+        assert self.central_user_db is not None
+        mock_request = UserUpdatePhoneRequest.parse_raw(request.content)
+
+        phones = [phone.to_dict() for phone in mock_request.phone_numbers]
+        self.central_user_db.unverify_phones(user_id=db_user.user_id, phones=phones)
+        db_user.phone_numbers = PhoneNumberList(elements=mock_request.phone_numbers)
+        return Response(200, text='{"status": "true"}')
+
+    def _side_effect_update_terminate(self, request: Request) -> Response:
+        db_user = self._side_effect_update(func_name="_side_effect_update_terminate")
+        assert self.central_user_db is not None
+
+        db_user.terminated = utc_now()
+        return Response(200, text='{"status": "true"}')

src/eduid/common/clients/gnap_client/base.py

@@ -31,7 +31,8 @@ class GNAPClientException(Exception):
 
 
 class GNAPClientAuthData(BaseModel):
-    authn_server_url: str
+    auth_server_url: str
+    auth_server_verify: bool = True
     key_name: str
     client_jwk: JWK
     access: list[Union[str, Access]] = Field(default_factory=list)
@@ -51,7 +52,7 @@ class GNAPBearerTokenMixin(ABC):
 
     @property
     def transaction_uri(self) -> str:
-        return urlappend(self._auth_data.authn_server_url, "transaction")
+        return urlappend(self._auth_data.auth_server_url, "transaction")
 
     def _create_grant_request_jws(self) -> str:
         req = GrantRequest(

src/eduid/common/clients/gnap_client/sync_client.py

@@ -17,7 +17,6 @@ def __init__(self, auth_data: GNAPClientAuthData, **kwargs):
 
         super().__init__(**kwargs)
 
-        self.verify = kwargs.get("verify", True)
         self._auth_data = auth_data
 
     @staticmethod
@@ -34,7 +33,7 @@ def _request_bearer_token(self) -> GrantResponse:
             url=self.transaction_uri,
             content=data,
             headers={"Content-Type": "application/jose+json"},
-            verify=self.verify,
+            verify=self._auth_data.auth_server_verify,
         )
         resp.raise_for_status()
         return GrantResponse.parse_raw(resp.text)

src/eduid/common/rpc/msg_relay.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 import logging
 from enum import Enum
 from typing import Optional
@@ -123,17 +124,20 @@ def is_deregistered(self) -> bool:
         return bool(self.deregistration_information.cause_code or self.deregistration_information.date)
 
 
-class NavetData(NavetModelConfig):
-    case_information: CaseInformation = Field(alias="CaseInformation")
-    person: Person = Field(alias="Person")
-
-
 # Used to parse data from get_postal_address
 class FullPostalAddress(NavetModelConfig):
     name: Name = Field(default_factory=Name, alias="Name")
     official_address: OfficialAddress = Field(default_factory=OfficialAddress, alias="OfficialAddress")
 
 
+class NavetData(NavetModelConfig):
+    case_information: CaseInformation = Field(alias="CaseInformation")
+    person: Person = Field(alias="Person")
+
+    def get_full_postal_address(self) -> FullPostalAddress:
+        return FullPostalAddress(name=self.person.name, official_address=self.person.postal_addresses)
+
+
 class MsgRelay:
     """
     This is the interface to the RPC task to fetch data from NAVET, and to send SMSs.

src/eduid/common/utils.py

@@ -1,5 +1,13 @@
 __author__ = "lundberg"
 
+import logging
+from typing import List, Optional
+
+from eduid.common.rpc.msg_relay import FullPostalAddress
+from eduid.userdb.user import TUserSubclass
+
+logger = logging.getLogger(__name__)
+
 
 def urlappend(base: str, path: str) -> str:
     """
@@ -40,3 +48,70 @@ def removesuffix(s: str, suffix: str) -> str:
         return s[: -len(suffix)]
     else:
         return s[:]
+
+
+def get_marked_given_name(given_name: str, given_name_marking: Optional[str]) -> str:
+    """
+    Given name marking denotes up to two given names, and is used to determine
+    which of the given names are to be primarily used in addressing a person.
+    For this purpose, the given_name_marking is two numbers:
+        indexing starting at 1
+        the second can be 0 for only one mark
+        hyphenated names are counted separately (i.e. Jan-Erik are two separate names)
+            If they are both marked they should be re-hyphenated
+
+    current version of documentation:
+    https://www.skatteverket.se/download/18.2cf1b5cd163796a5c8bf20e/1530691773712/AllmanBeskrivning.pdf
+
+    :param given_name: Given name
+    :param given_name_marking: Given name marking
+
+    :return: Marked given name (Tilltalsnamn)
+    """
+    if not given_name_marking or "00" == given_name_marking:
+        return given_name
+
+    # cheating with indexing
+    _given_names: List[Optional[str]] = [None]
+    for name in given_name.split():
+        if "-" in name:
+            # hyphenated names are counted separately
+            _given_names.extend(name.split("-"))
+        else:
+            _given_names.append(name)
+
+    _optional_marked_names: List[Optional[str]] = []
+    for i in given_name_marking:
+        _optional_marked_names.append(_given_names[int(i)])
+    # remove None values
+    # i.e. 0 index and hyphenated names second part placeholder
+    _marked_names: List[str] = [name for name in _optional_marked_names if name is not None]
+    if "-".join(_marked_names) in given_name:
+        return "-".join(_marked_names)
+    else:
+        return " ".join(_marked_names)
+
+
+def set_user_names_from_official_address(user: TUserSubclass, user_postal_address: FullPostalAddress) -> TUserSubclass:
+    """
+    :param user: Proofing app private userdb user
+    :param user_postal_address: user postal address
+
+    :returns: User object
+    """
+    user.given_name = user_postal_address.name.given_name
+    user.surname = user_postal_address.name.surname
+    if user.given_name is None or user.surname is None:
+        # please mypy
+        raise RuntimeError("No given name or surname found in proofing log user postal address")
+    given_name_marking = user_postal_address.name.given_name_marking
+    user.display_name = f"{user.given_name} {user.surname}"
+    if given_name_marking:
+        _given_name = get_marked_given_name(user.given_name, given_name_marking)
+        user.display_name = f"{_given_name} {user.surname}"
+    logger.info("User names set from official address")
+    logger.debug(
+        f"{user_postal_address.name} resulted in given_name: {user.given_name}, "
+        f"surname: {user.surname} and display_name: {user.display_name}"
+    )
+    return user

src/eduid/userdb/tests/test_user_cleaner.py

@@ -0,0 +1,14 @@
+from unittest import TestCase
+
+from eduid.userdb.testing import MongoTemporaryInstance
+from eduid.userdb.user_cleaner.cache import CacheUser
+from eduid.userdb.user_cleaner.cachedb import CacheDB
+
+
+class TestUserCleanerCache(TestCase):
+    def setUp(self):
+        self.tmp_db = MongoTemporaryInstance.get_instance()
+        self.user_cleaner_meta_db = CacheDB(db_uri=self.tmp_db.uri, collection="skv_cache")
+
+    def tearDown(self):
+        self.user_cleaner_meta_db._drop_whole_collection()

src/eduid/userdb/user_cleaner/cache.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Mapping, Optional
+
+from pydantic import BaseModel, Field
+
+from eduid.common.misc.timeutil import utc_now
+from eduid.userdb.db import TUserDbDocument
+from eduid.userdb.user import User
+
+
+class CacheUser(BaseModel):
+    eppn: str
+    created_ts: datetime = Field(default_factory=utc_now)
+    next_run_ts: Optional[datetime] = None
+
+    def to_dict(self) -> TUserDbDocument:
+        """
+        Convert Element to a dict in eduid format, that can be used to reconstruct the
+        Element later.
+        """
+        data = self.dict(exclude_none=True)
+
+        return TUserDbDocument(data)
+
+    @classmethod
+    def from_dict(cls, data: Mapping[str, Any]) -> CacheUser:
+        """Convert a dict to a Element object."""
+        return cls(**data)
+
+    def from_user(self, data: User) -> CacheUser:
+        """
+        Convert a User object to a CacheUser object.
+        """
+        cache_user = CacheUser(
+            eppn=data.eppn,
+        )
+        return cache_user