Ajout des fonctions et REST endpoints pour gérer les dictionnaires job-utilisateur

.github/workflows/ci_frontend.yml

@@ -17,4 +17,4 @@ jobs:
     - name: Launch frontend tests
       run: |
         . ./env.sh
-        docker-compose run clockwork_dev scripts/launch_frontend_tests_in_clockwork_dev.sh
+        docker compose run clockwork_dev scripts/launch_frontend_tests_in_clockwork_dev.sh

clockwork_frontend_test/test_jobs_search.py

@@ -642,7 +642,7 @@ def test_filter_by_job_array(page: Page):
 
 def test_filter_by_job_user_props(page: Page):
     # Login
-    page.goto(f"{BASE_URL}/login/testing?user_id=student00@mila.quebec")
+    page.goto(f"{BASE_URL}/login/testing?user_id=student01@mila.quebec")
     # Go to settings.
     page.goto(f"{BASE_URL}/settings/")
     radio_job_user_props = page.locator("input#jobs_list_job_user_props_toggle")

clockwork_frontend_test/test_settings.py

@@ -227,7 +227,19 @@ def test_jobs_search_columns(page: Page):
 
 def test_jobs_search_column_job_user_props(page: Page):
     # Login
-    page.goto(f"{BASE_URL}/login/[email protected]")
+    page.goto(f"{BASE_URL}/login/[email protected]")
+
+    # Change language in settings
+    page.goto(f"{BASE_URL}/settings/")
+    # Get language select.
+    select = page.locator("select#language_selection")
+    # Check default language is english.
+    expect(select).to_have_value("fr")
+    # Switch to French.
+    select.select_option("en")
+    # Check french is selected.
+    expect(select).to_have_value("en")
+
     # Check default jobs search columns.
     page.goto(f"{BASE_URL}/jobs/search")
     headers = page.locator("table#search_table thead tr th")

clockwork_tools/clockwork_tools/client.py

@@ -60,7 +60,7 @@ def _get_headers(self):
         encoded_s = str(encoded_bytes, "utf-8")
         return {"Authorization": f"Basic {encoded_s}"}
 
-    def _request(self, endpoint, params, method="GET"):
+    def _request(self, endpoint, params, method="GET", send_json=True):
         """Helper method for REST API calls.
 
         Internal helper method to make the calls to the REST API endpoints
@@ -69,6 +69,9 @@ def _request(self, endpoint, params, method="GET"):
         Args:
             endpoint (str): The REST endpoint, omitting the server address.
             params (dict): Arguments to be provided to the REST endpoint.
+            send_json (bool): Optional. If True and if method is PUT,
+                then request will be sent as a JSON request.
+
 
         Returns:
             Depends on the call made.
@@ -86,9 +89,14 @@ def _request(self, endpoint, params, method="GET"):
                 complete_address, params=params, headers=self._get_headers()
             )
         elif method == "PUT":
-            response = requests.put(
-                complete_address, data=params, headers=self._get_headers()
-            )
+            if send_json:
+                headers = self._get_headers()
+                headers["Content-type"] = "application/json"
+                response = requests.put(complete_address, json=params, headers=headers)
+            else:
+                response = requests.put(
+                    complete_address, data=params, headers=self._get_headers()
+                )
 
         # Check code instead and raise exception if it's the wrong one.
         if response.status_code == 200:
@@ -154,6 +162,60 @@ def jobs_one(self, job_id: str = None, cluster_name: str = None) -> dict[str, an
                 params[k] = a
         return self._request(endpoint, params)
 
+    def get_user_props(self, job_id: str, cluster_name: str) -> dict[str, any]:
+        """REST call to api/v1/clusters/jobs/user_props/get.
+
+        Call for get_user_props().
+
+        Args:
+            job_id (str): ID of job to retrieve props
+            cluster_name (str): Name of job cluster
+
+        Returns:
+            dict[any,any]: props.
+        """
+        endpoint = "api/v1/clusters/jobs/user_props/get"
+        params = {"job_id": job_id, "cluster_name": cluster_name}
+        return self._request(endpoint, params)
+
+    def set_user_props(
+        self, job_id: str, cluster_name: str, updates: dict
+    ) -> dict[str, any]:
+        """REST call to api/v1/clusters/jobs/user_props/set.
+
+        Call for set_user_props().
+
+        Args:
+            job_id (str): ID of job to retrieve props
+            cluster_name (str): Name of job cluster
+            updates (dict): Dict of props to update.
+
+        Returns:
+            dict[any,any]: Returns the updated props.
+        """
+        endpoint = "api/v1/clusters/jobs/user_props/set"
+        params = {"job_id": job_id, "cluster_name": cluster_name, "updates": updates}
+        return self._request(endpoint, params, method="PUT")
+
+    def delete_user_props(
+        self, job_id: str, cluster_name: str, keys: str | list
+    ) -> dict[str, any]:
+        """REST call to api/v1/clusters/jobs/user_props/delete.
+
+        Call for delete_user_props().
+
+        Args:
+            job_id (str): ID of job to retrieve props
+            cluster_name (str): Name of job cluster
+            keys (list): List of keys to delete.
+
+        Returns:
+            dict[any,any]: Returns the updated props.
+        """
+        endpoint = "api/v1/clusters/jobs/user_props/delete"
+        params = {"job_id": job_id, "cluster_name": cluster_name, "keys": keys}
+        return self._request(endpoint, params, method="PUT")
+
     def jobs_user_dict_update(
         self, job_id: str = None, cluster_name: str = None, update_pairs: dict = {}
     ) -> dict[str, any]:
@@ -191,7 +253,7 @@ def jobs_user_dict_update(
         # Due to current constraints, we have to pass "update_pairs"
         # as a string representing a structure in json.
         params["update_pairs"] = json.dumps(update_pairs)
-        return self._request(endpoint, params, method="PUT")
+        return self._request(endpoint, params, method="PUT", send_json=False)
 
     def nodes_list(self, cluster_name: str = None) -> list[dict[str, any]]:
         """REST call to api/v1/clusters/nodes/list.

clockwork_tools_test/test_mt_job_user_props.py

@@ -0,0 +1,69 @@
+import random
+
+
+def _get_test_user_props(mtclient, fake_data):
+    # Find an entry that's associated with the user that's currently logged in.
+    # This becomes the ground truth against which we compare the retrieved user props.
+    LD_candidates = [
+        D_job_user_props_entry
+        for D_job_user_props_entry in fake_data["job_user_props"]
+        if (
+            D_job_user_props_entry["mila_email_username"] == mtclient.email
+            and len(D_job_user_props_entry["props"]) > 0
+        )
+    ]
+    assert (
+        len(LD_candidates) > 0
+    ), "There should be at least one job_user_props entry for the user that's currently logged in."
+    D_job_user_props_entry = random.choice(LD_candidates)
+
+    job_id = D_job_user_props_entry["job_id"]
+    cluster_name = D_job_user_props_entry["cluster_name"]
+    original_props = D_job_user_props_entry["props"]
+    return job_id, cluster_name, original_props
+
+
+def test_cw_tools_get_user_props(mtclient, fake_data):
+    job_id, cluster_name, original_props = _get_test_user_props(mtclient, fake_data)
+    props = mtclient.get_user_props(job_id, cluster_name)
+    assert original_props == props
+
+
+def test_cw_tools_set_user_props(mtclient, fake_data):
+    job_id, cluster_name, original_props = _get_test_user_props(mtclient, fake_data)
+
+    new_prop_name = "a new name"
+    assert new_prop_name not in original_props
+    props = mtclient.set_user_props(job_id, cluster_name, {new_prop_name: "a new prop"})
+    assert props == {**original_props, new_prop_name: "a new prop"}
+
+
+def test_cw_tools_delete_user_props(mtclient, fake_data):
+    job_id, cluster_name, original_props = _get_test_user_props(mtclient, fake_data)
+
+    new_props = {"a new name": "a new prop", "aa": "aa", "bb": "bb", "cc": "cc"}
+    for new_prop_name in new_props.keys():
+        assert new_prop_name not in original_props
+
+    props = mtclient.set_user_props(job_id, cluster_name, new_props)
+    assert len(props) == len(original_props) + len(new_props)
+    assert props == {**original_props, **new_props}
+
+    assert mtclient.delete_user_props(job_id, cluster_name, "aa") == ""
+    props = mtclient.get_user_props(job_id, cluster_name)
+    assert len(props) == len(original_props) + len(new_props) - 1
+    assert props == {
+        **original_props,
+        "a new name": "a new prop",
+        "bb": "bb",
+        "cc": "cc",
+    }
+
+    assert mtclient.delete_user_props(job_id, cluster_name, ["a new name"]) == ""
+    props = mtclient.get_user_props(job_id, cluster_name)
+    assert len(props) == len(original_props) + len(new_props) - 2
+    assert props == {**original_props, "bb": "bb", "cc": "cc"}
+
+    assert mtclient.delete_user_props(job_id, cluster_name, ["bb", "cc"]) == ""
+    props = mtclient.get_user_props(job_id, cluster_name)
+    assert props == original_props

clockwork_web/core/job_user_props_helper.py

@@ -0,0 +1,163 @@
+from flask_login import current_user
+from ..db import get_db
+import json
+
+
+# Max length allowed for a JSON-string representing a job-user props dict.
+# Currently, 2 Mb.
+MAX_PROPS_LENGTH = 2 * 1024 * 1024
+
+
+def get_user_props(
+    job_id: str, cluster_name: str, mila_email_username: str = None
+) -> dict:
+    """
+    Get job-user props.
+
+    Parameters:
+        job_id                  ID of job for which we want to get user props.
+        cluster_name            Name of cluster to which the job belongs.
+        mila_email_username     Optional email of user who sets the props we want to get.
+                                Default is current logged user.
+
+    Returns:
+        Dictionary of user-props, empty if no props were found.
+        Each prop is a key-value pair in returned dictionary.
+    """
+    # Get matching MongoDB document.
+    props_dict = _get_user_props_document(job_id, cluster_name, mila_email_username)
+    # Return props if available.
+    return props_dict.get("props", {})
+
+
+def set_user_props(
+    job_id: str, cluster_name: str, updates: dict, mila_email_username: str = None
+) -> dict:
+    """
+    Update job-user-props.
+
+    Parameters:
+        job_id                  ID of job for which we want to update user props.
+        cluster_name            Name of cluster to which the job belongs.
+        updates                 Dictionary of props to add.
+                                Each key-value represents a prop.
+        mila_email_username     Optional email of user who wants to update his props.
+                                Default is current logged user.
+
+    Returns:
+        Dictionary of Updated job-user props.
+    """
+    # Get previous props and check that
+    # previous + updates props do not exceed a size limit.
+    previous_doc = _get_user_props_document(job_id, cluster_name, mila_email_username)
+    previous_props = previous_doc.get("props", {})
+    new_props = previous_props.copy()
+    new_props.update(updates)
+    if _get_dict_size(new_props) > MAX_PROPS_LENGTH:
+        raise ValueError(
+            f"Too huge job-user props: {_get_megabytes(MAX_PROPS_LENGTH)} Mbytes "
+            f"(previous: {_get_megabytes(_get_dict_size(previous_props))} Mbytes, "
+            f"updates: {_get_megabytes(_get_dict_size(updates))} Mbytes)"
+        )
+
+    # Update or insert job-user props.
+    mc = get_db()
+    db = mc["job_user_props"]
+    if "_id" in previous_doc:
+        db.update_one({"_id": previous_doc["_id"]}, {"$set": {"props": new_props}})
+    else:
+        db.insert_one(
+            {
+                "job_id": job_id,
+                "cluster_name": str(cluster_name),
+                "mila_email_username": current_user.mila_email_username,
+                "props": new_props,
+            }
+        )
+
+    return new_props
+
+
+def delete_user_props(job_id, cluster_name, key_or_keys, mila_email_username=None):
+    """
+    Delete some job-user props.
+
+    Parameters:
+        job_id                  ID of job for which we want to delete some user props.
+        cluster_name            Name of cluster to which the job belongs.
+        key_or_keys             Either a key or a sequence of keys,
+                                representing prop names to delete.
+        mila_email_username     Optional email of user who wants to delete props.
+                                Default is current logged user.
+    """
+    # Parsekey_or_keys to get keys to delete.
+    if isinstance(key_or_keys, str):
+        keys = [key_or_keys]
+    else:
+        assert isinstance(key_or_keys, (list, tuple, set))
+        keys = list(key_or_keys)
+
+    # Get previous props
+    previous_doc = _get_user_props_document(job_id, cluster_name, mila_email_username)
+    previous_props = previous_doc.get("props", {})
+    new_props = previous_props.copy()
+    # Remove keys
+    for key in keys:
+        new_props.pop(key, None)
+    # Register in MongoDB
+    if len(new_props) < len(previous_props):
+        mc = get_db()
+        db = mc["job_user_props"]
+        db.update_one({"_id": previous_doc["_id"]}, {"$set": {"props": new_props}})
+
+
+def _get_user_props_document(
+    job_id: str, cluster_name: str, mila_email_username: str = None
+) -> dict:
+    """
+    Get MongoDB document representing job-user props corresponding to given parameters.
+
+    Parameters:
+        job_id                  ID of job for which we want to get user props.
+        cluster_name            Name of cluster to which the job belongs.
+        mila_email_username     Optional email of user who sets the props we want to get.
+                                Default is current logged user.
+
+    Returns:
+        MongoDB document representing job-user props.
+        Document is a dictionary with following format:
+        {
+            job_id: str                 # ID of job associated to props.
+            cluster_name: str           # cluster name of job associated to props.
+            mila_email_username: str    # user who created these props.
+            props: dict                 # actual user-props, each prop is a key-value.
+        }
+    """
+    mc = get_db()
+    result = list(
+        mc["job_user_props"].find(
+            {
+                "job_id": job_id,
+                "cluster_name": str(cluster_name),
+                "mila_email_username": (
+                    mila_email_username or current_user.mila_email_username
+                ),
+            }
+        )
+    )
+    if not result:
+        return {}
+    else:
+        assert len(result) == 1
+        (props,) = result
+        return props
+
+
+def _get_dict_size(dct: dict) -> int:
+    """Get length of JSON-string representation for given dictionary."""
+    return len(json.dumps(dct))
+
+
+def _get_megabytes(size: int) -> float:
+    """Convert bytes to megabytes."""
+    return size / (1024 * 1024)

clockwork_web/core/jobs_helper.py

@@ -167,9 +167,7 @@ def get_filtered_and_paginated_jobs(
             mc["job_user_props"].find(
                 combine_all_mongodb_filters(
                     {
-                        "job_id": {
-                            "$in": [int(job["slurm"]["job_id"]) for job in LD_jobs]
-                        },
+                        "job_id": {"$in": [job["slurm"]["job_id"] for job in LD_jobs]},
                         "mila_email_username": current_user.mila_email_username,
                     }
                 )
@@ -190,7 +188,7 @@ def get_filtered_and_paginated_jobs(
             for job in LD_jobs:
                 key = (
                     current_user.mila_email_username,
-                    int(job["slurm"]["job_id"]),
+                    job["slurm"]["job_id"],
                     job["slurm"]["cluster_name"],
                 )
                 if key in user_props_map: