Merge pull request #189 from notoraptor/cw-467-gen-fake-data-with-use…

…r-dicts [CW-467] Automatically insert fake job user props on fake data generation.
mila-iqia · May 23, 2024 · c2936c8 · c2936c8
2 parents d31e544 + e025dcf
commit c2936c8
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 9 deletions.
diff --git a/scripts/insert_hardcoded_values.py b/scripts/insert_hardcoded_values.py
@@ -47,6 +47,35 @@ def get_jobs_hardcoded_values():
     ]
 
 
+def get_job_user_props_hardcoded_values(fake_data: dict):
+    """
+    Return partially hardcoded data to be inserted as fake job user props.
+
+    This function will generate 5 job user props
+    associated to the 5 first jobs from fake data
+    edited by 2nd user available in fake data (i.e. `student01`)
+    """
+    mila_email_username = fake_data["users"][1]["mila_email_username"]
+    jobs = fake_data["jobs"][:5]
+    user_props = [
+        {"name": "je suis une user prop 1"},
+        {"name": "je suis une user prop 2"},
+        {"name": "je suis une user prop 3"},
+        {"name": "je suis une user prop 3", "name2": "je suis une user prop 4"},
+        {"name": "je suis une user prop 1"},
+    ]
+
+    return [
+        {
+            "mila_email_username": mila_email_username,
+            "job_id": job["slurm"]["job_id"],
+            "cluster_name": job["slurm"]["cluster_name"],
+            "props": props,
+        }
+        for job, props in zip(jobs, user_props)
+    ]
+
+
 def main(argv):
 
     my_parser = argparse.ArgumentParser()
@@ -77,6 +106,9 @@ def main(argv):
     for job_to_insert in get_jobs_hardcoded_values():
         fake_data["jobs"].append(job_to_insert)
 
+    # Insert fake job user props
+    fake_data["job_user_props"] = get_job_user_props_hardcoded_values(fake_data)
+
     # Write the new fake data in the output file
     with open(output_file, "w") as f:
         json.dump(fake_data, f, indent=2)

diff --git a/scripts/produce_fake_data.sh b/scripts/produce_fake_data.sh
@@ -51,19 +51,18 @@ for SUBFOLDER in ${CLOCKWORK_ROOT}/tmp/slurm_report/*; do
           --output_file ${CLOCKWORK_ROOT}/tmp/slurm_report/${CLUSTER_NAME}/sacct_anonymized
 
       # "job anonymized"  to  "jobs anonymized dump file"
-      ANONYMIZED_JOBS_FILE=${CLOCKWORK_ROOT}/tmp/slurm_report/${CLUSTER_NAME}/job_anonymized_dump_file.json
-      python3 -m slurm_state.read_report_commit_to_db \
-          --cluster_name ${CLUSTER_NAME} \
-          --jobs_file ${CLOCKWORK_ROOT}/tmp/slurm_report/${CLUSTER_NAME}/sacct_anonymized \
-          --dump_file $ANONYMIZED_JOBS_FILE
-      ANONYMIZED_JOBS_FILES+=(${ANONYMIZED_JOBS_FILE})
-
       # "node anonymized"  to  "nodes anonymized dump file"
+      ANONYMIZED_JOBS_FILE=${CLOCKWORK_ROOT}/tmp/slurm_report/${CLUSTER_NAME}/job_anonymized_dump_file.json
       ANONYMIZED_NODES_FILE=${CLOCKWORK_ROOT}/tmp/slurm_report/${CLUSTER_NAME}/node_anonymized_dump_file.json
       python3 -m slurm_state.read_report_commit_to_db \
           --cluster_name ${CLUSTER_NAME} \
-          --nodes_file ${CLOCKWORK_ROOT}/tmp/slurm_report/${CLUSTER_NAME}/sinfo_anonymized \
-          --dump_file $ANONYMIZED_NODES_FILE
+          --from_existing_jobs_file \
+          --slurm_jobs_file ${CLOCKWORK_ROOT}/tmp/slurm_report/${CLUSTER_NAME}/sacct_anonymized \
+          --cw_jobs_file $ANONYMIZED_JOBS_FILE \
+          --from_existing_nodes_file \
+          --slurm_nodes_file ${CLOCKWORK_ROOT}/tmp/slurm_report/${CLUSTER_NAME}/sinfo_anonymized \
+          --cw_nodes_file $ANONYMIZED_NODES_FILE
+      ANONYMIZED_JOBS_FILES+=(${ANONYMIZED_JOBS_FILE})
       ANONYMIZED_NODES_FILES+=($ANONYMIZED_NODES_FILE)
     fi
   fi