From 384ae21c83fc3f75469ac85ca1b332658d77286b Mon Sep 17 00:00:00 2001 From: Andreas Eknes Lie Date: Tue, 19 Sep 2023 09:15:25 +0200 Subject: [PATCH 01/19] Remove includes from enkf_obs --- src/clib/lib/enkf/enkf_obs.cpp | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/clib/lib/enkf/enkf_obs.cpp b/src/clib/lib/enkf/enkf_obs.cpp index 2594d73ece8..f9b64b9804a 100644 --- a/src/clib/lib/enkf/enkf_obs.cpp +++ b/src/clib/lib/enkf/enkf_obs.cpp @@ -1,22 +1,10 @@ -#include #include +#include #include -#include -#include -#include -#include -#include - -#include #include -#include #include #include -#include -#include -#include -#include ERT_CLIB_SUBMODULE("enkf_obs", m) { using namespace py::literals; From f3e227b564ea92e72654c3fd7f5cd80d52d039f4 Mon Sep 17 00:00:00 2001 From: Andreas Eknes Lie Date: Tue, 19 Sep 2023 09:38:52 +0200 Subject: [PATCH 02/19] Remove unused JOB_DRIVER_ENUM_SIZE --- src/clib/lib/include/ert/job_queue/queue_driver.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/clib/lib/include/ert/job_queue/queue_driver.hpp b/src/clib/lib/include/ert/job_queue/queue_driver.hpp index 328817e5bdf..e1a46dbb654 100644 --- a/src/clib/lib/include/ert/job_queue/queue_driver.hpp +++ b/src/clib/lib/include/ert/job_queue/queue_driver.hpp @@ -13,8 +13,6 @@ typedef enum { SLURM_DRIVER = 5 } job_driver_type; -#define JOB_DRIVER_ENUM_SIZE 5 - typedef struct queue_driver_struct queue_driver_type; typedef void *(submit_job_ftype)(void *data, const char *cmd, int num_cpu, From 5e39cdbaf507feb4eadb456f18f265416d2b16ba Mon Sep 17 00:00:00 2001 From: Andreas Eknes Lie Date: Tue, 19 Sep 2023 09:39:29 +0200 Subject: [PATCH 03/19] Fix includes in job_node --- src/clib/lib/job_queue/job_node.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/clib/lib/job_queue/job_node.cpp b/src/clib/lib/job_queue/job_node.cpp index 0d65aab19f0..d32c65ebb6f 100644 --- a/src/clib/lib/job_queue/job_node.cpp +++ b/src/clib/lib/job_queue/job_node.cpp @@ -2,8 +2,7 @@ #include #include -#include -#include +#include #include #include From 93e181f805b4b03e1763ad83b556f30085994480 Mon Sep 17 00:00:00 2001 From: Andreas Eknes Lie Date: Tue, 19 Sep 2023 09:43:40 +0200 Subject: [PATCH 04/19] Fix includes in slurm_driver --- src/clib/lib/job_queue/slurm_driver.cpp | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/clib/lib/job_queue/slurm_driver.cpp b/src/clib/lib/job_queue/slurm_driver.cpp index df872737fc4..bd13bc6a106 100644 --- a/src/clib/lib/job_queue/slurm_driver.cpp +++ b/src/clib/lib/job_queue/slurm_driver.cpp @@ -1,28 +1,24 @@ -#include -#include -#include -#include -#include -#include - #include +#include +#include +#include #include +#include +#include #include #include #include #include +#include #include -#include #include -#include -#include -#include - -#include #include #include +#include #include +#include +#include static auto logger = ert::get_logger("job_queue.slurm_driver"); From b9b0f070b5573930f0975f7e07587687965e5efd Mon Sep 17 00:00:00 2001 From: Andreas Eknes Lie Date: Tue, 19 Sep 2023 09:46:41 +0200 Subject: [PATCH 05/19] Fix includes in torque_driver --- src/clib/lib/job_queue/torque_driver.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/clib/lib/job_queue/torque_driver.cpp b/src/clib/lib/job_queue/torque_driver.cpp index 52424ad542a..2450bcc2235 100644 --- a/src/clib/lib/job_queue/torque_driver.cpp +++ b/src/clib/lib/job_queue/torque_driver.cpp @@ -1,21 +1,18 @@ -#include - #include #include +#include +#include +#include #include #include -#include #include -#include -#include #include #include -#include - #include #include #include +#include namespace fs = std::filesystem; From 7369cfd698c03f68b64c87906133f6ce059da258 Mon Sep 17 00:00:00 2001 From: Andreas Eknes Lie Date: Tue, 19 Sep 2023 09:47:29 +0200 Subject: [PATCH 06/19] Fix includes in local_driver --- src/clib/lib/job_queue/local_driver.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/clib/lib/job_queue/local_driver.cpp b/src/clib/lib/job_queue/local_driver.cpp index e8b1d839c39..b316c1028c5 100644 --- a/src/clib/lib/job_queue/local_driver.cpp +++ b/src/clib/lib/job_queue/local_driver.cpp @@ -1,7 +1,7 @@ +#include +#include #include #include -#include -#include #include #include From ef5cb2520b405dedb680707dc5a854e9184b2f53 Mon Sep 17 00:00:00 2001 From: Andreas Eknes Lie Date: Tue, 19 Sep 2023 09:48:20 +0200 Subject: [PATCH 07/19] Fix includes in lsf_driver --- src/clib/lib/job_queue/lsf_driver.cpp | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/clib/lib/job_queue/lsf_driver.cpp b/src/clib/lib/job_queue/lsf_driver.cpp index 16eeb3d3479..04daac16816 100644 --- a/src/clib/lib/job_queue/lsf_driver.cpp +++ b/src/clib/lib/job_queue/lsf_driver.cpp @@ -1,30 +1,27 @@ #include -#include -#include -#include -#include -#include -#include - #include #include #include -#include +#include +#include #include +#include #include +#include +#include #include +#include #include -#include -#include -#include -#include - #include #include #include #include +#include #include +#include +#include +#include namespace fs = std::filesystem; static auto logger = ert::get_logger("job_queue.lsf_driver"); From fbe963041ee6eb8a8db32855505e45a926a93784 Mon Sep 17 00:00:00 2001 From: Andreas Eknes Lie Date: Tue, 19 Sep 2023 09:52:29 +0200 Subject: [PATCH 08/19] Fix torque_driver header includes --- src/clib/lib/include/ert/job_queue/torque_driver.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clib/lib/include/ert/job_queue/torque_driver.hpp b/src/clib/lib/include/ert/job_queue/torque_driver.hpp index af33b6d2f82..38472c77e57 100644 --- a/src/clib/lib/include/ert/job_queue/torque_driver.hpp +++ b/src/clib/lib/include/ert/job_queue/torque_driver.hpp @@ -1,7 +1,7 @@ #ifndef TORQUE_DRIVER_H #define TORQUE_DRIVER_H -#include +#include #include #include From 212ffcba5fd88e6730181e04987c5986baed2824 Mon Sep 17 00:00:00 2001 From: Andreas Eknes Lie Date: Tue, 19 Sep 2023 09:52:46 +0200 Subject: [PATCH 09/19] Fix lsf driver includes --- src/clib/lib/job_queue/lsf_driver.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/clib/lib/job_queue/lsf_driver.cpp b/src/clib/lib/job_queue/lsf_driver.cpp index 04daac16816..0234d847850 100644 --- a/src/clib/lib/job_queue/lsf_driver.cpp +++ b/src/clib/lib/job_queue/lsf_driver.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include From 65408e582e81d190ae6c4587989cf64e694b315e Mon Sep 17 00:00:00 2001 From: Andreas Eknes Lie Date: Tue, 19 Sep 2023 09:53:00 +0200 Subject: [PATCH 10/19] Fix queue_driver includes --- src/clib/lib/job_queue/queue_driver.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/clib/lib/job_queue/queue_driver.cpp b/src/clib/lib/job_queue/queue_driver.cpp index 81e8001b9dc..22c3504e866 100644 --- a/src/clib/lib/job_queue/queue_driver.cpp +++ b/src/clib/lib/job_queue/queue_driver.cpp @@ -1,8 +1,4 @@ -#include -#include - #include - #include #include #include From 9d15a4cbe93b331579d898b4eb8d00f5a0c60aff Mon Sep 17 00:00:00 2001 From: Andreas Eknes Lie Date: Tue, 19 Sep 2023 09:53:14 +0200 Subject: [PATCH 11/19] Replace NULL with nullptr in queue_driver --- src/clib/lib/job_queue/queue_driver.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/clib/lib/job_queue/queue_driver.cpp b/src/clib/lib/job_queue/queue_driver.cpp index 22c3504e866..1426942a5e6 100644 --- a/src/clib/lib/job_queue/queue_driver.cpp +++ b/src/clib/lib/job_queue/queue_driver.cpp @@ -62,15 +62,15 @@ bool queue_driver_set_option(queue_driver_type *driver, const char *option_key, */ static queue_driver_type *queue_driver_alloc_empty() { auto driver = new queue_driver_type; - driver->submit = NULL; - driver->get_status = NULL; - driver->kill_job = NULL; - driver->free_job = NULL; - driver->free_driver = NULL; - driver->get_option = NULL; - driver->set_option = NULL; - driver->data = NULL; - driver->init_options = NULL; + driver->submit = nullptr; + driver->get_status = nullptr; + driver->kill_job = nullptr; + driver->free_job = nullptr; + driver->free_driver = nullptr; + driver->get_option = nullptr; + driver->set_option = nullptr; + driver->data = nullptr; + driver->init_options = nullptr; return driver; } From f50a349c51f115211f333b95e622092914c69d30 Mon Sep 17 00:00:00 2001 From: Andreas Eknes Lie Date: Tue, 19 Sep 2023 10:14:54 +0200 Subject: [PATCH 12/19] Remove typedef to unused struct --- src/clib/lib/include/ert/job_queue/slurm_driver.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/clib/lib/include/ert/job_queue/slurm_driver.hpp b/src/clib/lib/include/ert/job_queue/slurm_driver.hpp index ad9b54fd8f3..f815175e700 100644 --- a/src/clib/lib/include/ert/job_queue/slurm_driver.hpp +++ b/src/clib/lib/include/ert/job_queue/slurm_driver.hpp @@ -15,7 +15,6 @@ */ typedef struct slurm_driver_struct slurm_driver_type; -typedef struct slurm_job_struct slurm_job_type; #define SLURM_SBATCH_OPTION "SBATCH" #define SLURM_SCANCEL_OPTION "SCANCEL" From fe997ea9d7e64e90dca56abb7ae5b2499654739e Mon Sep 17 00:00:00 2001 From: Andreas Eknes Lie Date: Tue, 19 Sep 2023 10:17:29 +0200 Subject: [PATCH 13/19] Initialize lsf struct --- src/clib/lib/job_queue/lsf_driver.cpp | 65 ++++++++++----------------- 1 file changed, 23 insertions(+), 42 deletions(-) diff --git a/src/clib/lib/job_queue/lsf_driver.cpp b/src/clib/lib/job_queue/lsf_driver.cpp index 0234d847850..b83a1dd7b8d 100644 --- a/src/clib/lib/job_queue/lsf_driver.cpp +++ b/src/clib/lib/job_queue/lsf_driver.cpp @@ -92,31 +92,30 @@ static auto logger = ert::get_logger("job_queue.lsf_driver"); #define DEFAULT_BHIST_CMD "bhist" struct lsf_job_struct { - long int lsf_jobnr; + long int lsf_jobnr = 0; /** Used to look up the job status in the bjobs_cache hash table */ - char *lsf_jobnr_char; - char *job_name; + char *lsf_jobnr_char = nullptr; + char *job_name = nullptr; }; struct lsf_driver_struct { - char *queue_name; - char *resource_request; - std::vector exclude_hosts; - char *login_shell; - char *project_code; + char *queue_name = nullptr; + char *resource_request = nullptr; + std::vector exclude_hosts{}; + char *login_shell = nullptr; + char *project_code = nullptr; pthread_mutex_t submit_lock; - lsf_submit_method_enum submit_method; - int submit_sleep; - - int error_count; - int submit_error_sleep; + lsf_submit_method_enum submit_method = LSF_SUBMIT_LOCAL_SHELL; + int submit_sleep = 0; + int error_count = 0; + int submit_error_sleep = SUBMIT_ERROR_SLEEP * 1000000; /*-----------------------------------------------------------------*/ /* Fields used by the shell based functions */ - bool debug_output; + bool debug_output = false; int bjobs_refresh_interval; - time_t last_bjobs_update; + time_t last_bjobs_update = time(nullptr); /** A hash table of all jobs submitted by this ERT instance - to ensure * that we do not check status of old jobs in e.g. ZOMBIE status. */ hash_type *my_jobs; @@ -124,12 +123,12 @@ struct lsf_driver_struct { hash_type *bjobs_cache; /** Only one thread should update the bjobs_chache table. */ pthread_mutex_t bjobs_mutex; - char *remote_lsf_server; - char *rsh_cmd; - char *bsub_cmd; - char *bjobs_cmd; - char *bkill_cmd; - char *bhist_cmd; + char *remote_lsf_server = nullptr; + char *rsh_cmd = nullptr; + char *bsub_cmd = nullptr; + char *bjobs_cmd = nullptr; + char *bkill_cmd = nullptr; + char *bhist_cmd = nullptr; }; const std::map status_map = { @@ -157,8 +156,6 @@ const std::map convert_status_map = { static lsf_job_type *lsf_job_alloc(const char *job_name) { auto job = new lsf_job_type; - job->lsf_jobnr = 0; - job->lsf_jobnr_char = NULL; job->job_name = util_alloc_string_copy(job_name); return job; } @@ -922,15 +919,11 @@ void lsf_driver_set_bjobs_refresh_interval(lsf_driver_type *driver, } static void lsf_driver_shell_init(lsf_driver_type *lsf_driver) { - lsf_driver->last_bjobs_update = time(NULL); + lsf_driver->last_bjobs_update = time(nullptr); lsf_driver->bjobs_cache = hash_alloc(); lsf_driver->my_jobs = hash_alloc(); - lsf_driver->bsub_cmd = NULL; - lsf_driver->bjobs_cmd = NULL; - lsf_driver->bkill_cmd = NULL; - lsf_driver->bhist_cmd = NULL; - pthread_mutex_init(&lsf_driver->bjobs_mutex, NULL); + pthread_mutex_init(&lsf_driver->bjobs_mutex, nullptr); } bool lsf_driver_has_project_code(const lsf_driver_type *driver) { @@ -939,20 +932,8 @@ bool lsf_driver_has_project_code(const lsf_driver_type *driver) { void *lsf_driver_alloc() { lsf_driver_type *lsf_driver = new lsf_driver_type(); - - lsf_driver->submit_method = LSF_SUBMIT_INVALID; - lsf_driver->login_shell = NULL; - lsf_driver->queue_name = NULL; - lsf_driver->remote_lsf_server = NULL; - lsf_driver->rsh_cmd = NULL; - lsf_driver->resource_request = NULL; - lsf_driver->project_code = NULL; - lsf_driver->error_count = 0; - lsf_driver->submit_error_sleep = SUBMIT_ERROR_SLEEP * 1000000; - pthread_mutex_init(&lsf_driver->submit_lock, NULL); - + pthread_mutex_init(&lsf_driver->submit_lock, nullptr); lsf_driver_shell_init(lsf_driver); - lsf_driver->submit_method = LSF_SUBMIT_LOCAL_SHELL; lsf_driver_set_option(lsf_driver, LSF_SERVER, NULL); lsf_driver_set_option(lsf_driver, LSF_RSH_CMD, DEFAULT_RSH_CMD); From 54c5c4e3b18df5baccc54496897e66ba1ccdea10 Mon Sep 17 00:00:00 2001 From: Andreas Eknes Lie Date: Tue, 19 Sep 2023 10:18:00 +0200 Subject: [PATCH 14/19] Initialize torque struct and replace NULL nullptr --- src/clib/lib/job_queue/torque_driver.cpp | 86 +++++++++--------------- 1 file changed, 33 insertions(+), 53 deletions(-) diff --git a/src/clib/lib/job_queue/torque_driver.cpp b/src/clib/lib/job_queue/torque_driver.cpp index 2450bcc2235..c33ace7a8ee 100644 --- a/src/clib/lib/job_queue/torque_driver.cpp +++ b/src/clib/lib/job_queue/torque_driver.cpp @@ -21,50 +21,33 @@ fs::path DEFAULT_QSTAT_CMD = TORQUE_DEFAULT_QSTAT_CMD; } struct torque_driver_struct { - char *queue_name; - char *qsub_cmd; - char *qstat_cmd; - char *qstat_opts; - char *qdel_cmd; - char *num_cpus_per_node_char; - char *memory_per_job; - char *job_prefix; - char *num_nodes_char; - char *timeout_char; - bool keep_qsub_output; - int num_cpus_per_node; - int num_nodes; - char *cluster_label; - int submit_sleep; - int timeout; - FILE *debug_stream; + char *queue_name = nullptr; + char *qsub_cmd = nullptr; + char *qstat_cmd = nullptr; + char *qstat_opts = nullptr; + char *qdel_cmd = nullptr; + char *num_cpus_per_node_char = nullptr; + char *memory_per_job = nullptr; + char *job_prefix = nullptr; + char *num_nodes_char = nullptr; + char *timeout_char = nullptr; + bool keep_qsub_output = false; + int num_cpus_per_node = 1; + int num_nodes = 1; + char *cluster_label = nullptr; + int submit_sleep = 0; + int timeout = 0; + FILE *debug_stream = nullptr; }; struct torque_job_struct { - long int torque_jobnr; - char *torque_jobnr_char; + long int torque_jobnr = 0; + char *torque_jobnr_char = nullptr; }; void *torque_driver_alloc() { auto torque_driver = new torque_driver_type; - torque_driver->queue_name = NULL; - torque_driver->qsub_cmd = NULL; - torque_driver->qstat_cmd = NULL; - torque_driver->qstat_opts = NULL; - torque_driver->qdel_cmd = NULL; - torque_driver->num_cpus_per_node_char = NULL; - torque_driver->memory_per_job = NULL; - torque_driver->num_nodes_char = NULL; - torque_driver->keep_qsub_output = false; - torque_driver->num_cpus_per_node = 1; - torque_driver->num_nodes = 1; - torque_driver->cluster_label = NULL; - torque_driver->job_prefix = NULL; - torque_driver->debug_stream = NULL; - torque_driver->timeout = 0; - torque_driver->timeout_char = NULL; - torque_driver_set_option(torque_driver, TORQUE_QSUB_CMD, TORQUE_DEFAULT_QSUB_CMD); torque_driver_set_option(torque_driver, TORQUE_QSTAT_CMD, @@ -88,7 +71,7 @@ static void torque_driver_set_debug_output(torque_driver_type *driver, if (driver->debug_stream) fclose(driver->debug_stream); - driver->debug_stream = NULL; + driver->debug_stream = nullptr; if (debug_file) { auto path = fs::path(debug_file); @@ -293,9 +276,6 @@ void torque_driver_init_option_list(stringlist_type *option_list) { torque_job_type *torque_job_alloc() { torque_job_type *job; job = (torque_job_type *)util_malloc(sizeof *job); - job->torque_jobnr_char = NULL; - job->torque_jobnr = 0; - return job; } @@ -331,13 +311,13 @@ stringlist_type *torque_driver_alloc_cmd(torque_driver_type *driver, { stringlist_append_copy(argv, "-l"); std::string cluster_label; - if (driver->cluster_label == NULL) { + if (driver->cluster_label == nullptr) { cluster_label = std::string(""); } else { cluster_label = std::string(driver->cluster_label); } std::string memory_per_job; - if (driver->memory_per_job == NULL) { + if (driver->memory_per_job == nullptr) { memory_per_job = std::string(""); } else { memory_per_job = std::string(driver->memory_per_job); @@ -349,12 +329,12 @@ stringlist_type *torque_driver_alloc_cmd(torque_driver_type *driver, .c_str()); } - if (driver->queue_name != NULL) { + if (driver->queue_name != nullptr) { stringlist_append_copy(argv, "-q"); stringlist_append_copy(argv, driver->queue_name); } - if (job_name != NULL) { + if (job_name != nullptr) { stringlist_append_copy(argv, "-N"); stringlist_append_copy(argv, job_name); } @@ -408,7 +388,7 @@ static int torque_job_parse_qsub_stdout(const torque_driver_type *driver, char *jobid_string = util_fscanf_alloc_upto(stdout_stream, ".", false); bool possible_jobid; - if (jobid_string == NULL) { + if (jobid_string == nullptr) { /* We get here if the '.' separator is not found */ possible_jobid = util_fscanf_int(stdout_stream, &jobid); torque_debug(driver, "Torque job ID int: '%d'", jobid); @@ -419,9 +399,9 @@ static int torque_job_parse_qsub_stdout(const torque_driver_type *driver, if (!possible_jobid) { char *stdout_content = - util_fread_alloc_file_content(stdout_file, NULL); + util_fread_alloc_file_content(stdout_file, nullptr); char *stderr_content = - util_fread_alloc_file_content(stderr_file, NULL); + util_fread_alloc_file_content(stderr_file, nullptr); fprintf(stderr, "Failed to get torque job id from file: %s \n", stdout_file); fprintf(stderr, "qsub command: %s \n", driver->qsub_cmd); @@ -440,7 +420,7 @@ static int torque_job_parse_qsub_stdout(const torque_driver_type *driver, void torque_job_create_submit_script(const char *script_filename, const char *submit_cmd, int argc, const char *const *job_argv) { - if (submit_cmd == NULL) { + if (submit_cmd == nullptr) { util_abort("%s: cannot create submit script, because there is no " "executing commmand specified.", __func__); @@ -594,7 +574,7 @@ void *torque_driver_submit_job(void *__driver, const char *submit_cmd, torque_debug(driver, "Submitting job in:%s", run_path); { - char *local_job_name = NULL; + char *local_job_name = nullptr; if (driver->job_prefix) local_job_name = util_alloc_sprintf("%s%s", driver->job_prefix, job_name); @@ -615,7 +595,7 @@ void *torque_driver_submit_job(void *__driver, const char *submit_cmd, // The submit failed - the queue system shall handle // NULL return values. torque_job_free(job); - return NULL; + return nullptr; } } @@ -714,7 +694,7 @@ job_status_type torque_driver_parse_status(const char *qstat_file, job_status_type status = JOB_QUEUE_STATUS_FAILURE; int jobnr_no_namespace = -1; - if (jobnr_char != NULL) { + if (jobnr_char != nullptr) { /* Remove namespace from incoming job_id */ std::string jobnr_namespaced(jobnr_char); int dot_position; @@ -881,7 +861,7 @@ void torque_driver_kill_job(void *__driver, void *__job) { "qdel failed for job %s, no (more) retries", job->torque_jobnr_char); char *stderr_content = - util_fread_alloc_file_content(tmp_err_file, NULL); + util_fread_alloc_file_content(tmp_err_file, nullptr); torque_debug(driver, "qdel stderr: %s\n", stderr_content); free(stderr_content); break; @@ -900,7 +880,7 @@ void torque_driver_kill_job(void *__driver, void *__job) { } void torque_driver_free(torque_driver_type *driver) { - torque_driver_set_debug_output(driver, NULL); + torque_driver_set_debug_output(driver, nullptr); free(driver->queue_name); free(driver->qsub_cmd); free(driver->qstat_cmd); From 07cd3694885576121fef48d7ed6b8e1a29d968e8 Mon Sep 17 00:00:00 2001 From: Andreas Eknes Lie Date: Tue, 19 Sep 2023 10:19:15 +0200 Subject: [PATCH 15/19] clang-format --- src/clib/lib/job_queue/job_node.cpp | 2 +- src/clib/lib/job_queue/queue_driver.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/clib/lib/job_queue/job_node.cpp b/src/clib/lib/job_queue/job_node.cpp index d32c65ebb6f..3472f882767 100644 --- a/src/clib/lib/job_queue/job_node.cpp +++ b/src/clib/lib/job_queue/job_node.cpp @@ -1,8 +1,8 @@ #include #include -#include #include +#include #include #include diff --git a/src/clib/lib/job_queue/queue_driver.cpp b/src/clib/lib/job_queue/queue_driver.cpp index 1426942a5e6..e70ecc43a4d 100644 --- a/src/clib/lib/job_queue/queue_driver.cpp +++ b/src/clib/lib/job_queue/queue_driver.cpp @@ -1,9 +1,9 @@ -#include #include #include #include #include #include +#include /* This file implements the datatype queue_driver_type which is an From 1270e907e4517fef9b86ea1e8e604f80dbb08ee8 Mon Sep 17 00:00:00 2001 From: Andreas Eknes Lie Date: Tue, 19 Sep 2023 10:31:12 +0200 Subject: [PATCH 16/19] Initialize queue_driver --- src/clib/lib/job_queue/queue_driver.cpp | 31 ++++++++----------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/src/clib/lib/job_queue/queue_driver.cpp b/src/clib/lib/job_queue/queue_driver.cpp index e70ecc43a4d..7a07f2508f3 100644 --- a/src/clib/lib/job_queue/queue_driver.cpp +++ b/src/clib/lib/job_queue/queue_driver.cpp @@ -33,17 +33,17 @@ struct queue_driver_struct { /** Function pointers - pointing to low level functions in the * implementations of e.g. lsf_driver. */ - submit_job_ftype *submit; - free_job_ftype *free_job; - kill_job_ftype *kill_job; - get_status_ftype *get_status; - free_queue_driver_ftype *free_driver; - set_option_ftype *set_option; - get_option_ftype *get_option; - init_option_list_ftype *init_options; + submit_job_ftype *submit = nullptr; + free_job_ftype *free_job = nullptr; + kill_job_ftype *kill_job = nullptr; + get_status_ftype *get_status = nullptr; + free_queue_driver_ftype *free_driver = nullptr; + set_option_ftype *set_option = nullptr; + get_option_ftype *get_option = nullptr; + init_option_list_ftype *init_options = nullptr; /** Driver specific data - passed as first argument to the driver functions above. */ - void *data; + void *data = nullptr; }; bool queue_driver_set_option(queue_driver_type *driver, const char *option_key, @@ -61,18 +61,7 @@ bool queue_driver_set_option(queue_driver_type *driver, const char *option_key, NOT properly initialized and NOT ready for use. */ static queue_driver_type *queue_driver_alloc_empty() { - auto driver = new queue_driver_type; - driver->submit = nullptr; - driver->get_status = nullptr; - driver->kill_job = nullptr; - driver->free_job = nullptr; - driver->free_driver = nullptr; - driver->get_option = nullptr; - driver->set_option = nullptr; - driver->data = nullptr; - driver->init_options = nullptr; - - return driver; + return new queue_driver_type; } // The driver created in this function has all the function pointers From df78841d7a7a47e55bf80693e48671fd6cb098c5 Mon Sep 17 00:00:00 2001 From: Andreas Eknes Lie Date: Tue, 19 Sep 2023 10:37:12 +0200 Subject: [PATCH 17/19] Small fixups lsf driver --- src/clib/lib/job_queue/lsf_driver.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/clib/lib/job_queue/lsf_driver.cpp b/src/clib/lib/job_queue/lsf_driver.cpp index b83a1dd7b8d..b39482d5e02 100644 --- a/src/clib/lib/job_queue/lsf_driver.cpp +++ b/src/clib/lib/job_queue/lsf_driver.cpp @@ -104,7 +104,7 @@ struct lsf_driver_struct { std::vector exclude_hosts{}; char *login_shell = nullptr; char *project_code = nullptr; - pthread_mutex_t submit_lock; + pthread_mutex_t submit_lock{}; lsf_submit_method_enum submit_method = LSF_SUBMIT_LOCAL_SHELL; int submit_sleep = 0; @@ -114,7 +114,7 @@ struct lsf_driver_struct { /*-----------------------------------------------------------------*/ /* Fields used by the shell based functions */ bool debug_output = false; - int bjobs_refresh_interval; + int bjobs_refresh_interval = 0; time_t last_bjobs_update = time(nullptr); /** A hash table of all jobs submitted by this ERT instance - to ensure * that we do not check status of old jobs in e.g. ZOMBIE status. */ @@ -122,7 +122,7 @@ struct lsf_driver_struct { /** The output of calling bjobs is cached in this table. */ hash_type *bjobs_cache; /** Only one thread should update the bjobs_chache table. */ - pthread_mutex_t bjobs_mutex; + pthread_mutex_t bjobs_mutex{}; char *remote_lsf_server = nullptr; char *rsh_cmd = nullptr; char *bsub_cmd = nullptr; @@ -922,8 +922,6 @@ static void lsf_driver_shell_init(lsf_driver_type *lsf_driver) { lsf_driver->last_bjobs_update = time(nullptr); lsf_driver->bjobs_cache = hash_alloc(); lsf_driver->my_jobs = hash_alloc(); - - pthread_mutex_init(&lsf_driver->bjobs_mutex, nullptr); } bool lsf_driver_has_project_code(const lsf_driver_type *driver) { @@ -932,7 +930,6 @@ bool lsf_driver_has_project_code(const lsf_driver_type *driver) { void *lsf_driver_alloc() { lsf_driver_type *lsf_driver = new lsf_driver_type(); - pthread_mutex_init(&lsf_driver->submit_lock, nullptr); lsf_driver_shell_init(lsf_driver); lsf_driver_set_option(lsf_driver, LSF_SERVER, NULL); From c39fe28bde3d1370868ba07da513b84071f0d02b Mon Sep 17 00:00:00 2001 From: Andreas Eknes Lie Date: Tue, 19 Sep 2023 11:53:32 +0200 Subject: [PATCH 18/19] Simplification of torque_driver --- src/clib/lib/job_queue/torque_driver.cpp | 72 ++++++++++-------------- 1 file changed, 30 insertions(+), 42 deletions(-) diff --git a/src/clib/lib/job_queue/torque_driver.cpp b/src/clib/lib/job_queue/torque_driver.cpp index c33ace7a8ee..907812015ba 100644 --- a/src/clib/lib/job_queue/torque_driver.cpp +++ b/src/clib/lib/job_queue/torque_driver.cpp @@ -116,8 +116,8 @@ static bool torque_driver_set_submit_sleep(torque_driver_type *driver, if (util_sscanf_double(submit_sleep, &seconds_sleep)) { driver->submit_sleep = (int)(seconds_sleep * 1000000); return true; - } else - return false; + } + return false; } static bool torque_driver_set_num_nodes(torque_driver_type *driver, @@ -128,9 +128,8 @@ static bool torque_driver_set_num_nodes(torque_driver_type *driver, driver->num_nodes_char = util_realloc_string_copy(driver->num_nodes_char, num_nodes_char); return true; - } else { - return false; } + return false; } static bool @@ -141,9 +140,8 @@ torque_driver_set_keep_qsub_output(torque_driver_type *driver, if (util_sscanf_bool(keep_output_bool_as_char, &keep_output_parsed)) { driver->keep_qsub_output = keep_output_parsed; return true; - } else { - return false; } + return false; } static void torque_driver_set_job_prefix(torque_driver_type *driver, @@ -167,9 +165,8 @@ torque_driver_set_num_cpus_per_node(torque_driver_type *driver, driver->num_cpus_per_node_char = util_realloc_string_copy( driver->num_cpus_per_node_char, num_cpus_per_node_char); return true; - } else { - return false; } + return false; } static bool torque_driver_set_memory_per_job(torque_driver_type *driver, @@ -187,8 +184,8 @@ static bool torque_driver_set_timeout(torque_driver_type *driver, driver->timeout_char = util_realloc_string_copy(driver->timeout_char, timeout_char); return true; - } else - return false; + } + return false; } bool torque_driver_set_option(void *__driver, const char *option_key, @@ -308,26 +305,19 @@ stringlist_type *torque_driver_alloc_cmd(torque_driver_type *driver, stringlist_append_copy(argv, "oe"); } - { - stringlist_append_copy(argv, "-l"); - std::string cluster_label; - if (driver->cluster_label == nullptr) { - cluster_label = std::string(""); - } else { - cluster_label = std::string(driver->cluster_label); - } - std::string memory_per_job; - if (driver->memory_per_job == nullptr) { - memory_per_job = std::string(""); - } else { - memory_per_job = std::string(driver->memory_per_job); - } - stringlist_append_copy( - argv, - build_resource_string(driver->num_nodes, cluster_label, - driver->num_cpus_per_node, memory_per_job) - .c_str()); + stringlist_append_copy(argv, "-l"); + std::string cluster_label{}; + if (driver->cluster_label != nullptr) { + cluster_label = std::string(driver->cluster_label); + } + std::string memory_per_job{}; + if (driver->memory_per_job != nullptr) { + memory_per_job = std::string(driver->memory_per_job); } + stringlist_append_copy( + argv, build_resource_string(driver->num_nodes, cluster_label, + driver->num_cpus_per_node, memory_per_job) + .c_str()); if (driver->queue_name != nullptr) { stringlist_append_copy(argv, "-q"); @@ -573,21 +563,19 @@ void *torque_driver_submit_job(void *__driver, const char *submit_cmd, torque_job_type *job = torque_job_alloc(); torque_debug(driver, "Submitting job in:%s", run_path); - { - char *local_job_name = nullptr; - if (driver->job_prefix) - local_job_name = - util_alloc_sprintf("%s%s", driver->job_prefix, job_name); - else - local_job_name = util_alloc_string_copy(job_name); + char *local_job_name = nullptr; + if (driver->job_prefix) + local_job_name = + util_alloc_sprintf("%s%s", driver->job_prefix, job_name); + else + local_job_name = util_alloc_string_copy(job_name); - job->torque_jobnr = torque_driver_submit_shell_job( - driver, run_path, local_job_name, submit_cmd, num_cpu, argc, argv); - job->torque_jobnr_char = util_alloc_sprintf("%ld", job->torque_jobnr); + job->torque_jobnr = torque_driver_submit_shell_job( + driver, run_path, local_job_name, submit_cmd, num_cpu, argc, argv); + job->torque_jobnr_char = util_alloc_sprintf("%ld", job->torque_jobnr); - torque_debug(driver, "Job:%s Id:%d", run_path, job->torque_jobnr); - free(local_job_name); - } + torque_debug(driver, "Job:%s Id:%d", run_path, job->torque_jobnr); + free(local_job_name); if (job->torque_jobnr > 0) return job; From 688451eac1fd0160c8ce052fe381fe56fc2b185f Mon Sep 17 00:00:00 2001 From: Andreas Eknes Lie Date: Tue, 19 Sep 2023 13:03:12 +0200 Subject: [PATCH 19/19] Inline lsf_driver_shell_init --- src/clib/lib/job_queue/lsf_driver.cpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/clib/lib/job_queue/lsf_driver.cpp b/src/clib/lib/job_queue/lsf_driver.cpp index b39482d5e02..9de28ee2641 100644 --- a/src/clib/lib/job_queue/lsf_driver.cpp +++ b/src/clib/lib/job_queue/lsf_driver.cpp @@ -918,19 +918,15 @@ void lsf_driver_set_bjobs_refresh_interval(lsf_driver_type *driver, driver->bjobs_refresh_interval = refresh_interval; } -static void lsf_driver_shell_init(lsf_driver_type *lsf_driver) { - lsf_driver->last_bjobs_update = time(nullptr); - lsf_driver->bjobs_cache = hash_alloc(); - lsf_driver->my_jobs = hash_alloc(); -} - bool lsf_driver_has_project_code(const lsf_driver_type *driver) { return (driver->project_code); } void *lsf_driver_alloc() { lsf_driver_type *lsf_driver = new lsf_driver_type(); - lsf_driver_shell_init(lsf_driver); + lsf_driver->last_bjobs_update = time(nullptr); + lsf_driver->bjobs_cache = hash_alloc(); + lsf_driver->my_jobs = hash_alloc(); lsf_driver_set_option(lsf_driver, LSF_SERVER, NULL); lsf_driver_set_option(lsf_driver, LSF_RSH_CMD, DEFAULT_RSH_CMD);