Skip to content

Commit

Permalink
Remove unused includes and use default struct initialization
Browse files Browse the repository at this point in the history
* Remove includes from enkf_obs
* Remove unused JOB_DRIVER_ENUM_SIZE
* Fix includes in job_node
* Fix includes in slurm_driver
* Fix includes in torque_driver
* Fix includes in local_driver
* Fix includes in lsf_driver
* Fix torque_driver header includes
* Fix lsf driver includes
* Fix queue_driver includes
* Replace NULL with nullptr in queue_driver
* Remove typedef to unused struct
* Initialize lsf struct
* Initialize torque struct and replace NULL nullptr
* Initialize queue_driver
* Simplification of torque_driver
* Inline lsf_driver_shell_init
  • Loading branch information
andreas-el authored Sep 19, 2023
1 parent 5f097a8 commit 080c8df
Show file tree
Hide file tree
Showing 10 changed files with 125 additions and 225 deletions.
14 changes: 1 addition & 13 deletions src/clib/lib/enkf/enkf_obs.cpp
Original file line number Diff line number Diff line change
@@ -1,22 +1,10 @@
#include <cmath>
#include <cppitertools/enumerate.hpp>
#include <map>
#include <utility>

#include <ert/ecl/ecl_grid.h>
#include <ert/ecl/ecl_sum.h>
#include <ert/util/hash.h>
#include <ert/util/type_vector_functions.h>
#include <ert/util/vector.h>

#include <ert/ecl/ecl_grid.h>
#include <ert/ecl/ecl_sum.h>
#include <ert/except.hpp>
#include <ert/python.hpp>
#include <ert/res_util/string.hpp>
#include <ert/util/hash.h>
#include <ert/util/type_vector_functions.h>
#include <ert/util/vector.h>
#include <map>

ERT_CLIB_SUBMODULE("enkf_obs", m) {
using namespace py::literals;
Expand Down
2 changes: 0 additions & 2 deletions src/clib/lib/include/ert/job_queue/queue_driver.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,6 @@ typedef enum {
SLURM_DRIVER = 5
} job_driver_type;

#define JOB_DRIVER_ENUM_SIZE 5

typedef struct queue_driver_struct queue_driver_type;

typedef void *(submit_job_ftype)(void *data, const char *cmd, int num_cpu,
Expand Down
1 change: 0 additions & 1 deletion src/clib/lib/include/ert/job_queue/slurm_driver.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
*/

typedef struct slurm_driver_struct slurm_driver_type;
typedef struct slurm_job_struct slurm_job_type;

#define SLURM_SBATCH_OPTION "SBATCH"
#define SLURM_SCANCEL_OPTION "SCANCEL"
Expand Down
2 changes: 1 addition & 1 deletion src/clib/lib/include/ert/job_queue/torque_driver.hpp
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#ifndef TORQUE_DRIVER_H
#define TORQUE_DRIVER_H

#include <stdio.h>
#include <cstdio>
#include <string>
#include <vector>

Expand Down
3 changes: 1 addition & 2 deletions src/clib/lib/job_queue/job_node.cpp
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
#include <filesystem>
#include <string>

#include <cstdlib>
#include <pthread.h>
#include <stdbool.h>
#include <stdlib.h>
#include <tuple>

#include <ert/logging.hpp>
Expand Down
4 changes: 2 additions & 2 deletions src/clib/lib/job_queue/local_driver.cpp
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#include <csignal>
#include <cstdlib>
#include <mutex>
#include <optional>
#include <signal.h>
#include <stdlib.h>
#include <sys/wait.h>
#include <thread>

Expand Down
102 changes: 36 additions & 66 deletions src/clib/lib/job_queue/lsf_driver.cpp
Original file line number Diff line number Diff line change
@@ -1,30 +1,26 @@
#include <algorithm>
#include <filesystem>
#include <fstream>
#include <map>
#include <sstream>
#include <string>
#include <vector>

#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <dlfcn.h>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <map>
#include <pthread.h>
#include <string>
#include <unistd.h>
#include <vector>

#include <ert/except.hpp>
#include <ert/logging.hpp>
#include <ert/res_util/string.hpp>
#include <ert/util/hash.hpp>
#include <ert/util/util.hpp>

#include <ert/job_queue/lsf_driver.hpp>
#include <ert/job_queue/lsf_job_stat.hpp>
#include <ert/job_queue/queue_driver.hpp>
#include <ert/job_queue/spawn.hpp>
#include <ert/logging.hpp>
#include <ert/python.hpp>
#include <ert/res_util/string.hpp>
#include <ert/util/hash.hpp>
#include <ert/util/util.hpp>

namespace fs = std::filesystem;
static auto logger = ert::get_logger("job_queue.lsf_driver");
Expand Down Expand Up @@ -96,44 +92,43 @@ static auto logger = ert::get_logger("job_queue.lsf_driver");
#define DEFAULT_BHIST_CMD "bhist"

struct lsf_job_struct {
long int lsf_jobnr;
long int lsf_jobnr = 0;
/** Used to look up the job status in the bjobs_cache hash table */
char *lsf_jobnr_char;
char *job_name;
char *lsf_jobnr_char = nullptr;
char *job_name = nullptr;
};

struct lsf_driver_struct {
char *queue_name;
char *resource_request;
std::vector<std::string> exclude_hosts;
char *login_shell;
char *project_code;
pthread_mutex_t submit_lock;

lsf_submit_method_enum submit_method;
int submit_sleep;

int error_count;
int submit_error_sleep;
char *queue_name = nullptr;
char *resource_request = nullptr;
std::vector<std::string> exclude_hosts{};
char *login_shell = nullptr;
char *project_code = nullptr;
pthread_mutex_t submit_lock{};

lsf_submit_method_enum submit_method = LSF_SUBMIT_LOCAL_SHELL;
int submit_sleep = 0;
int error_count = 0;
int submit_error_sleep = SUBMIT_ERROR_SLEEP * 1000000;

/*-----------------------------------------------------------------*/
/* Fields used by the shell based functions */
bool debug_output;
int bjobs_refresh_interval;
time_t last_bjobs_update;
bool debug_output = false;
int bjobs_refresh_interval = 0;
time_t last_bjobs_update = time(nullptr);
/** A hash table of all jobs submitted by this ERT instance - to ensure
* that we do not check status of old jobs in e.g. ZOMBIE status. */
hash_type *my_jobs;
/** The output of calling bjobs is cached in this table. */
hash_type *bjobs_cache;
/** Only one thread should update the bjobs_chache table. */
pthread_mutex_t bjobs_mutex;
char *remote_lsf_server;
char *rsh_cmd;
char *bsub_cmd;
char *bjobs_cmd;
char *bkill_cmd;
char *bhist_cmd;
pthread_mutex_t bjobs_mutex{};
char *remote_lsf_server = nullptr;
char *rsh_cmd = nullptr;
char *bsub_cmd = nullptr;
char *bjobs_cmd = nullptr;
char *bkill_cmd = nullptr;
char *bhist_cmd = nullptr;
};

const std::map<const std::string, int> status_map = {
Expand Down Expand Up @@ -161,8 +156,6 @@ const std::map<const int, const job_status_type> convert_status_map = {

static lsf_job_type *lsf_job_alloc(const char *job_name) {
auto job = new lsf_job_type;
job->lsf_jobnr = 0;
job->lsf_jobnr_char = NULL;
job->job_name = util_alloc_string_copy(job_name);
return job;
}
Expand Down Expand Up @@ -925,38 +918,15 @@ void lsf_driver_set_bjobs_refresh_interval(lsf_driver_type *driver,
driver->bjobs_refresh_interval = refresh_interval;
}

static void lsf_driver_shell_init(lsf_driver_type *lsf_driver) {
lsf_driver->last_bjobs_update = time(NULL);
lsf_driver->bjobs_cache = hash_alloc();
lsf_driver->my_jobs = hash_alloc();
lsf_driver->bsub_cmd = NULL;
lsf_driver->bjobs_cmd = NULL;
lsf_driver->bkill_cmd = NULL;
lsf_driver->bhist_cmd = NULL;

pthread_mutex_init(&lsf_driver->bjobs_mutex, NULL);
}

bool lsf_driver_has_project_code(const lsf_driver_type *driver) {
return (driver->project_code);
}

void *lsf_driver_alloc() {
lsf_driver_type *lsf_driver = new lsf_driver_type();

lsf_driver->submit_method = LSF_SUBMIT_INVALID;
lsf_driver->login_shell = NULL;
lsf_driver->queue_name = NULL;
lsf_driver->remote_lsf_server = NULL;
lsf_driver->rsh_cmd = NULL;
lsf_driver->resource_request = NULL;
lsf_driver->project_code = NULL;
lsf_driver->error_count = 0;
lsf_driver->submit_error_sleep = SUBMIT_ERROR_SLEEP * 1000000;
pthread_mutex_init(&lsf_driver->submit_lock, NULL);

lsf_driver_shell_init(lsf_driver);
lsf_driver->submit_method = LSF_SUBMIT_LOCAL_SHELL;
lsf_driver->last_bjobs_update = time(nullptr);
lsf_driver->bjobs_cache = hash_alloc();
lsf_driver->my_jobs = hash_alloc();

lsf_driver_set_option(lsf_driver, LSF_SERVER, NULL);
lsf_driver_set_option(lsf_driver, LSF_RSH_CMD, DEFAULT_RSH_CMD);
Expand Down
37 changes: 11 additions & 26 deletions src/clib/lib/job_queue/queue_driver.cpp
Original file line number Diff line number Diff line change
@@ -1,13 +1,9 @@
#include <stdlib.h>
#include <string.h>

#include <ert/util/util.hpp>

#include <ert/job_queue/local_driver.hpp>
#include <ert/job_queue/lsf_driver.hpp>
#include <ert/job_queue/queue_driver.hpp>
#include <ert/job_queue/slurm_driver.hpp>
#include <ert/job_queue/torque_driver.hpp>
#include <ert/util/util.hpp>

/*
This file implements the datatype queue_driver_type which is an
Expand Down Expand Up @@ -37,17 +33,17 @@
struct queue_driver_struct {
/** Function pointers - pointing to low level functions in the
* implementations of e.g. lsf_driver. */
submit_job_ftype *submit;
free_job_ftype *free_job;
kill_job_ftype *kill_job;
get_status_ftype *get_status;
free_queue_driver_ftype *free_driver;
set_option_ftype *set_option;
get_option_ftype *get_option;
init_option_list_ftype *init_options;
submit_job_ftype *submit = nullptr;
free_job_ftype *free_job = nullptr;
kill_job_ftype *kill_job = nullptr;
get_status_ftype *get_status = nullptr;
free_queue_driver_ftype *free_driver = nullptr;
set_option_ftype *set_option = nullptr;
get_option_ftype *get_option = nullptr;
init_option_list_ftype *init_options = nullptr;

/** Driver specific data - passed as first argument to the driver functions above. */
void *data;
void *data = nullptr;
};

bool queue_driver_set_option(queue_driver_type *driver, const char *option_key,
Expand All @@ -65,18 +61,7 @@ bool queue_driver_set_option(queue_driver_type *driver, const char *option_key,
NOT properly initialized and NOT ready for use.
*/
static queue_driver_type *queue_driver_alloc_empty() {
auto driver = new queue_driver_type;
driver->submit = NULL;
driver->get_status = NULL;
driver->kill_job = NULL;
driver->free_job = NULL;
driver->free_driver = NULL;
driver->get_option = NULL;
driver->set_option = NULL;
driver->data = NULL;
driver->init_options = NULL;

return driver;
return new queue_driver_type;
}

// The driver created in this function has all the function pointers
Expand Down
22 changes: 9 additions & 13 deletions src/clib/lib/job_queue/slurm_driver.cpp
Original file line number Diff line number Diff line change
@@ -1,28 +1,24 @@
#include <pthread.h>
#include <pwd.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <pthread.h>
#include <pwd.h>
#include <set>
#include <stdexcept>
#include <string>
#include <thread>
#include <unistd.h>
#include <unordered_map>
#include <unordered_set>
#include <vector>

#include <ert/logging.hpp>
#include <ert/util/stringlist.hpp>
#include <ert/util/util.hpp>

#include <ert/job_queue/queue_driver.hpp>
#include <ert/job_queue/slurm_driver.hpp>
#include <ert/job_queue/spawn.hpp>
#include <ert/logging.hpp>
#include <ert/python.hpp>
#include <ert/util/stringlist.hpp>
#include <ert/util/util.hpp>

static auto logger = ert::get_logger("job_queue.slurm_driver");

Expand Down
Loading

0 comments on commit 080c8df

Please sign in to comment.