diff --git a/R/attach.R b/R/attach.R index 5ff6fa4..a30665c 100644 --- a/R/attach.R +++ b/R/attach.R @@ -8,70 +8,57 @@ #' slurmtools_options_message() #' } slurmtools_options_message <- function() { - set_header <- cli::rule( - left = cli::style_bold("Set slurmtools options") - ) - - unset_header <- cli::rule( - left = cli::style_bold("Needed slurmtools options") - ) - set_options <- c() unset_options <- c() - print_set <- FALSE - print_unset <- FALSE + # Check for each used options tmpl_path <- getOption('slurmtools.slurm_job_template_path') if (is.null(tmpl_path)) { unset_options <- c(unset_options, "option('slurmtools.slurm_job_template_path') is not set.") - print_unset <- TRUE } else { set_options <- c(set_options, paste("slurmtools.slurm_jon_template_path:", tmpl_path)) - print_set <- TRUE } root <- getOption('slurmtools.submission_root') if (is.null(root)) { unset_options <- c(unset_options, "option('slurmtools.submission_root') is not set.") - print_unset <- TRUE } else { set_options <- c(set_options, paste("slurmtools.submission_root:", root)) - print_set <- TRUE } - set_bullets <- paste0( - cli::col_green(cli::symbol$tick), " ", set_options, collapse = "\n" - ) + bbi_config <- getOption("slurmtools.bbi_config_path") + if (is.null(bbi_config)) { + unset_options <- c(unset_options, "option('slurmtools.bbi_config_path') is not set.") + } else { + set_options <- c(set_options, paste("slurmtools.bbi_config_path:", bbi_config)) + } - unset_bullets <- paste0( - cli::col_red(cli::symbol$cross), " ", unset_options, collapse = "\n" - ) + #format .onAttach message + msg <- "" + if (length(set_options)) { + msg <- paste0( + msg, + cli::rule( + left = cli::style_bold("Set slurmtools options")), "\n", + paste0( + cli::col_green(cli::symbol$tick), " ", set_options, collapse = "\n"), "\n" + ) + } - unset <- paste0( - cli::col_cyan(cli::symbol$info), " ", - cli::format_inline("Please set all options for job submission defaults to work.") - ) - if (print_set) { - if (print_unset) { + if (length(unset_options)) { + msg <- paste0( + msg, + cli::rule( + left = cli::style_bold("Needed slurmtools options")), "\n", paste0( - set_header, "\n", - set_bullets, "\n", - unset_header, "\n", - unset_bullets, "\n", - unset - ) - } else { + cli::col_red(cli::symbol$cross), " ", unset_options, collapse = "\n"), "\n", paste0( - set_header, "\n", - set_bullets, "\n" + cli::col_cyan(cli::symbol$info), " ", + cli::format_inline("Please set all options for job submission defaults to work.") ) - } - } else { - paste0( - unset_header, "\n", - unset_bullets, "\n", - unset ) } + + msg } diff --git a/R/nmm-config.R b/R/nmm-config.R index 529dc83..d947343 100644 --- a/R/nmm-config.R +++ b/R/nmm-config.R @@ -11,6 +11,7 @@ #' @param level What level to log at. Default is info. Available options are Trace, Debug, Info, Warn, Fatal #' @param email if alert is set to Slack, this should be the email associated with slack to get messages sent directly to you. #' @param threads number of threads if running a parallel job. Default is 1 +#' @param topic the ntfy.sh topic to send alerts to. #' #' @return none #' @keywords internal @@ -30,7 +31,8 @@ generate_nmm_config <- function( alert = "None", level = "Debug", email = "", - threads = 1 + threads = 1, + topic = "" ) { if (model_number == "") { model_number <- basename(.mod$absolute_model_path) @@ -60,7 +62,8 @@ generate_nmm_config <- function( alert = alert, level = level, email = email, - threads = threads + threads = threads, + topic = topic ) config_toml_path <- paste0(.mod$absolute_model_path, ".toml") diff --git a/R/submit-model.R b/R/submit-model.R index d2d7bc8..8f87df3 100644 --- a/R/submit-model.R +++ b/R/submit-model.R @@ -8,6 +8,7 @@ #' @param ... arguments to pass to processx::run #' @param slurm_job_template_path path to slurm job template #' @param submission_root directory to track job submission scripts and output +#' @param bbi_config_path path to bbi.yaml file for bbi configuration #' @param slurm_template_opts choose slurm template #' #' @export @@ -20,6 +21,7 @@ submit_nonmem_model <- ..., slurm_job_template_path = getOption('slurmtools.slurm_job_template_path'), submission_root = getOption('slurmtools.submission_root'), + bbi_config_path = getOption('slurmtools.bbi_config_path'), slurm_template_opts = list()) { if (is.null(partition)) { @@ -58,14 +60,27 @@ submit_nonmem_model <- rlang::warn(sprintf("config.toml file not found, if submitting the job with nmm this is required. Please run generate_nmm_config()")) } + if (is.null(slurm_template_opts$nmm_exe_path)) { + nmm_exe_path <- Sys.which("nmm") + } else { + nmm_exe_path <- slurm_template_opts$nmm_exe_path + } + if (is.null(slurm_template_opts$bbi_exe_path)) { + bbi_exe_path <- Sys.which("bbi") + } else { + bbi_exe_path <- slurm_template_opts$bbi_exe_path + } + default_template_list = list( partition = partition, parallel = parallel, ncpu = ncpu, job_name = sprintf("%s-nonmem-run", basename(.mod$absolute_model_path)), + bbi_exe_path = bbi_exe_path, + bbi_config_path = bbi_config_path, model_path = .mod$absolute_model_path, config_toml_path = config_toml_path, - nmm_exe_path = Sys.which("nmm") + nmm_exe_path = nmm_exe_path ) template_list = c( diff --git a/man/generate_nmm_config.Rd b/man/generate_nmm_config.Rd index 7888c86..2c1676f 100644 --- a/man/generate_nmm_config.Rd +++ b/man/generate_nmm_config.Rd @@ -15,7 +15,8 @@ generate_nmm_config( alert = "None", level = "Debug", email = "", - threads = 1 + threads = 1, + topic = "" ) } \arguments{ @@ -40,6 +41,8 @@ generate_nmm_config( \item{email}{if alert is set to Slack, this should be the email associated with slack to get messages sent directly to you.} \item{threads}{number of threads if running a parallel job. Default is 1} + +\item{topic}{the ntfy.sh topic to send alerts to.} } \value{ none diff --git a/man/submit_nonmem_model.Rd b/man/submit_nonmem_model.Rd index ba8f723..e2c1f8d 100644 --- a/man/submit_nonmem_model.Rd +++ b/man/submit_nonmem_model.Rd @@ -13,6 +13,7 @@ submit_nonmem_model( ..., slurm_job_template_path = getOption("slurmtools.slurm_job_template_path"), submission_root = getOption("slurmtools.submission_root"), + bbi_config_path = getOption("slurmtools.bbi_config_path"), slurm_template_opts = list() ) } @@ -33,6 +34,8 @@ submit_nonmem_model( \item{submission_root}{directory to track job submission scripts and output} +\item{bbi_config_path}{path to bbi.yaml file for bbi configuration} + \item{slurm_template_opts}{choose slurm template} } \description{ diff --git a/vignettes/Running-nonmem.Rmd b/vignettes/Running-nonmem.Rmd index c09e263..360ac6a 100644 --- a/vignettes/Running-nonmem.Rmd +++ b/vignettes/Running-nonmem.Rmd @@ -44,7 +44,9 @@ install.packages("slurmtools") library(slurmtools) ``` -We are given a message when loading slurmtools that some options are not set and that default job submission will not work without them. These options are used for default arguments in the `submit_nonmem_model` function. Running `?submit_nonmem_model` we can see the documentation![Help view for submit_nonmem_model function](data/images/submit_nonmem_model_help.png) +We are given a message when loading slurmtools that some options are not set and that default job submission will not work without them. These options are used for default arguments in the `submit_nonmem_model` function. Running `?submit_nonmem_model` we can see the documentation + +![Help view for `submit_nonmem_model` function](data/images/submit_nonmem_model_help.png) This function uses the inputs to populate a template Bash shell script that submits the NONMEM job to slurm. A default template file is supplied with the Project Starter and it can be modified to do additional tasks as long as they are possible within Bash. @@ -56,6 +58,8 @@ list( parallel = parallel, ncpu = ncpu, job_name = sprintf("%s-nonmem-run", basename(.mod$absolute_model_path)), + bbi_exe_path = Sys.which("bbi"), + bbi_config_path = bbi_config_path, model_path = .mod$absolute_model_path, config_toml_path = config_toml_path, nmm_exe_path = Sys.which("nmm") @@ -70,6 +74,10 @@ list( - `job_name` is created from the `.mod` argument supplied to `submit_nonmem_model` +- `bbi_exe_path` is determined via \`Sys.which("bbi") + +- `bbi_config_path` is determined via getOption("slurmtools.bbi_config_path") + - `model_path` is determined from the `.mod` argument supplied to `submit_nonmem_model` - `config_toml_path` is determined from the `.mod` argument supplied to `submit_nonmem_model` and is requried to use `nmm` (NONMEM monitor) @@ -80,7 +88,7 @@ If you need to feed more arguments to the template you simply supply them in the ## Submitting a NONMEM job with `bbi` -To submit a NONMEM job, we need to supply either the path to a mod file or create a model object from `bbr`, and supply a `slurm-template.tmpl` file. To use `bbi` we also need a `bbi.yaml` file, which I've also supplied in `/model/nonmem/bbi.yaml`. +To submit a NONMEM job, we need to supply either the path to a mod file or create a model object from `bbr`, and supply a `slurm-template.tmpl` file. To use `bbi` we also need a `bbi.yaml` file, which I've also supplied in `/model/nonmem/bbi.yaml` (and is also supplied with the R project starter). Here is an example of a template file that will call `bbi`: @@ -105,9 +113,13 @@ Here is an example of a template file that will call `bbi`: {{/parallel}} ``` -This file will call `bbi` to run our supplied model (`{{model_path}}.mod`) if `ncpu > 1` then parallel will be true and the code between `{{#parallel}}` and `{{/parallel}}` will be populated. if `ncpu = 1` then parallel will be false and the code between `{{^parallel}}` and `{{/parallel}}` will be populated. Notice we'll have to supply the `bbi_exe_path` for it to start the NONMEM run because `submit_nonmem_model` does not supply that variable to the template by default. +This file will call `bbi` to run our supplied model (`{{model_path}}.mod`) if `ncpu > 1` then parallel will be true and the code between `{{#parallel}}` and `{{/parallel}}` will be populated. if `ncpu = 1` then parallel will be false and the code between `{{^parallel}}` and `{{/parallel}}` will be populated. By default, `submit_nonmem_model` will inject `Sys.which("bbi")` into the template, so if `bbi` is not on your path we'll have to supply the `bbi_exe_path` for it to start the NONMEM run. -We will use a few different template files with different functionality so we'll inject those template file paths to `submit_nonmem_model`. However, we'll use the `submission-log` directory for the output, so we'll set that option so `submit_nonmem_model` default can be used. The slurm template files are saved in `~/model/nonmem/` Additionally, there is a simple NONMEM control stream in `1001.mod` in the same directory that we can use for testing. +```{r} +Sys.which("bbi") +``` + +We will use a few different template files with different functionality so we'll inject those template file paths to `submit_nonmem_model`. However, we'll use the `submission-log` directory for the output, so we'll set that option as well as `bbi_config_path` so `submit_nonmem_model` defaults can be used. The slurm template files are saved in `~/model/nonmem/` Additionally, there is a simple NONMEM control stream in `1001.mod` in the same directory that we can use for testing. ```{r} library(bbr) @@ -116,9 +128,10 @@ library(here) nonmem = file.path(here::here(), "vignettes", "model", "nonmem") options('slurmtools.submission_root' = file.path(nonmem, "submission-log")) +options('slurmtools.bbi_config_path' = file.path(nonmem, "bbi.yaml")) ``` -To create the bbr model object, we need to have both `1001.mod` and `1001.yaml` which contains metadata about the model in the supplied directory (`/model/nonmem/`). +To create the `bbr` model object, we need to have both `1001.mod` and `1001.yaml` which contains metadata about the model in the supplied directory (`./model/nonmem/`). We'll check for mod_number.yaml and if it exists, read in the model otherwise create it and then read it. ```{r} mod_number <- "1001" @@ -130,21 +143,18 @@ if (file.exists(file.path(nonmem, paste0(mod_number, ".yaml")))) { } ``` -We can now submit the job and point to the template file in `model/nonmem/slurm-job-bbi.tmpl` +We can now submit the job and point to the template file in `model/nonmem/slurm-job-bbi.tmpl`. ```{r} submission <- slurmtools::submit_nonmem_model( mod, slurm_job_template_path = file.path(nonmem, "slurm-job-bbi.tmpl"), - slurm_template_opts = list( - bbi_exe_path = Sys.which("bbi"), - bbi_config_path = file.path(nonmem, "bbi.yaml")) ) submission ``` -Notice we get a Warning about a config.toml file not being found. I'll expand on that later, but for now we see a `status` with an exit code of 0 suggesting a successful command, and the `stdout` gives us the batch job number. We can use `slurmtools::get_slurm_jobs()` to monitor the status of the job. Here, I've filtered the results to only show the job I just submitted above. +Notice we get a Warning about a config.toml file not being found. I'll expand on that later, but for now we see a `status` with an exit code of 0 suggesting a successful command, and the `stdout` gives us the batch job number. We can use `slurmtools::get_slurm_jobs()` to monitor the status of the job. Here, we can supply the user = "matthews" argument to filter to just the jobs I've submitted. ```{r} slurmtools::get_slurm_jobs(user = 'matthews') @@ -202,8 +212,6 @@ submission_ntfy <- slurmtools::submit_nonmem_model( slurm_job_template_path = file.path(nonmem, "slurm-job-bbi-ntfy.tmpl"), overwrite = TRUE, slurm_template_opts = list( - bbi_exe_path = Sys.which("bbi"), - bbi_config_path = file.path(nonmem, "bbi.yaml"), ntfy = "ntfy_demo") ) diff --git a/vignettes/custom-alerts.Rmd b/vignettes/custom-alerts.Rmd index 9826a40..dd90ae7 100644 --- a/vignettes/custom-alerts.Rmd +++ b/vignettes/custom-alerts.Rmd @@ -10,6 +10,17 @@ editor_options: wrap: 72 --- +```{r, include = FALSE} +#removing generated files from running this vignette +nonmem <- file.path("model", "nonmem") + +unlink(file.path(nonmem, "1001"), recursive = TRUE) +unlink(file.path(nonmem, "1001.yaml")) +unlink(file.path(nonmem, "1001.toml")) +unlink(file.path(nonmem, "submission-log"), recursive = TRUE) +unlink(file.path(nonmem, "in_progress"), recursive = TRUE) +``` + ```{r, include = FALSE} knitr::opts_chunk$set( collapse = TRUE, @@ -28,9 +39,13 @@ options('slurmtools.submission_root' = file.path(nonmem, "submission-log")) ## Submitting a NONMEM job with nmm -Instead of using bbi we can use nmm (NONMEM Monitor) which currently has -some additional functionality of sending slack notifications as well as -gradient monitoring. +Instead of using bbi we can use `nmm` ([NONMEM +Monitor](https://github.com/A2-ai/nonmem-monitor)) which currently has +some additional functionality of sending notifications about zero +gradients, missing -1E9 lines in ext file, and some very basic control +stream errors. Currently, only slack or [ntfy.sh](ntfy.sh) is supported +for receiving notifications. To use `nmm` you can install the latest +release from the github repository linked above. We can update the template file accordingly: @@ -46,11 +61,11 @@ We can update the template file accordingly: ``` default, `submit_nonmem_model` will provide `nmm_exe_path` and -`config_toml_path` to the template. Behind the scenes `nmm_exe_path` is -determined with `Sys.which("nmm")` which may or may not give you the -path to the nmm binary if it is on your path or not. We can inject the -`nmm_exe_path` like we did with `bbi_exe_path` and assume it's not on -our path. +`config_toml_path` to the template. Just like with `bbi_exe_path`, +`nmm_exe_path` is determined with `Sys.which("nmm")` which may or may +not give you the path to the nmm binary if it is on your path or not. We +can inject the `nmm_exe_path` like we did with `bbi_exe_path` and assume +it's not on our path. Remember that error about the `config.toml` file not existing? Well this is the reason. This file controls what `nmm` will monitor and where to @@ -74,66 +89,163 @@ slurmtools::generate_nmm_config(mod) ``` This generates the following toml file. Notice that alert is set to -'None', and email is empty. Since we're in vignettes we'll need to -update the watched_dir and output_dir. +'None', and both email and topic are empty. Since we're in vignettes +we'll need to update the `watched_dir` and `output_dir`. + +``` 1001.toml +model_number = '1001' +files_to_track = [ 'lst', 'ext', 'grd' ] +tmp_dir = '/tmp' +watched_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/model/nonmem' +output_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/model/nonmem/in_progress' +poll_duration = 1 +alert = 'None' +level = 'Debug' +email = '' +threads = 1 +topic = '' +``` -Additionally, If we want to take advatnage of nmm's slack notifications -we'll have to add a few arguments to the generate_nmm_config function. +```{r} +slurmtools::generate_nmm_config( + mod, + watched_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem", + output_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress") +``` + +This updates the `1001.toml` config file to: ``` 1001.toml -model_number = '1001' -files_to_track = [ 'lst', 'ext', 'grd' ] -tmp_dir ='/tmp' -watched_dir ='/cluster-data/user-homes/matthews/Packages/slurmtools/model/nonmem' -output_dir ='/cluster-data/user-homes/matthews/Packages/slurmtools/model/nonmem/in_progress' -poll_duration = 1 -alert = 'None' -level = 'Debug' -email = '' +model_number = '1001' +files_to_track = [ 'lst', 'ext', 'grd' ] +tmp_dir = '/tmp' +watched_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem' +output_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress' +poll_duration = 1 +alert = 'None' +level = 'Debug' +email = '' threads = 1 +topic = '' +``` + +We can now run `submit_nonmem_model` and get essentially the same +behavior as running with `bbi`. On linux `~/.local/bin/` will be on your path so saving binaries there is a good approach. + +```{r} +submission_nmm <- slurmtools::submit_nonmem_model( + mod, + overwrite = TRUE, + slurm_job_template_path = file.path(nonmem, "slurm-job-nmm.tmpl"), + slurm_template_opts = list( + nmm_exe_path = normalizePath("~/.local/bin/nmm")) +) + +submission_nmm +``` + +```{r} +slurmtools::get_slurm_jobs() +``` + +The one difference between using `nmm` compared to `bbi` is that a new +directory is created that contains a log file that caught some issues +with our run. This file is updated as nonmem is running and monitors +gradient values, parameters that hit zero, as well as other errors from +bbi. Looking at the first few lines we can see that `bbi` was +successfully able to call nonmem. We also see an info level log that +OMEGA(2,1) has 0 value – in our mod file we don't specify any omega +values off the diagonal so these are fixed at 0. Finally we see that +GRD(6) hit 0 relatively early in the run. + +``` vignettes/model/nonmem/in_progress/1001/modeling_run_20240827201226.log +20:12:36 [INFO] bbi log: time="2024-08-27T20:12:36Z" level=info msg="Successfully loaded default configuration from /cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/bbi.yaml" +20:12:36 [INFO] bbi log: time="2024-08-27T20:12:36Z" level=info msg="Beginning Local Path" +20:12:36 [INFO] bbi log: time="2024-08-27T20:12:36Z" level=info msg="A total of 1 models have completed the initial preparation phase" +20:12:36 [INFO] bbi log: time="2024-08-27T20:12:36Z" level=info msg="[1001] Beginning local work phase" +20:12:58 [INFO] OMEGA(2,1) has 0 value +20:12:58 [INFO] SIGMA(2,1) has 0 value +20:13:00 [INFO] SIGMA(2,1) has 0 value +20:13:00 [INFO] OMEGA(2,1) has 0 value +20:13:04 [INFO] SIGMA(2,1) has 0 value +20:13:04 [INFO] OMEGA(2,1) has 0 value +20:13:04 [WARN] "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/1001/1001.grd" has 0 gradient for parameter: GRD(6) +``` + +After a run has finished several messages are sent to the log after a +final check of the files listed in the `files_to_track` field of the +`1001.toml` file. + +``` vignettes/model/nonmem/in_progress/1001/modeling_run_20240827201226.log +20:13:16 [INFO] Received Exit code: exit status: 0 +20:13:16 [WARN] 1001.ext: Missing ext final output lines. Observed lines were: [-1000000000.0, -1000000004.0, -1000000006.0, -1000000007.0] +20:13:16 [WARN] "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/1001/1001.grd": The following parameters hit zero gradient through the run: ["GRD(6)"] ``` +We see that GRD(6) hit zero during the run and that only a subset of the +-1E9 lines were present in the .ext file. + +## Getting alerted during a run + +Like we did with `bbi` and altering the slurm template file to get +notifications from [ntfy.sh](ntfy.sh) `nmm` has this feature built in! +The messages in the log file that relate to zero gradients, missing -1E9 +lines, and 0 parameter values can also be sent to ntfy by altering the +`1001.toml` file. We can get these alerts in real time without having to +dig through a noisy log file. + +Let's update our call to `generate_nmm_config` to have `nmm` send +notifications to the `NONMEMmonitor` topic on [ntfy.sh](ntfy.sh). + ```{r} slurmtools::generate_nmm_config( mod, - alert = "slack", - email ="matthews@a2-ai.com", + alert = "Ntfy", + topic = "NONMEMmonitor", watched_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem", output_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress") ``` +This updates the `1001.toml` file to this: + ``` 1001.toml -model_number = '1001' -files_to_track = [ 'lst', 'ext', 'grd' ] -tmp_dir ='/tmp' +model_number = '1001' +files_to_track = [ 'lst', 'ext', 'grd' ] +tmp_dir = '/tmp' watched_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem' output_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress' -poll_duration = 1 -alert = 'Slack' -level = 'Debug' -email ='matthews@a2-ai.com' +poll_duration = 1 +alert = 'Ntfy' +level = 'Debug' +email = '' threads = 1 +topic = 'NONMEMmonitor' ``` -We can now run `submit_nonmem_model` and get slack notifications about -our run. +When we re-run the `submit_nonmem_model` call we will now get ntfy +notifications. One thing to note is that `nmm` will print full paths in +the log, but will only send notifications with the `model_number` (or +`model_number.file_extension`). ```{r} -submission_nmm <- slurmtools::submit_nonmem_model( mod, overwrite = -TRUE, slurm_job_template_path = file.path(nonmem, "slurm-job-nmm.tmpl"), -slurm_template_opts = list(nmm_exe_path = -normalizePath("~/.local/bin/nmm")) ) +submission_nmm <- slurmtools::submit_nonmem_model( + mod, + overwrite = TRUE, + slurm_job_template_path = file.path(nonmem, "slurm-job-nmm.tmpl"), + slurm_template_opts = list( + nmm_exe_path = normalizePath("~/.local/bin/nmm-x86_64-unknown-linux-gnu/nmm")) +) submission_nmm ``` -With `alert = 'Slack'` and `email` set in the config.toml file `nmm` -should send slack notifications directly to you when a NONMEM run starts -and it will reply to that message with notifications if any gradients -hit 0 and when the run finishes it checks if all -1E9 lines are present -in the .ext file and gives another message about any parameters that hit -0 gradient.![slack notifications from -nmm](data/images/nmm_slack_notifications.png) +```{r} +slurmtools::get_slurm_jobs(user = "matthews") +``` + +This gives us the notifications in a much more digestible format + +![nmm ntfy.sh alerts](data/images/nmm_ntfy_alerts.png) ```{r, include = FALSE} #cancelling any running nonmem jobs diff --git a/vignettes/data/images/generate_nmm_config_help.png b/vignettes/data/images/generate_nmm_config_help.png index 86bf514..0c8ace9 100644 Binary files a/vignettes/data/images/generate_nmm_config_help.png and b/vignettes/data/images/generate_nmm_config_help.png differ diff --git a/vignettes/data/images/nmm_ntfy_alerts.png b/vignettes/data/images/nmm_ntfy_alerts.png new file mode 100644 index 0000000..12a4f5d Binary files /dev/null and b/vignettes/data/images/nmm_ntfy_alerts.png differ diff --git a/vignettes/data/images/nmm_slack_notifications.png b/vignettes/data/images/nmm_slack_notifications.png index bd161d4..8fdb45b 100644 Binary files a/vignettes/data/images/nmm_slack_notifications.png and b/vignettes/data/images/nmm_slack_notifications.png differ diff --git a/vignettes/data/images/submit_nonmem_model_help.png b/vignettes/data/images/submit_nonmem_model_help.png index 10053ae..bc1b936 100644 Binary files a/vignettes/data/images/submit_nonmem_model_help.png and b/vignettes/data/images/submit_nonmem_model_help.png differ diff --git a/vignettes/slack-alerts.Rmd b/vignettes/slack-alerts.Rmd new file mode 100644 index 0000000..f78bdca --- /dev/null +++ b/vignettes/slack-alerts.Rmd @@ -0,0 +1,144 @@ +--- +title: "slack-alerts" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{slack-alerts} + %\VignetteEncoding{UTF-8} + %\VignetteEngine{knitr::rmarkdown} +editor_options: + markdown: + wrap: 72 +--- + +```{r, include = FALSE} +#removing generated files from running this vignette +nonmem <- file.path("model", "nonmem") + +unlink(file.path(nonmem, "1001"), recursive = TRUE) +unlink(file.path(nonmem, "1001.yaml")) +unlink(file.path(nonmem, "1001.toml")) +unlink(file.path(nonmem, "submission-log"), recursive = TRUE) +unlink(file.path(nonmem, "in_progress"), recursive = TRUE) +``` + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>" +) +``` + +```{r setup} +library(slurmtools) +library(bbr) +library(here) + +nonmem = file.path(here::here(), "vignettes", "model", "nonmem") +options('slurmtools.submission_root' = file.path(nonmem, "submission-log")) +``` + +```{r} +mod_number <- "1001" + +if (file.exists(file.path(nonmem, paste0(mod_number, ".yaml")))) { + mod <- bbr::read_model(file.path(nonmem, mod_number)) +} else { + mod <- bbr::new_model(file.path(nonmem, mod_number)) +} +``` + +# Cut me some Slack + +There is also functionality to pair `nmm` with +[slack_notifier](https://github.com/A2-ai/slack_notifier/releases/) and +get messages sent directly to you via a slack bot. This requires you to +download the slack_notifier binaries and added them to your path so +`nmm` can find it. You can download the latest release and extract the +binary and again save it to `~/.local/bin`. + +```{r} +Sys.which("slack_notifier") +``` + +slack_notifier requires an additional `slack_notifier/config.yaml` file +that contains the slack bot OAuth token which is found from +[[[https://api.slack.com/apps/\\\\](https://api.slack.com/apps/\\){.uri}]([https://api.slack.com/apps/\\](https://api.slack.com/apps/\){.uri}){.uri}\ /oauth?]. + +``` slack_notifier/config.yaml +SLACK_OAUTH_TOKEN: "encrypted(Bot User OAuth Token)" +``` + +Again, we need to update the `1001.toml` file to get slack +notifications. We need to set `alert = "slack"` and provide the `email` +associated with the slack account in `generate_nmm_config`. + +```{r} +slurmtools::generate_nmm_config( + mod, + alert = "slack", + email = "matthews@a2-ai.com", + watched_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem", + output_dir = "/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress") +``` + +This generates the following toml file: + +``` 1001.toml +model_number = '1001' +files_to_track = [ 'lst', 'ext', 'grd' ] +tmp_dir = '/tmp' +watched_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem' +output_dir = '/cluster-data/user-homes/matthews/Packages/slurmtools/vignettes/model/nonmem/in_progress' +poll_duration = 1 +alert = 'Slack' +level = 'Debug' +email = 'matthews@a2-ai.com' +threads = 1 +topic = '' +``` + +With `alert = 'Slack'` and `email` set in the `1001.toml` file `nmm` +will send slack notifications directly to you when a NONMEM run starts +and it will reply to that message with notifications if any gradients +hit 0 and when the run finishes it checks if all -1E9 lines are present +in the .ext file and gives another message about any parameters that hit +0 gradient. + +```{r} +submission_nmm <- slurmtools::submit_nonmem_model( + mod, + overwrite = TRUE, + slurm_job_template_path = file.path(nonmem, "slurm-job-nmm.tmpl"), + slurm_template_opts = list( + nmm_exe_path = normalizePath("~/.local/bin/nmm")) +) + +submission_nmm +``` + +```{r} +slurmtools::get_slurm_jobs() +``` + +![nmm slack alerts](data/images/nmm_slack_notifications.png) + +```{r, include = FALSE} +#cancelling any running nonmem jobs +state <- slurmtools::get_slurm_jobs(user = "matthews") + +if (any(state$job_state %in% c("RUNNING", "CONFIGURING"))) { + for (job_id in state %>% dplyr::filter(job_state == "RUNNING") %>% dplyr::pull("job_id")) { + processx::run("scancel", args = paste0(job_id)) + } +} + +#removing generated files from running this vignette +nonmem <- file.path("model", "nonmem") + +unlink(file.path(nonmem, "1001"), recursive = TRUE) +unlink(file.path(nonmem, "1001.yaml")) +unlink(file.path(nonmem, "1001.toml")) +unlink(file.path(nonmem, "submission-log"), recursive = TRUE) +unlink(file.path(nonmem, "in_progress"), recursive = TRUE) +```