From c39deef54c9ddf75159531aa9dc77ecb392580b7 Mon Sep 17 00:00:00 2001 From: Laurence James-Woodley Date: Mon, 3 Jun 2024 19:07:10 -0400 Subject: [PATCH 1/5] log project and project instnace --- NAMESPACE | 4 +++ R/logging.R | 66 +++++++++++++++++++++++++++++++++++++ R/utils.R | 6 +++- inst/schema/rcc_job_log.sql | 4 +++ man/init_etl.Rd | 6 ++++ 5 files changed, 85 insertions(+), 1 deletion(-) diff --git a/NAMESPACE b/NAMESPACE index 1127fe9..d21430b 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -23,7 +23,9 @@ export(get_current_time) export(get_institutional_person_data) export(get_job_duration) export(get_package_scope_var) +export(get_project_instance) export(get_project_life_cycle) +export(get_project_name) export(get_redcap_credentials) export(get_redcap_db_connection) export(get_redcap_email_revisions) @@ -45,6 +47,8 @@ export(scrape_user_api_tokens) export(send_email) export(set_package_scope_var) export(set_project_api_token) +export(set_project_instance) +export(set_project_name) export(set_script_name) export(set_script_run_time) export(set_super_api_token) diff --git a/R/logging.R b/R/logging.R index 4b0e559..47381df 100644 --- a/R/logging.R +++ b/R/logging.R @@ -29,6 +29,8 @@ build_etl_job_log_df <- function(job_duration, job_summary, level) { ) %>% dplyr::mutate( log_date = get_current_time(), + project = get_project_name(), + instance = get_project_instance(), script_name = get_script_name(), script_run_time = get_script_run_time(), job_duration = .data$job_duration, @@ -65,6 +67,8 @@ build_formatted_df_from_result <- function(result, database_written, table_writt dplyr::mutate(record_level_data = purrr::pmap(.data, ~ rjson::toJSON(c(...)))) %>% dplyr::select(primary_key = pk_col, .data$record_level_data) %>% dplyr::mutate( + project = get_project_name(), + instance = get_project_instance(), script_name = get_script_name(), script_run_time = get_script_run_time(), log_date = get_current_time(), @@ -130,6 +134,19 @@ get_script_run_time <- function() { return(redcapcustodian.env$script_run_time) } +#' Fetches the package-scoped value of project_name +#' @export +get_project_name <- function() { + return(redcapcustodian.env$project_name) +} + +#' Fetches the package-scoped value of project_instance +#' @export +get_project_instance <- function() { + return(redcapcustodian.env$project_instance) +} + + #' Initialize the connection to the log db and set redcapcustodian.env$log_con #' #' @param drv, an object that inherits from DBIDriver (e.g. RMariaDB::MariaDB()), or an existing DBIConnection object (in order to clone an existing connection). @@ -205,6 +222,55 @@ set_script_run_time <- function(fake_runtime = lubridate::NA_POSIXct_) { return(redcapcustodian.env$script_run_time) } +#' Sets the package-scoped value of project_name +#' +#' @param project_name Defaults to NULL. If provided and not NULL, this value is used. +#' If NULL, the function attempts to fetch the value from the environment variable. +#' @return the package-scoped value of project_name + +#' @examples +#' \dontrun{ +#' project_name <- set_project_name() +#' project_name <- set_project_name("project_name") +#' } +#' +#' @export +set_project_name <- function(project_name = "") { + if (project_name == "") { + project_name <- Sys.getenv("PROJECT") + } + + assign("project_name", + project_name, + envir = redcapcustodian.env) + + return(redcapcustodian.env$project_name) +} + +#' SEts the package-scoped value of project_instance +#' @param project_instance Defaults to NULL. If provided and not NULL, this value is used. +#' If NULL, the function attempts to fetch the value from the environment variable. +#' +#' @return the package-scoped value of project_instance +#' @examples +#' \dontrun{ +#' project_instance <- set_project_instance() +#' project_instance <- set_project_instance("project_instance") +#' } +#' +#' @export +set_project_instance <- function(project_instance = "") { + if (project_instance == "") { + project_instance <- Sys.getenv("INSTANCE") + } + + assign("project_instance", + project_instance, + envir = redcapcustodian.env) + + return(redcapcustodian.env$project_instance) +} + #' Attempts to connect to the DB using all LOG_DB_* environment variables. Returns an empty list if a connection is established, returns an `error_list` entry otherwise. #' #' @param drv, an object that inherits from DBIDriver (e.g. RMariaDB::MariaDB()), or an existing DBIConnection object (in order to clone an existing connection). diff --git a/R/utils.R b/R/utils.R index bf4180b..b14ded8 100644 --- a/R/utils.R +++ b/R/utils.R @@ -47,6 +47,8 @@ get_package_scope_var <- function(key) { #' Initialize all etl dependencies #' +#' @param project_name name passed to \code{\link{set_project_name}} +#' @param project_instance name passed to \code{\link{set_project_instance}} #' @param script_name name passed to \code{\link{set_script_name}} #' @param fake_runtime An optional asserted script run time passed to \code{\link{set_script_run_time}}, defaults to the time this function is called #' @param log_db_drv, an object that inherits from DBIDriver (e.g. RMariaDB::MariaDB()), or an existing DBIConnection object (in order to clone an existing connection). @@ -57,7 +59,9 @@ get_package_scope_var <- function(key) { #' init_etl("name_of_file") #' } #' -init_etl <- function(script_name = "", fake_runtime = NULL, log_db_drv = RMariaDB::MariaDB()) { +init_etl <- function(script_name = "", project_name = "", project_instance = "", fake_runtime = NULL, log_db_drv = RMariaDB::MariaDB()) { + set_project_name(project_name) + set_project_instance(project_instance) set_script_name(script_name) if (!is.null(fake_runtime)) { set_script_run_time(fake_runtime) diff --git a/inst/schema/rcc_job_log.sql b/inst/schema/rcc_job_log.sql index 7369ed8..ca28a0f 100644 --- a/inst/schema/rcc_job_log.sql +++ b/inst/schema/rcc_job_log.sql @@ -1,6 +1,8 @@ CREATE TABLE `rcc_job_log` ( `id` bigint NOT NULL AUTO_INCREMENT, `log_date` datetime NOT NULL, + `project` VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL, + `instance` VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL, `script_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, `script_run_time` datetime NOT NULL, `job_summary_data` mediumtext DEFAULT NULL, @@ -8,6 +10,8 @@ CREATE TABLE `rcc_job_log` ( `level` enum('SUCCESS','DEBUG','ERROR') CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, PRIMARY KEY (`id`), KEY `log_date` (`log_date`), + KEY `project` (`project`), + KEY `instance` (`instance`), KEY `script_name` (`script_name`), KEY `script_run_time` (`script_run_time`), KEY `level` (`level`) diff --git a/man/init_etl.Rd b/man/init_etl.Rd index d68508b..dce216e 100644 --- a/man/init_etl.Rd +++ b/man/init_etl.Rd @@ -6,6 +6,8 @@ \usage{ init_etl( script_name = "", + project_name = "", + project_instance = "", fake_runtime = NULL, log_db_drv = RMariaDB::MariaDB() ) @@ -13,6 +15,10 @@ init_etl( \arguments{ \item{script_name}{name passed to \code{\link{set_script_name}}} +\item{project_name}{name passed to \code{\link{set_project_name}}} + +\item{project_instance}{name passed to \code{\link{set_project_instance}}} + \item{fake_runtime}{An optional asserted script run time passed to \code{\link{set_script_run_time}}, defaults to the time this function is called} \item{log_db_drv, }{an object that inherits from DBIDriver (e.g. RMariaDB::MariaDB()), or an existing DBIConnection object (in order to clone an existing connection).} From 3bd0889748e968e6cdec930ec2a92b86aa001462 Mon Sep 17 00:00:00 2001 From: Laurence James-Woodley Date: Mon, 3 Jun 2024 19:08:52 -0400 Subject: [PATCH 2/5] fix typo --- R/logging.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/logging.R b/R/logging.R index 47381df..de89b29 100644 --- a/R/logging.R +++ b/R/logging.R @@ -247,7 +247,7 @@ set_project_name <- function(project_name = "") { return(redcapcustodian.env$project_name) } -#' SEts the package-scoped value of project_instance +#' Sets the package-scoped value of project_instance #' @param project_instance Defaults to NULL. If provided and not NULL, this value is used. #' If NULL, the function attempts to fetch the value from the environment variable. #' From 4ba12c05a00b699089d3417e012a88bb5e7147c6 Mon Sep 17 00:00:00 2001 From: Laurence James-Woodley Date: Mon, 3 Jun 2024 19:12:47 -0400 Subject: [PATCH 3/5] add docs --- man/get_project_instance.Rd | 11 +++++++++++ man/get_project_name.Rd | 11 +++++++++++ man/set_project_instance.Rd | 24 ++++++++++++++++++++++++ man/set_project_name.Rd | 25 +++++++++++++++++++++++++ 4 files changed, 71 insertions(+) create mode 100644 man/get_project_instance.Rd create mode 100644 man/get_project_name.Rd create mode 100644 man/set_project_instance.Rd create mode 100644 man/set_project_name.Rd diff --git a/man/get_project_instance.Rd b/man/get_project_instance.Rd new file mode 100644 index 0000000..0152427 --- /dev/null +++ b/man/get_project_instance.Rd @@ -0,0 +1,11 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/logging.R +\name{get_project_instance} +\alias{get_project_instance} +\title{Fetches the package-scoped value of project_instance} +\usage{ +get_project_instance() +} +\description{ +Fetches the package-scoped value of project_instance +} diff --git a/man/get_project_name.Rd b/man/get_project_name.Rd new file mode 100644 index 0000000..1cc191c --- /dev/null +++ b/man/get_project_name.Rd @@ -0,0 +1,11 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/logging.R +\name{get_project_name} +\alias{get_project_name} +\title{Fetches the package-scoped value of project_name} +\usage{ +get_project_name() +} +\description{ +Fetches the package-scoped value of project_name +} diff --git a/man/set_project_instance.Rd b/man/set_project_instance.Rd new file mode 100644 index 0000000..03f7c07 --- /dev/null +++ b/man/set_project_instance.Rd @@ -0,0 +1,24 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/logging.R +\name{set_project_instance} +\alias{set_project_instance} +\title{Fetches the package-scoped value of project_instance} +\usage{ +set_project_instance(project_instance = "") +} +\arguments{ +\item{project_instance}{Defaults to NULL. If provided and not NULL, this value is used. + If NULL, the function attempts to fetch the value from the environment variable. + +the package-scoped value of project_instance} +} +\description{ +Fetches the package-scoped value of project_instance +} +\examples{ +\dontrun{ +project_instance <- set_project_instance() +project_instance <- set_project_instance("project_instance") +} + +} diff --git a/man/set_project_name.Rd b/man/set_project_name.Rd new file mode 100644 index 0000000..9aafafc --- /dev/null +++ b/man/set_project_name.Rd @@ -0,0 +1,25 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/logging.R +\name{set_project_name} +\alias{set_project_name} +\title{Sets the package-scoped value of project_name} +\usage{ +set_project_name(project_name = "") +} +\arguments{ +\item{project_name}{Defaults to NULL. If provided and not NULL, this value is used. +If NULL, the function attempts to fetch the value from the environment variable.} +} +\value{ +the package-scoped value of project_name +} +\description{ +Sets the package-scoped value of project_name +} +\examples{ +\dontrun{ +project_name <- set_project_name() +project_name <- set_project_name("project_name") +} + +} From a624786980a3a8dce9ad24ed27394246026638b9 Mon Sep 17 00:00:00 2001 From: Philip Chase Date: Thu, 13 Jun 2024 17:33:51 -0400 Subject: [PATCH 4/5] Update documentation for and templates for PROJECT and INSTANCE logging --- man/set_project_instance.Rd | 11 ++++++----- rcc.log.db/schema/rcc_job_log.sql | 4 ++++ study_template/example.env | 1 + vignettes/custom_rscript.Rmd | 7 +++++-- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/man/set_project_instance.Rd b/man/set_project_instance.Rd index 03f7c07..bbaddf2 100644 --- a/man/set_project_instance.Rd +++ b/man/set_project_instance.Rd @@ -2,18 +2,19 @@ % Please edit documentation in R/logging.R \name{set_project_instance} \alias{set_project_instance} -\title{Fetches the package-scoped value of project_instance} +\title{Sets the package-scoped value of project_instance} \usage{ set_project_instance(project_instance = "") } \arguments{ \item{project_instance}{Defaults to NULL. If provided and not NULL, this value is used. - If NULL, the function attempts to fetch the value from the environment variable. - -the package-scoped value of project_instance} +If NULL, the function attempts to fetch the value from the environment variable.} +} +\value{ +the package-scoped value of project_instance } \description{ -Fetches the package-scoped value of project_instance +Sets the package-scoped value of project_instance } \examples{ \dontrun{ diff --git a/rcc.log.db/schema/rcc_job_log.sql b/rcc.log.db/schema/rcc_job_log.sql index 7369ed8..ca28a0f 100644 --- a/rcc.log.db/schema/rcc_job_log.sql +++ b/rcc.log.db/schema/rcc_job_log.sql @@ -1,6 +1,8 @@ CREATE TABLE `rcc_job_log` ( `id` bigint NOT NULL AUTO_INCREMENT, `log_date` datetime NOT NULL, + `project` VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL, + `instance` VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL, `script_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, `script_run_time` datetime NOT NULL, `job_summary_data` mediumtext DEFAULT NULL, @@ -8,6 +10,8 @@ CREATE TABLE `rcc_job_log` ( `level` enum('SUCCESS','DEBUG','ERROR') CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, PRIMARY KEY (`id`), KEY `log_date` (`log_date`), + KEY `project` (`project`), + KEY `instance` (`instance`), KEY `script_name` (`script_name`), KEY `script_run_time` (`script_run_time`), KEY `level` (`level`) diff --git a/study_template/example.env b/study_template/example.env index 5ccf91c..e803639 100644 --- a/study_template/example.env +++ b/study_template/example.env @@ -1,3 +1,4 @@ +PROJECT=study_template INSTANCE=Development TIME_ZONE=America/New_York diff --git a/vignettes/custom_rscript.Rmd b/vignettes/custom_rscript.Rmd index cb72436..5275408 100644 --- a/vignettes/custom_rscript.Rmd +++ b/vignettes/custom_rscript.Rmd @@ -53,13 +53,16 @@ If this is not your first REDCap Custodian rodeo, you might remember what you ne You'll need to talk to at least a REDCap or a MySQL database. You will probably want both. Rename the `example.env` to `testing.env` and configure it for development on your computer. That file is composed of five sections. ```sh +PROJECT=study_template INSTANCE=Development TIME_ZONE=America/New_York ``` -`INSTANCE` names the REDCap system or _instance_ you'll be talking to. This file assumes you are talking to only one REDCap in your script. There are other tools for multiple-instances. +`PROJECT` names the project, study, or git repository that owns the scripts. This is useful when reading log data from a shared log database. A unique value in the PROJECT field allows you to identify log entries specific to a project. -`TIME_ZONE` should be the local timezone of your REDCap instance. Note that REDCap time facts in local time. The MariaDB driver and the Lubridate library default to UTC. That can get complicated. Be careful with time. For more details see [`stupid_date_tricks.R`](https://gist.github.com/pbchase/ed55ab5dacbcc5d8a702a9cb935cccb5) +`INSTANCE` names the _instance_ or a project, study, or REDCap system. If scripts are deployed in development, testing, and production, a unique value in this field allows you to identify log entries from different deployments. + +`TIME_ZONE` should be the local timezone of your REDCap instance. Note that REDCap records time facts in local time. The MariaDB driver and the Lubridate library default to UTC. That can get complicated. Be careful with time. For more details see [`stupid_date_tricks.R`](https://gist.github.com/pbchase/ed55ab5dacbcc5d8a702a9cb935cccb5) ```sh # Email config From e1224e8154aae701744f3884f3ae5a0cd9cc9c63 Mon Sep 17 00:00:00 2001 From: Philip Chase Date: Thu, 13 Jun 2024 17:40:57 -0400 Subject: [PATCH 5/5] Bump VERSION and update NEWS.md for release 1.23.0 --- DESCRIPTION | 2 +- NEWS.md | 3 +++ VERSION | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 812183f..daa0d42 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: redcapcustodian Type: Package Title: Data automation for R-centric workflows with a nod towards REDCap -Version: 1.22.2 +Version: 1.23.0 Authors@R: c( person("Philip", "Chase", email = "pbc@ufl.edu", diff --git a/NEWS.md b/NEWS.md index 8870571..1bcebf3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,6 @@ +# redcapcustodian 1.23.0 (released 2024-06-13) +- Add project and instance to logging (@ljwoodley, @pbchase, #159, #160) + # redcapcustodian 1.22.2 (released 2024-04-26) - Restore 'writexl' to Dockerfile (@pbchase) diff --git a/VERSION b/VERSION index 6fee2fe..a6c2798 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.22.2 +1.23.0