Merge branch 'release/1.5.0'

.Rbuildignore

@@ -21,3 +21,4 @@ docs/*
 local.env.txt
 ^hosts/.*
 ^data-raw$
+^credentials$

.gitignore

@@ -17,6 +17,9 @@ site
 !host_template/.env
 local.env.txt
 
+# local credentials databases
+credentials/*
+
 # produced vignettes
 vignettes/*.html
 vignettes/*.R

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: redcapcustodian
 Type: Package
 Title: System data cleaning for REDCap
-Version: 1.4.1
+Version: 1.5.0
 Authors@R: c(
   person("Philip", "Chase",
         email = "[email protected]",
@@ -58,7 +58,8 @@ Suggests:
     digest,
     RSQLite,
     knitr (>= 1.18),
-    rmarkdown (>= 2.0)
+    rmarkdown (>= 2.0),
+    fs
 VignetteBuilder: knitr
 Config/testthat/edition: 3
 RoxygenNote: 7.2.1

NAMESPACE

@@ -3,6 +3,7 @@
 export(connect_to_db)
 export(connect_to_log_db)
 export(connect_to_redcap_db)
+export(convert_schema_to_sqlite)
 export(create_test_table)
 export(create_test_tables)
 export(dataset_diff)
@@ -27,6 +28,7 @@ export(is_on_ci)
 export(log_job_debug)
 export(log_job_failure)
 export(log_job_success)
+export(mutate_columns_to_posixct)
 export(quit_non_interactive_run)
 export(scrape_user_api_tokens)
 export(send_email)
@@ -42,6 +44,7 @@ export(sync_table_2)
 export(update_redcap_email_addresses)
 export(write_error_log_entry)
 export(write_info_log_entry)
+export(write_summary_metrics)
 export(write_to_sql_db)
 importFrom(magrittr,"%>%")
 importFrom(rlang,.data)

NEWS.md

@@ -3,6 +3,17 @@ All notable changes to the redcapcustodian package and its contained scripts wil
 This project adheres to [Semantic Versioning](http://semver.org/).
 
 
+## [1.5.0] - 2023-01-25
+### Added
+- Create write_summary_metrics function, corresponding schema and test (Kyle Chesney)
+- Add render_report to /report (Laurence James-Woodley)
+- Port convert_schema_to_sqlite from rcc.billing, altering it to accept path to sql file as input (Kyle Chesney)
+- Port mutate_columns_to_posixct from rcc.billing (Kyle Chesney)
+
+### Changed
+- Ignore local credentials DBs (Philip Chase)
+
+
 ## [1.4.1] - 2022-12-15
 ### Changed
 - Install latex packages directly in Dockerfile (Laurence James-Woodley)

R/devtools.R

@@ -83,3 +83,66 @@ create_test_tables <- function(conn, table_names = c()) {
   )
 
 }
+
+#' Converts a MySQL schema file to a sqlite schema.
+#' Facilitates easier creation of in-memory (i.e. sqlite) tables.
+#'
+#' @param schema_file_path, the path of the schema file to convert
+#'
+#' @importFrom magrittr "%>%"
+#'
+#' @return the translated schema as a character string
+#'
+#' @examples
+#' \dontrun{
+#' mem_conn <- DBI::dbConnect(RSQLite::SQLite(), dbname = ":memory:")
+#' translated_schema <- convert_schema_to_sqlite("~/documents/my_cool_schema.sql")
+#' DBI::dbSendQuery(mem_conn, schema)
+#' }
+#' @export
+convert_schema_to_sqlite <- function(schema_file_path) {
+  pl_to_sqlite <- system.file("", "to_sqlite.pl", package = "redcapcustodian")
+
+  if (!file.exists(schema_file_path)) {
+    stop(paste("Schema file does not exist at", schema_file_path))
+  }
+  # TODO: consider supporting raw SQL input, assume raw sql given if file does not exist
+  # raw_sql <- readr::read_file(schema_file_path)
+  # cmd <- echo "${raw_sql}" | perl to_sqlite.pl
+
+  # convert to sqlite
+  cmd <- paste("cat", schema_file_path, "|", "perl", pl_to_sqlite)
+
+  result <- system(cmd, intern = TRUE) %>% paste(collapse = "")
+  return(result)
+}
+
+#' mutate_columns_to_posixct
+#'
+#' Mutates column data types to POSIXct.
+#' Especially useful when working with in-memory tables where dates are often converted to int.
+#'
+#' @param data - a dataframe to mutate
+#' @param column_names - a vector of column names to mutate
+#'
+#' @return The input dataframe with revised data types
+#' @export
+#'
+#' @importFrom magrittr "%>%"
+#' @importFrom rlang .data
+#'
+#' @examples
+#' \dontrun{
+#' time_columns <- c("created", "updated")
+#' mutate_columns_to_posixct(data, time_columns)
+#' }
+#' @export
+mutate_columns_to_posixct <- function(data, column_names) {
+  result <- data %>%
+    dplyr::mutate(dplyr::across(
+      dplyr::any_of(column_names),
+      ~ as.POSIXct(., origin = "1970-01-01 00:00.00 UTC", tz = "UTC")
+    ))
+
+  return(result)
+}

R/summary_metrics.R

@@ -0,0 +1,52 @@
+#' Format and write summary metrics to the redcap_summary_metrics table in your LOG_DB
+#'
+#' @param reporting_period_start a datetime object, e.g. ymd_hms("2022-11-01 00:00:00")
+#' @param reporting_period_end a datetime object, e.g. ymd_hms("2022-12-01 00:00:00")
+#' @param metric_type a character string representing the metric type, e.g. "flux", "state"
+#' @param metric_dataframe A wide data frame of key-value pairs with a single row of data
+#'
+#' @return nothing
+#'
+#' @export
+#' @examples
+#' \dontrun{
+#'  write_summary_metrics(
+#'    reporting_period_start = ymd_hms("2022-01-01 00:00:00", tz=Sys.getenv("TIME_ZONE")),
+#'    reporting_period_end = ceiling_date(reporting_period_start, "month", change_on_boundary = T)
+#'    metric_type = "state",
+#'    metric_dataframe = my_cool_df
+#'  )
+#' }
+write_summary_metrics <- function(reporting_period_start,
+                                  reporting_period_end,
+                                  metric_type,
+                                  metric_dataframe) {
+
+  tall_df <- metric_dataframe %>%
+    tidyr::pivot_longer(
+      cols = dplyr::everything(),
+      names_to = "key",
+      values_to = "value"
+    ) %>%
+    cbind(
+      reporting_period_start,
+      reporting_period_end,
+      metric_type,
+      script_name = get_script_name(),
+      script_run_time = get_script_run_time()
+    ) %>%
+    dplyr::select(
+      reporting_period_start,
+      reporting_period_end,
+      .data$key,
+      .data$value,
+      .data$metric_type,
+      .data$script_name,
+      .data$script_run_time
+    )
+
+  log_conn <- get_package_scope_var("log_con")
+
+  # log data in redcap_summary_metrics
+  DBI::dbAppendTable(log_conn, "redcap_summary_metrics", tall_df)
+}

VERSION

@@ -1 +1 @@
-1.4.1
+1.5.0

inst/schema/redcap_summary_metrics.sql

@@ -0,0 +1,13 @@
+CREATE TABLE `redcap_summary_metrics` (
+  `id` INTEGER PRIMARY KEY NOT NULL AUTO_INCREMENT,
+  `script_run_time` datetime NOT NULL,
+  `script_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
+  `reporting_period_start` datetime NOT NULL,
+  `reporting_period_end` datetime NOT NULL,
+  `key` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
+  `value` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
+  `metric_type` enum('flux', 'state') CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
+  KEY `script_name` (`script_name`),
+  KEY `script_run_time` (`script_run_time`),
+  KEY `metric_type` (`metric_type`)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

inst/to_sqlite.pl

@@ -0,0 +1,33 @@
+#!/usr/bin/perl
+while (<>){
+    if (m/^SET/) { next };
+    s/CHARACTER SET \S+ //;    # remove CHARACTER SET mumble
+    s/ENGINE=\S+ *//;          # remove ENGINE
+    s/DEFAULT CHARSET=\S+ *//; # remove DEFAULT CHARSET
+    s/COLLATE [^, ]+//;         # remove COLLATE on column
+    s/ UNSIGNED//i;            # remove unsigned on column
+    s/COLLATE=\S+ *//;         # remove COLLATE on table
+    s/COMMENT '.+'//;          # remove COMMENT on column
+    s/COMMENT='.+'//;          # remove COMMENT on table 
+    s/enum\(.*\)/varchar(255)/;              # replace enum
+    if (m/^ALTER TABLE/) { next }; # remove ALTER TABLE
+    if (m/^\s*ADD /) { next };     # Remove indented ADD. Note: this is very crude
+    if (m/^\s*MODIFY /) { next };  # Remove indented MODIFY. Note: this is very crude
+    s/int\(\d+\)/integer/g;    # Replace int(NN) with integer
+    s/\\'/''/g;                # Use '' instead of \'
+    s/\\"/"/g;                 # Use " instead of \"
+    s/\\r\\n/\r\n/g;           # Convert escaped \r\n to literal
+    s/\\\\/\\/g;               # Convert escaped \ to literal
+    s/ auto_increment=?\d*//gi;      # Remove auto_increment
+    s/^[UN]*?LOCK TABLES.*//g; # Remove locking statements
+    if (m/^\s*KEY /) { next }; # Remove indented KEY
+    if (m/^\s*UNIQUE KEY /) { next };   # Remove indented KEY
+    if (m/^\s*PRIMARY KEY /) { next };  # Remove indented KEY
+    $lines .= $_;
+}
+
+# remove the comma from the last param before the close paren
+local $/ = undef;
+$lines =~ s/,\n\)/\n\)/;
+
+print $lines;

report/render_report.R

@@ -0,0 +1,49 @@
+library(tidyverse)
+library(dotenv)
+library(REDCapR)
+library(lubridate)
+library(rmarkdown)
+library(sendmailR)
+library(redcapcustodian)
+
+init_etl("render_report")
+
+if (!dir.exists("output")){
+  dir.create("output")
+}
+
+if (!interactive()) {
+  args <- commandArgs(trailingOnly = T)
+  script_name <- word(args, 2, sep = "=")
+} else {
+  script_name <- "sample_report.Rmd"
+}
+
+report_name <- word(script_name, 1, sep = "\\.")
+
+script_run_time <- set_script_run_time()
+
+output_file <- here::here(
+  "output",
+  paste0(report_name,
+  "_",
+  format(script_run_time, "%Y%m%d%H%M%S"))
+)
+
+full_path_to_output_file <- render(
+  here::here("report", script_name),
+  output_file = output_file
+)
+
+output_file_extension <- word(full_path_to_output_file, 2 , sep = "\\.")
+attachment_object <- mime_part(full_path_to_output_file, basename(full_path_to_output_file))
+
+email_subject <- paste(report_name, "|", script_run_time)
+body <- "Please see the attached report."
+
+email_body <- list(body, attachment_object)
+
+# send the email with the attached output file
+send_email(email_body, email_subject)
+
+log_job_success(jsonlite::toJSON(script_name))