Merge branch 'release/1.14.0'

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: redcapcustodian
 Type: Package
 Title: Data automation for R-centric workflows with a nod towards REDCap
-Version: 1.13.1
+Version: 1.14.0
 Authors@R: c(
   person("Philip", "Chase",
         email = "[email protected]",

Dockerfile

@@ -1,4 +1,4 @@
-FROM rocker/verse:4.2.1
+FROM --platform=linux/amd64 rocker/verse:4.2.1
 
 WORKDIR /home/rocker
 
@@ -59,7 +59,7 @@ RUN R -e "tinytex::tlmgr_install(c(\
 
 # build and install this package
 ADD . /home/rocker/redcapcustodian
-RUN R CMD build redcapcustodian
+RUN R CMD build --no-build-vignettes redcapcustodian
 RUN R CMD INSTALL redcapcustodian_*.tar.gz
 RUN rm -rf redcapcustodian
 

NEWS.md

@@ -1,3 +1,7 @@
+# redcapcustodian 1.14.0 (released 2023-08-14)
+- Fix build issues on Apple silicon (@pbchase, #133, #134)
+- Add delete_erroneous_survey_reminders.R (@pbchase, #131, #132)
+
 # redcapcustodian 1.13.1 (released 2023-08-04)
 - update render_report to handle qmd files (@ljwoodley, #117, #129)
 - Add LICENSE (@pbchase, #119, #124)

README.md

@@ -46,3 +46,5 @@ While much of the REDCap Custodian repository and package is about automating wo
 - For tools and procedures for moving production projects that use randomization, See [Randomization Management](https://ctsit.github.io/redcapcustodian/articles/randomization-management.html)
 - For bulk rights expiration, see the function [`expire_user_project_rights()`](https://ctsit.github.io/redcapcustodian/reference/expire_user_project_rights.html)
 - To get the history of projects on a REDCap system, see [`get_project_life_cycle()`](https://ctsit.github.io/redcapcustodian/reference/get_project_life_cycle.html). It runs speedy queries against the REDCap log event tables to get all of the events in the life cycle of every project on the system. 
+- To delete survey reminders that persist after survey completion, see 
+[Deleting Erroneous Survey Reminders](https://ctsit.github.io/redcapcustodian/articles/delete-erroneous-survey-reminders.html)

VERSION

@@ -1 +1 @@
-1.13.1
+1.14.0

etl/delete_erroneous_survey_reminders.R

@@ -0,0 +1,94 @@
+library(tidyverse)
+library(lubridate)
+library(dotenv)
+library(redcapcustodian) # devtools::install_github("ctsit/redcapcustodian")
+library(DBI)
+library(RMariaDB)
+
+init_etl("delete_erroneous_survey_reminders")
+
+rc_conn <- connect_to_redcap_db()
+
+# identify future scheduled reminders
+sql_scheduled_reminders <- paste("
+  select a.project_id, q.record, p.event_id, s.form_name, q.ssq_id
+  from redcap_surveys_emails_recipients r, redcap_surveys_participants p,
+    redcap_surveys s, redcap_projects a, redcap_surveys_scheduler_queue q
+  where q.time_sent is null and
+    q.email_recip_id = r.email_recip_id and
+    p.survey_id = s.survey_id and
+    r.participant_id = p.participant_id and
+    s.project_id = a.project_id and
+    a.date_deleted is null and
+    a.completed_time is null and
+    q.scheduled_time_to_send >= now() and
+    q.status = 'QUEUED' and
+    a.status <= 1 and
+    a.online_offline = 1
+  order by q.scheduled_time_to_send, q.ssq_id;
+")
+
+scheduled_reminders <- dbGetQuery(rc_conn, sql_scheduled_reminders) %>%
+  mutate(field_name = paste0(form_name, "_complete"))
+
+# check form completed status
+df_params <- scheduled_reminders %>%
+  select(
+    project_id,
+    event_id,
+    record,
+    field_name
+  ) %>%
+  distinct()
+
+data_statement <- dbSendQuery(rc_conn, "
+  SELECT * FROM redcap_data
+  WHERE
+  project_id = ? and
+  event_id = ? and
+  record = ? and
+  field_name = ?
+")
+
+dbBind(data_statement, list(
+  df_params$project_id,
+  df_params$event_id,
+  df_params$record,
+  df_params$field_name
+))
+
+form_status <- dbFetch(data_statement)
+
+reminders_to_deactivate <-
+  scheduled_reminders %>%
+  inner_join(form_status, by = c("project_id", "event_id", "record", "field_name")) %>%
+  filter(value == 2)
+
+if (nrow(reminders_to_deactivate) > 0) {
+  # describe what needs to be deleted
+  sql_reminders_to_delete <-
+    paste(
+      "delete from redcap_surveys_scheduler_queue where ssq_id in (",
+      paste(reminders_to_deactivate$ssq_id, collapse = " "),
+      ");"
+    )
+  rows_to_be_deleted <- tbl(rc_conn, "redcap_surveys_scheduler_queue") %>%
+    filter(ssq_id %in% !!reminders_to_deactivate$ssq_id) %>%
+    collect()
+
+  # delete the erroneous rows
+  rows_affected <- dbExecute(rc_conn, sql_reminders_to_delete)
+
+  # Log what we did
+  if (rows_affected == nrow(reminders_to_deactivate)) {
+    deletions <- rows_to_be_deleted
+    activity_log <- lst(
+      deletions
+    )
+    log_job_success(jsonlite::toJSON(activity_log))
+  } else {
+    log_job_failure(jsonlite::toJSON(lst(rows_affected)))
+  }
+}
+
+dbDisconnect(rc_conn)

study_template/cron/delete_erroneous_survey_reminders

@@ -0,0 +1,3 @@
+# Run delete_erroneous_survey_reminders daily.
+# Increase the frequency as needed to address more frequent survey reminders
+# 5 7 * * * root /usr/bin/docker run --rm --env-file /rcc/example/.env redcapcustodian Rscript redcapcustodian/etl/delete_erroneous_survey_reminders.R

vignettes/delete-erroneous-survey-reminders.Rmd

@@ -0,0 +1,26 @@
+---
+title: "Deleting Erroneous Survey Reminders"
+output: rmarkdown::html_vignette
+vignette: >
+  %\VignetteIndexEntry{Deleting Erroneous Survey Reminders}
+  %\VignetteEngine{knitr::rmarkdown}
+  %\VignetteEncoding{UTF-8}
+---
+
+```{r, include = FALSE}
+knitr::opts_chunk$set(
+  collapse = TRUE,
+  comment = "#>",
+  include = FALSE
+)
+```
+
+There is an ancient bug in REDCap, long ago fixed, that created a data bug that plagues some old REDCap projects on old REDCap systems. It causes novel pre-scheduled survey reminders to not get purged after the survey is completed. The bug persists in mid-2023 and is elusive. 
+
+Whereas we don't know where the data bug is, we can see the symptoms in the form of non-deleted invitation reminders for completed surveys. These are easy to see by looking in `Surveys/invite_participants.php` for future invitations with completed responses. It's easy to delete them manually there as well, but as new ones are being created every day, it presents a maintenance problem. They have to be deleted before a reminder to a completed survey goes out.
+
+Should this bug exist on your REDCap system, the script `delete_erroneous_survey_reminders.R` Can provide some relief. It enumerates all the future survey reminders on a REDCap system that already have completed responses and deletes them. It logs this activity in the job log.
+
+## Activation
+
+Should you want to use this on your system, follow the instructions at [REDCap Custodian](https://ctsit.github.io/redcapcustodian/) to make a new R Project from the study_template if you don't already have such a project. Then revise `cron/delete_erroneous_survey_reminders` uncommenting the cron line and adjusting the frequency to your needs. Then build and deploy your R Project.