Skip to content

Commit

Permalink
Merge branch 'develop' into add_erase_move_project_field_log_records
Browse files Browse the repository at this point in the history
  • Loading branch information
pbchase committed Nov 26, 2024
2 parents ee63a95 + 16a8757 commit a4e728f
Show file tree
Hide file tree
Showing 93 changed files with 2,189 additions and 295 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/run-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ jobs:
runs-on: ubuntu-latest

container:
image: ghcr.io/ctsit/rstudio-ci:4.2.1
image: ghcr.io/ctsit/rstudio-ci:latest

env:
CI: "TRUE"
Expand Down
5 changes: 5 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@
*.zip
site
!inst/testdata/*.csv
report/*.html
report/*.pdf
report/*.txt



# env file
*.env
Expand Down
19 changes: 14 additions & 5 deletions DESCRIPTION
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
Package: redcapcustodian
Type: Package
Title: Data automation for R-centric workflows with a nod towards REDCap
Version: 1.14.1
Version: 1.26.2
Authors@R: c(
person("Philip", "Chase",
email = "[email protected]",
Expand All @@ -23,6 +23,10 @@ Authors@R: c(
email = "[email protected]",
role = "aut",
comment=c(ORCID = "0000-0002-5790-4268")),
person("Sai Pavan", "Kamma",
email = "[email protected]",
role = "aut",
comment=c(ORCID = "0009-0004-4619-0409")),
person("Christopher", "Barnes",
email = "[email protected]",
role = "ctb",
Expand Down Expand Up @@ -55,17 +59,22 @@ Imports:
tibble,
tidyr,
vctrs,
jsonlite
jsonlite,
openxlsx,
quarto
Suggests:
RSQLite,
digest,
dotenv,
duckdb,
fs,
kableExtra,
knitr (>= 1.18),
rmarkdown (>= 2.0),
testthat (>= 3.0.0)
testthat (>= 3.0.0),
tidyverse
VignetteBuilder: knitr
Config/testthat/edition: 3
RoxygenNote: 7.2.1
RoxygenNote: 7.3.2
Depends:
R (>= 2.10)
R (>= 3.5.0)
8 changes: 6 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
FROM --platform=linux/amd64 rocker/verse:4.2.1
FROM --platform=linux/amd64 rocker/verse:4.4.1

WORKDIR /home/rocker

RUN apt update -y && apt install -y libmariadb-dev libmariadbclient-dev
RUN apt update -y && apt install -y libmariadb-dev libmariadb-dev-compat
RUN apt install -y --no-install-recommends libxt6

# returns an error but tlmgr is updated to 2024 regardless
RUN wget ${CTAN_REPO}/update-tlmgr-latest.sh && bash update-tlmgr-latest.sh; exit 0

# install necessary libraries
RUN R -e "install.packages(c( \
'DBI', \
Expand All @@ -23,6 +26,7 @@ RUN R -e "install.packages(c( \
'sendmailR', \
'sqldf', \
'writexl', \
'openxlsx', \
'kableExtra' \
))"

Expand Down
12 changes: 12 additions & 0 deletions NAMESPACE
Original file line number Diff line number Diff line change
Expand Up @@ -9,23 +9,31 @@ export(create_allocation_rows)
export(create_randomization_row)
export(create_test_table)
export(create_test_tables)
export(dataframe_to_redcap_dictionary)
export(dataset_diff)
export(delete_project)
export(disable_non_interactive_quit)
export(enable_randomization_on_a_preconfigured_project_in_production)
export(evaluate_checksums)
export(expire_user_project_rights)
export(export_allocation_tables_from_project)
export(get_bad_emails_from_individual_emails)
export(get_bad_emails_from_listserv_digest)
export(get_current_time)
export(get_hipaa_disclosure_log_from_ehr_fhir_logs)
export(get_institutional_person_data)
export(get_job_duration)
export(get_package_scope_var)
export(get_project_instance)
export(get_project_life_cycle)
export(get_project_name)
export(get_redcap_credentials)
export(get_redcap_db_connection)
export(get_redcap_email_revisions)
export(get_redcap_emails)
export(get_script_name)
export(get_script_run_time)
export(get_table_checksum)
export(get_test_table_names)
export(init_etl)
export(init_log_con)
Expand All @@ -36,10 +44,13 @@ export(log_job_failure)
export(log_job_success)
export(mutate_columns_to_posixct)
export(quit_non_interactive_run)
export(render_report)
export(scrape_user_api_tokens)
export(send_email)
export(set_package_scope_var)
export(set_project_api_token)
export(set_project_instance)
export(set_project_name)
export(set_script_name)
export(set_script_run_time)
export(set_super_api_token)
Expand All @@ -56,5 +67,6 @@ export(write_info_log_entry)
export(write_summary_metrics)
export(write_to_sql_db)
importFrom(magrittr,"%>%")
importFrom(openxlsx,write.xlsx)
importFrom(rlang,.data)
importFrom(sendmailR,"sendmail")
84 changes: 84 additions & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,87 @@
# redcapcustodian 1.26.2 (released 2024-11-20)
- Add new production status (@saipavan10-git, #168, #170)
- Add log event tables 10,11,12 (@saipavan10-git, #168, #170)

# redcapcustodian 1.26.1 (released 2024-11-03)
- Use latest rstudio-ci in run-tests.yaml (@pbchase, #167)
- Address check errors (@pbchase, #167)
- remove log and output folder from study_template (@ljwoodley, #167)
- write log file to tempdir (@ljwoodley, #167)

# redcapcustodian 1.26.0 (released 2024-10-30)
- Update Dockerfile to rocker/verse:4.4.1 (@pbchase)

# redcapcustodian 1.25.0 (released 2024-10-24)
- Add report crash logging to render_report.R (@ljwoodley, #166)
- Update render_report.R to use render_report() function (@ljwoodley, #166)
- Add render_report() (@ljwoodley, #166)

# redcapcustodian 1.24.0 (released 2024-10-17)
- Add job failure alerts with run_etl.R and updates to send_mail() (@ljwoodley, #100, #165)
- Add Sai as author in DESCRIPTION (@pbchase)
- Filter out deleted projects in scrape_user_api_tokens() (@pbchase, #163, #164)
- Move Roxygen2 to 7.3.2 (@pbchase)
- Add get_hipaa_disclosure_log_from_ehr_fhir_logs() (@pbchase, #158)
- Update run-tests.yaml to use rstudio-ci:4.3.3 (@pbchase)

# redcapcustodian 1.23.0 (released 2024-06-13)
- Add project and instance to logging (@ljwoodley, @pbchase, #159, #160)

# redcapcustodian 1.22.2 (released 2024-04-26)
- Restore 'writexl' to Dockerfile (@pbchase)

# redcapcustodian 1.22.1 (released 2024-04-23)
- Update run-tests.yaml to use rstudio-ci:4.3.3 (@pbchase, #157)
- Switch send_mail() to use openxlsx (@ljwoodley, #152, #157)

# redcapcustodian 1.22.0 (released 2024-03-26)
- Implement hacky fix for tlmgr 2023 being unable to install packages (@ChemiKyle, #156)
- Fix bug that prevented email_body from being included in email (@ljwoodley, @ChemiKyle, #155)
- Add 'Scraping one user's API tokens' section to vignettes/credential-scraping.Rmd (@pbchase, @ChemiKyle, #154)
- Update scrape_user_api_tokens() to tidyselect 1.2 standards (@pbchase, #154)

# redcapcustodian 1.21.0 (released 2024-03-15)
- Add attachment management to send_email() allowing lists of files or dataframes to be attached to an email (@ljwoodley, #152, #153)

# redcapcustodian 1.20.0 (released 2024-02-28)
- Add get_redcap_credentials() (@ljwoodley, #149, #151)
- Revert "add redcap wrapper functions" (@ljwoodley, #149, #150)

# redcapcustodian 1.19.0 (released 2024-01-30)
- Add REDCapR wrapper functions (@ljwoodley, #147, #148)

# redcapcustodian 1.18.0 (released 2024-01-10)
- Update Dockerfile to verse:4.3.2 (@pbchase)

# redcapcustodian 1.17.4 (released 2023-11-22)
- Turn off code chunks in job_logging.Rmd to fix pkgdown::deploy_to_branch() errors (@pbchase)

# redcapcustodian 1.17.3 (released 2023-11-21)
- Fix 4 intermittent test failures caused by environment vars (@pbchase, #143)
- Fix error in test 'init_etl properly sets script name, script run time, and initializes log connection' (@pbchase, #143)
- Fix 11 warnings about dplyr::all_equal() in tests (@pbchase, #143)
- Address lots of pkgdown::build_site() complaints about garbage-collected connections (@pbchase, @ChemiKyle, #143)
- Address pkgdown::build_site() complaints about non-parsable code examples (@pbchase, @ChemiKyle, #104, #143)
- Update vignettes/job_logging.Rmd to fix build errors (@pbchase, #143)

# redcapcustodian 1.17.2 (released 2023-11-17)
- Update vignettes/job_logging.Rmd to fix build errors (@pbchase, #142)

# redcapcustodian 1.17.1 (released 2023-11-17)
- Fix build failures caused by vignettes/job_logging.Rmd (@pbchase)

# redcapcustodian 1.17.0 (released 2023-11-17)
- Add job_logging vignette (@pbchase, #113, #115)
- Add objects_to_include param to unnest_job_summary_data_json_object() (@pbchase, #112, #115)

# redcapcustodian 1.16.0 (released 2023-11-16)
- Add MySQL database comparison tools (@pbchase, @ChemiKyle, #140)

# redcapcustodian 1.15.0 (released 2023-10-31)
- Fix test data for get_redcap_email_revisions() (@pbchase)
- Add delete_project() (@ljwoodley, #139)
- Add dataframe_to_redcap_dictionary() (@ljwoodley, #136)

# redcapcustodian 1.14.1 (released 2023-08-24)
- Update etl/delete_erroneous_survey_reminders.R (@pbchase, #137)

Expand Down
29 changes: 15 additions & 14 deletions R/credential_management.R
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ scrape_user_api_tokens <- function(conn, username_to_scrape = Sys.info()[["user"
# collect super API token if one exists
super_credentials <- dplyr::tbl(conn, "redcap_user_information") %>%
dplyr::filter(.data$username == username_to_scrape) %>%
dplyr::select(.data$username, .data$api_token) %>%
dplyr::select("username", "api_token") %>%
dplyr::collect() %>%
dplyr::mutate(project_id = 0) %>%
dplyr::filter(!is.na(.data$api_token)) %>%
Expand All @@ -32,25 +32,29 @@ scrape_user_api_tokens <- function(conn, username_to_scrape = Sys.info()[["user"
dplyr::filter(.data$username == username_to_scrape) %>%
dplyr::filter(!is.na(.data$api_token)) %>%
dplyr::select(
.data$project_id,
.data$username,
.data$api_token
"project_id",
"username",
"api_token"
) %>%
# add project information
dplyr::left_join(
dplyr::tbl(conn, "redcap_projects") %>%
dplyr::select(
.data$project_id,
.data$app_title
"project_id",
"app_title",
"date_deleted"
),
by = "project_id"
) %>%
dplyr::collect() %>%
# filter out deleted projects
dplyr::filter(is.na(.data$date_deleted)) |>
dplyr::select(-"date_deleted") |>
# bind_rows used over rbind to avoid need to align column order
dplyr::bind_rows(super_credentials) %>%
dplyr::rename(
project_display_name = .data$app_title,
token = .data$api_token # rename for compatibility with REDCapR credential objects
project_display_name = "app_title",
token = "api_token" # rename for compatibility with REDCapR credential objects
)

return(credentials)
Expand Down Expand Up @@ -125,12 +129,9 @@ set_project_api_token <- function(conn, username, project_id) {
DBI::dbExecute(conn, sql)
}



save_credentials <- function(
file_path,
project_id = "0",
token
) {
file_path,
project_id = "0",
token) {

}
72 changes: 72 additions & 0 deletions R/dataframe_to_redcap_dictionary.R
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
#' Create a REDCap data dictionary from a dataframe
#'
#' @param df the dataframe to generate the data dictionary for
#' @param form_name the form name to display in REDCap
#' @param record_id_col a column in the dataframe that uniquely identifies each record
#'
#' @return A redcap data dictionary
#' @export
#'
#' @examples
#' \dontrun{
#'
#' df <- data.frame(
#' pk_col = c("a1", "a2", "a3"),
#' integer_col = c(1, 2, 3),
#' numeric_col = 5.9,
#' character_col = c("a", "b", "c"),
#' date_col = as.Date("2011-03-07"),
#' date_time_col = as.POSIXct(strptime("2011-03-27 01:30:00", "%Y-%m-%d %H:%M:%S")),
#' email_col = c("[email protected]", "[email protected]", "[email protected]")
#' )
#'
#' redcap_data_dictionary <- dataframe_to_redcap_dictionary(df, "test_form")
#' redcap_data_dictionary <- dataframe_to_redcap_dictionary(df, "test_form", "character_col")
#' }
dataframe_to_redcap_dictionary <- function(df,
form_name,
record_id_col = NULL) {
contains_emails <- function(col) {
email_pattern <- "^[A-Za-z0-9._%-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,4}$"
return(any(grepl(email_pattern, col)))
}

get_validation_type <- function(col) {
if (contains_emails(col)) {
return("email")
}

col_type <- class(col)[1]
switch(
col_type,
"numeric" = "number",
"integer" = "number",
"Date" = "datetime_ymd",
"POSIXct" = "datetime_seconds_ymd",
as.character(NA)
)
}

# Rearrange the dataframe if a record_id_col is provided
if (!is.null(record_id_col)) {
ordered_df <- df |>
dplyr::select(record_id_col, dplyr::everything())
} else {
ordered_df <- df
}

redcap_data_dictionary <- data.frame(
field_name = names(ordered_df),
form_name = form_name,
section_header = as.character(NA),
field_type = "text",
field_label = names(ordered_df),
select_choices_or_calculations = as.character(NA),
field_note = as.character(NA),
text_validation_type_or_show_slider_number = sapply(ordered_df, get_validation_type)
)

rownames(redcap_data_dictionary) <- NULL

return(redcap_data_dictionary)
}
Loading

0 comments on commit a4e728f

Please sign in to comment.