Add activity after death

NAMESPACE

@@ -25,6 +25,7 @@ export(end_fy_quarter)
 export(end_next_fy_quarter)
 export(find_latest_file)
 export(fy_interval)
+export(get_all_slf_deaths_lookup_path)
 export(get_boxi_extract_path)
 export(get_ch_costs_path)
 export(get_dd_path)
@@ -91,6 +92,7 @@ export(process_costs_ch_rmd)
 export(process_costs_dn_rmd)
 export(process_costs_gp_ooh_rmd)
 export(process_costs_hc_rmd)
+export(process_deaths_lookup)
 export(process_extract_acute)
 export(process_extract_ae)
 export(process_extract_alarms_telecare)

R/add_activity_after_death_flag.R

@@ -0,0 +1,184 @@
+#' Match on BOXI NRS death dates to process activity after death flag
+#'
+#' @description Match on CHI number where available in the episode file, and add date of death from the BOXI NRS lookup.
+#' Create new activity after death flag
+#'
+#' @param data episode files
+#' @param year financial year, e.g. '1920'
+#' @param deaths_data The death data for the year
+#'
+#' @return data flagged if activity after death
+add_activity_after_death_flag <- function(
+    data,
+    year,
+    deaths_data = read_file(get_all_slf_deaths_lookup_path()) %>%
+      slfhelper::get_chi()) {
+  death_joined <- data %>%
+    dplyr::select(.data$year, .data$chi, .data$record_keydate1, .data$record_keydate2, .data$death_date, .data$deceased) %>%
+    dplyr::filter(!is.na(.data$chi) | .data$chi != "") %>%
+    dplyr::left_join(
+      deaths_data,
+      by = "chi",
+      suffix = c("", "_boxi")
+    ) %>%
+    dplyr::filter(.data$deceased == TRUE | .data$deceased_boxi == TRUE) %>%
+    dplyr::distinct()
+
+
+  # Check and print error message for records which already have a death_date in the episode file, but this doesn't match the BOXI death date
+  check_death_date_match <- death_joined %>%
+    dplyr::filter(.data$death_date != .data$death_date_boxi)
+
+  if (nrow(check_death_date_match) != 0) {
+    warning("There were records in the episode file which already have a death_date, but does not match the BOXI NRS death date.")
+  }
+
+
+  # Check and print error message for records which have a record_keydate1 after their BOXI death date
+  check_keydate1_death_date <- death_joined %>%
+    dplyr::filter(.data$record_keydate1 > .data$death_date_boxi)
+
+  if (nrow(check_death_date_match) != 0) {
+    warning("There were records in the episode file which have a record_keydate1 after the BOXI NRS death date.")
+  }
+
+
+  flag_data <- death_joined %>%
+    dplyr::mutate(
+      flag_keydate1 = dplyr::if_else(.data$record_keydate1 > .data$death_date_boxi, 1, 0),
+      flag_keydate2 = dplyr::if_else(.data$record_keydate2 > .data$death_date_boxi, 1, 0),
+
+      # Next flag records with 'ongoing' activity after date of death (available from BOXI) if keydate2 is missing and the death date occurs in
+      # in the current or a previous financial year.
+      flag_keydate2_missing = dplyr::if_else(((is.na(.data$record_keydate2) | .data$record_keydate2 == "") & (.data$death_date_boxi <= paste0("20", substr(.data$year, 3, 4), "-03-31"))), 1, 0),
+
+      # Also flag records without a death_date in the episode file, but the BOXI death date occurs in the current or a previous financial year.
+      flag_deathdate_missing = dplyr::if_else(((is.na(.data$death_date) | .data$death_date == "") & (.data$death_date_boxi <= paste0("20", substr(.data$year, 3, 4), "-03-31"))), 1, 0)
+    ) %>%
+    # These should be flagged by one of the two lines of code above, but in these cases, we will also fill in the blank death date if appropriate
+
+    # Search all variables beginning with "flag_" for value "1" and create new variable to flag cases where 1 is present
+    # Multiplying by 1 changes flag from true/false to 1/0
+    dplyr::mutate(activity_after_death = purrr::pmap_dbl(
+      dplyr::select(., tidyselect::contains("flag_")),
+      ~ any(grepl("^1$", c(...)),
+        na.rm = TRUE
+      ) * 1
+    ))
+
+
+  # Check and print error message for records which already are TRUE for the deceased variable in the episode file, but this doesn't match the
+  # BOXI deceased variable
+  check_deceased_match <- flag_data %>%
+    dplyr::filter(.data$deceased != .data$deceased_boxi)
+
+  if (nrow(check_deceased_match) != 0) {
+    warning("There were records in the episode file which have a deceased variable which does not match the BOXI NRS deceased variable")
+  }
+
+
+  # Fill in date of death if missing in the episode file but available in BOXI lookup, due to historic dates of death not being carried
+  # over from previous financial years
+  flag_data <- flag_data %>%
+    dplyr::filter(.data$activity_after_death == 1) %>%
+    # Remove temporary flag variables used to create activity after death flag and fill in missing death_date
+    dplyr::select(.data$year, .data$chi, .data$record_keydate1, .data$record_keydate2, .data$activity_after_death) %>%
+    dplyr::distinct()
+
+  # Match activity after death flag back to episode file
+  final_data <- data %>%
+    dplyr::left_join(
+      flag_data,
+      by = c("year", "chi", "record_keydate1", "record_keydate2"),
+      na_matches = "never"
+    )
+
+
+  return(final_data)
+}
+
+
+#' Create and read SLF Deaths lookup from processed BOXI NRS deaths extracts
+#'
+#' #' @description The BOXI NRS deaths extract lookup should be created after the extract files for all years have been processed,
+# but before an episode file has been produced. Therefore, all BOXI NRS years should be run before running episode files.
+#'
+#' @param file_path Path to the BOXI NRS file for each financial year - may not use this
+#' @param year The year to process, in FY format - may not use this
+#'
+#' @param write_to_disk (optional) Should the data be written to disk default is
+#' `TRUE` i.e. write the data to disk.
+#'
+#' @return the final data as a [tibble][tibble::tibble-package].
+#' @export
+#'
+#'
+#'
+# Read data------------------------------------------------
+process_deaths_lookup <- function(update = latest_update(),
+                                  write_to_disk = TRUE, ...) {
+  all_boxi_deaths <- read_file(get_slf_deaths_lookup_path("1415")) %>%
+    rbind(read_file(get_slf_deaths_lookup_path("1516"))) %>%
+    rbind(read_file(get_slf_deaths_lookup_path("1617"))) %>%
+    rbind(read_file(get_slf_deaths_lookup_path("1718"))) %>%
+    rbind(read_file(get_slf_deaths_lookup_path("1819"))) %>%
+    rbind(read_file(get_slf_deaths_lookup_path("1920"))) %>%
+    rbind(read_file(get_slf_deaths_lookup_path("2021"))) %>%
+    rbind(read_file(get_slf_deaths_lookup_path("2122"))) %>%
+    rbind(read_file(get_slf_deaths_lookup_path("2223"))) %>%
+    rbind(read_file(get_slf_deaths_lookup_path("2324"))) %>%
+    # Can this be automated to pick up files starting with name "get_slf_deaths_lookup_path"?
+    slfhelper::get_chi() %>%
+    # Remove rows with missing or blank CHI number - could also use na.omit?
+    # na.omit(all_boxi_deaths)
+    dplyr::filter(!is.na(.data$chi) | chi != "")
+
+  # Check all CHI numbers are valid
+  chi_check <- all_boxi_deaths %>%
+    dplyr::pull(.data$chi) %>%
+    phsmethods::chi_check()
+
+  if (!all(chi_check %in% c("Valid CHI", "Missing (Blank)", "Missing (NA)"))) {
+    # There are some Missing (NA) values in the extracts, but I have excluded them above as they cannot be matched to episode file
+    stop("There were bad CHI numbers in the BOXI NRS file")
+  }
+
+  # Check and print error message for chi numbers with more than one death date
+  duplicates <- all_boxi_deaths %>%
+    janitor::get_dupes(.data$chi)
+
+  if (nrow(duplicates) != 0) {
+    # There are some Missing (NA) values in the extracts, but I have excluded them above as they cannot be matched to episode file
+    warning("There were duplicate death dates in the BOXI NRS file.")
+  }
+
+
+  # We decided to include duplicates as unable to determine which is correct date (unless IT can tell us, however, they don't seem to know
+  # the process well enough), and overall impact will be negligible
+  # Get anon_chi and use this to match onto episode file later
+  all_boxi_deaths <- all_boxi_deaths %>%
+    slfhelper::get_anon_chi()
+
+  # Save out duplicates for further investigation if needed (as anon_chi)
+  if (!missing(duplicates)) {
+    write_file(
+      duplicates,
+      fs::path(get_slf_dir(), "Deaths",
+        file_name = stringr::str_glue("slf_deaths_duplicates_{update}.parquet")
+      )
+    )
+  }
+
+  # Maybe save as its own function
+  # Write the all BOXI NRS deaths lookup file to disk, so this can be used to populate activity after death flag in each episode file
+  if (write_to_disk) {
+    write_file(
+      all_boxi_deaths,
+      fs::path(get_slf_dir(), "Deaths",
+        file_name = stringr::str_glue("anon-all_slf_deaths_lookup_{update}.parquet")
+      )
+    )
+  }
+
+  return(all_boxi_deaths)
+}

R/create_episode_file.R

@@ -79,6 +79,8 @@
         "hscp",
         "datazone2011",
         "attendance_status",
+        "death_date",
+        "deceased",
         "deathdiag1",
         "deathdiag2",
         "deathdiag3",
@@ -139,6 +141,7 @@
       year,
       slf_deaths_lookup
     ) %>%
+    add_activity_after_death_flag(year, deaths_data = read_file(all_slf_deaths_lookup_path())) %>%
     load_ep_file_vars(year)
 
   if (!check_year_valid(year, type = c("ch", "hc", "at", "sds"))) {
@@ -260,12 +263,12 @@
 fill_missing_cij_markers <- function(data) {
   fixable_data <- data %>%
     dplyr::filter(
       .data[["recid"]] %in% c("01B", "04B", "GLS", "02B", "DD") & !is.na(.data[["chi"]])
     )
 
   non_fixable_data <- data %>%
     dplyr::filter(
       !(.data[["recid"]] %in% c("01B", "04B", "GLS", "02B", "DD")) | is.na(.data[["chi"]])
     )
 
   fixed_data <- fixable_data %>%
@@ -361,7 +364,7 @@
     dplyr::mutate(
       cost_total_net_inc_dnas = .data$cost_total_net,
       # In the Cost_Total_Net column set the cost for
       # those with attendance status 5 or 8 (CNWs and DNAs)
       cost_total_net = dplyr::if_else(
         .data$attendance_status %in% c(5L, 8L),
         0.0,

R/get_slf_lookup_paths.R

@@ -60,15 +60,42 @@
 #' @family slf lookup file path
 #' @seealso [get_file_path()] for the generic function.
 get_slf_deaths_lookup_path <- function(year, ...) {
+  # Review the naming convention of this path and file
   slf_deaths_lookup_path <- get_file_path(
     directory = fs::path(get_slf_dir(), "Deaths"),
-    file_name = stringr::str_glue("slf_deaths_lookup_{year}.parquet"),
+    file_name = stringr::str_glue("anon-slf_deaths_lookup_{year}.parquet"),
     ...
   )
 
   return(slf_deaths_lookup_path)
 }
 
+#' SLF death dates File Path
+#'
+#' @description Get the full path to the BOXI NRS Deaths lookup file for all financial years
+#'
+#' @inheritParams get_boxi_extract_path
+#' @param ... additional arguments passed to [get_file_path()]
+#' @param year financial year e.g. "1920"
+#'
+#' @export
+#' @family slf lookup file path
+#' @seealso [get_file_path()] for the generic function.
+
+get_all_slf_deaths_lookup_path <- function(update = latest_update(), ...) {
+  # Note this name is very similar to the existing slf_deaths_lookup_path which returnsthe path for
+  # the processed BOXI extract for each financial year. This function will return the combined financial
+  # years lookup i.e. all years put together.
+  all_slf_deaths_lookup_path <- get_file_path(
+    directory = fs::path(get_slf_dir(), "Deaths"),
+    file_name = stringr::str_glue("anon-all_slf_deaths_lookup_{update}.parquet"),
+    ...
+  )
+  return(all_slf_deaths_lookup_path)
+}
+
+
+
 #' SLF CHI Deaths File Path
 #'
 #' @description Get the full path to the CHI deaths file