Update older years to bring the data in line with our newest processes.

R/00-update_refs.R

@@ -64,19 +64,6 @@ get_dd_period <- function() {
   "Jul16_Jun24"
 }
 
-#' The latest financial year for Cost uplift setting
-#'
-#' @description Get the latest year for cost uplift
-#'
-#' @return The financial year format
-#'
-#' @export
-#'
-#' @family initialisation
-latest_cost_year <- function() {
-  "2324"
-}
-
 #' The year list for slf to update
 #'
 #' @description Get the vector of years to update slf

R/add_activity_after_death_flag.R

@@ -26,7 +26,7 @@ add_activity_after_death_flag <- function(
       by = "chi",
       suffix = c("", "_boxi")
     ) %>%
-    dplyr::filter(.data$deceased == TRUE | .data$deceased_boxi == TRUE) %>%
+    dplyr::filter(.data$deceased == TRUE) %>%
     dplyr::distinct()
 
 
@@ -72,16 +72,6 @@ add_activity_after_death_flag <- function(
     ))
 
 
-  # Check and print error message for records which already are TRUE for the deceased variable in the episode file, but this doesn't match the
-  # BOXI deceased variable
-  check_deceased_match <- flag_data %>%
-    dplyr::filter(.data$deceased != .data$deceased_boxi)
-
-  if (nrow(check_deceased_match) != 0) {
-    warning("There were records in the episode file which have a deceased variable which does not match the BOXI NRS deceased variable")
-  }
-
-
   # Fill in date of death if missing in the episode file but available in BOXI lookup, due to historic dates of death not being carried
   # over from previous financial years
   flag_data <- flag_data %>%
@@ -94,13 +84,15 @@ add_activity_after_death_flag <- function(
   final_data <- data %>%
     dplyr::left_join(
       flag_data,
+      # TODO: this join_by is not 100% accurate. Consider use ep_file_row_id to join
       by = c("year", "chi", "record_keydate1", "record_keydate2"),
       na_matches = "never"
     ) %>%
     dplyr::mutate(death_date = lubridate::as_date(ifelse(is.na(death_date) & !(is.na(death_date_boxi)),
       death_date_boxi, death_date
     ))) %>%
-    dplyr::select(-death_date_boxi)
+    dplyr::select(-death_date_boxi, -fy) %>%
+    dplyr::distinct()
 
 
 

R/check_year_valid.R

@@ -17,6 +17,7 @@ check_year_valid <- function(
       "ch",
       "client",
       "cmh",
+      "cost_dna",
       "dd",
       "deaths",
       "dn",
@@ -34,9 +35,9 @@ check_year_valid <- function(
     )) {
   if (year <= "1415" && type %in% c("dn", "sparra")) {
     return(FALSE)
-  } else if (year <= "1516" && type %in% c("cmh", "homelessness")) {
+  } else if (year <= "1516" && type %in% c("cmh", "homelessness", "dd")) {
     return(FALSE)
-  } else if (year <= "1617" && type %in% c("ch", "hc", "sds", "at")) {
+  } else if (year <= "1617" && type %in% c("ch", "hc", "sds", "at", "client", "cost_dna")) {
     return(FALSE)
   } else if (year <= "1718" && type %in% "hhg") {
     return(FALSE)
@@ -46,7 +47,7 @@ check_year_valid <- function(
     return(FALSE)
   } else if (year >= "2425" && type %in% "sparra") {
     return(FALSE)
-  } else if (year >= "2425" && type %in% c("ch", "hc", "sds", "at")) {
+  } else if (year >= "2526" && type %in% c("ch", "hc", "sds", "at")) {
     return(FALSE)
   }
 

R/cost_uplift.R

@@ -86,3 +86,21 @@ lookup_uplift <- function(data) {
 
   return(data)
 }
+
+#' The latest financial year for Cost uplift setting
+#'
+#' @description Get the latest year for cost uplift
+#' latest_cost_year() is hard coded in cost_uplift().
+#' 2223 is not changed automatically with time passes.
+#' It is changed only when we get a new instruction from somewhere about cost uplift.
+#' Do not change unless specific instructions.
+#' Changing this means that we need to change cost_uplift().
+#'
+#' @return The financial year format
+#'
+#' @export
+#'
+#' @family initialisation
+latest_cost_year <- function() {
+  "2223"
+}

R/create_episode_file.R

@@ -175,7 +175,65 @@ create_episode_file <- function(
         sc_social_worker = NA,
         sc_type_of_housing = NA,
         sc_meals = NA,
-        sc_day_care = NA
+        sc_day_care = NA,
+        social_care_id = NA,
+        sc_dementia = NA,
+        sc_learning_disability = NA,
+        sc_mental_health_disorders = NA,
+        sc_physical_and_sensory_disability = NA,
+        sc_drugs = NA,
+        sc_alcohol = NA,
+        sc_palliative_care = NA,
+        sc_carer = NA,
+        sc_elderly_frail = NA,
+        sc_neurological_condition = NA,
+        sc_autism = NA,
+        sc_other_vulnerable_groups = NA,
+        ch_provider_description = NA
+      )
+  }
+
+  if (!check_year_valid(year, type = "homelessness")) {
+    episode_file <- episode_file %>%
+      dplyr::mutate(
+        hl1_12_months_post_app = NA,
+        hl1_12_months_pre_app = NA,
+        hl1_6after_ep = NA,
+        hl1_6before_ep = NA,
+        hl1_application_ref = NA,
+        hl1_completeness = NA,
+        hl1_during_ep = NA,
+        hl1_in_fy = NA,
+        hl1_property_type = NA,
+        hl1_reason_ftm = NA,
+        hl1_sending_lca = NA
+      )
+  }
+
+  if (!check_year_valid(year, type = "dd")) {
+    episode_file <- episode_file %>%
+      dplyr::mutate(
+        cij_delay = NA,
+        dd_quality = NA,
+        dd_responsible_lca = NA,
+        delay_end_reason = NA,
+        primary_delay_reason = NA,
+        secondary_delay_reason = NA,
+      )
+  }
+
+  if (!check_year_valid(year, type = "dn")) {
+    episode_file <- episode_file %>%
+      dplyr::mutate(
+        ccm = NA,
+        total_no_dn_contacts = NA
+      )
+  }
+
+  if (!check_year_valid(year, type = "cost_dna")) {
+    episode_file <- episode_file %>%
+      dplyr::mutate(
+        cost_total_net_inc_dnas = NA
       )
   }
 
@@ -463,6 +521,11 @@ join_sc_client <- function(data,
                            file_type = c("episode", "individual")) {
   cli::cli_alert_info("Join social care client function started at {Sys.time()}")
 
+  if (!check_year_valid(year, type = "client")) {
+    data_file <- data
+    return(data_file)
+  }
+
   if (file_type == "episode") {
     # Match on client variables by chi
     data_file <- data %>%

R/create_individual_file.R

@@ -78,7 +78,7 @@ create_individual_file <- function(
 
   if (!check_year_valid(year, type = c("ch", "hc", "at", "sds"))) {
     individual_file <- individual_file %>%
-      aggregate_by_chi(exclude_sc_var = TRUE)
+      aggregate_by_chi(year = year, exclude_sc_var = TRUE)
   } else {
     individual_file <- individual_file %>%
       aggregate_ch_episodes() %>%
@@ -115,6 +115,9 @@ create_individual_file <- function(
         hc_personal_hours = NA,
         hc_non_personal_hours = NA,
         hc_reablement_hours = NA,
+        hc_non_personal_hours_cost = NA,
+        hc_personal_hours_cost = NA,
+        hc_reablement_hours_cost = NA,
         at_alarms = NA,
         at_telecare = NA,
         sds_option_1 = NA,
@@ -125,10 +128,20 @@ create_individual_file <- function(
         sc_support_from_unpaid_carer = NA,
         sc_social_worker = NA,
         sc_meals = NA,
-        sc_day_care = NA
+        sc_day_care = NA,
+        sc_type_of_housing = NA,
+        count_not_known = NA,
+        sc_latest_submission = NA,
+        social_care_id = NA
       )
   }
 
+  if (!check_year_valid(year, type = "homelessness")) {
+    individual_file <- individual_file %>%
+      dplyr::mutate(hl1_in_fy = NA)
+  }
+
+
   if (anon_chi_out) {
     individual_file <- individual_file %>%
       tidyr::replace_na(list(chi = "")) %>%

R/get_boxi_extract_path.R

@@ -86,9 +86,11 @@ get_boxi_extract_path <- function(
 #'
 #' @return an [fs::path()] to a dummy file which can be used with targets.
 get_dummy_boxi_extract_path <- function() {
-  get_file_path(
+  dummy_path <- get_file_path(
     directory = get_dev_dir(),
     file_name = ".dummy",
     create = TRUE
   )
+
+  return(dummy_path)
 }

R/get_sc_lookup_paths.R

@@ -38,11 +38,18 @@ get_sc_demog_lookup_path <- function(update = latest_update(), ...) {
 #' @family social care lookup file paths
 #' @seealso [get_file_path()] for the generic function.
 get_sc_client_lookup_path <- function(year, update = latest_update(), ...) {
-  sc_client_lookup_path <- get_file_path(
-    directory = fs::path(get_slf_dir(), "Social_care", "processed_sc_client_lookup"),
-    file_name = stringr::str_glue("anon-sc_client_lookup_{year}_{update}.parquet"),
-    ...
-  )
-
-  return(sc_client_lookup_path)
+  if (!check_year_valid(year, type = "client")) {
+    return(get_dummy_boxi_extract_path())
+  } else {
+    sc_client_lookup_path <- get_file_path(
+      directory = fs::path(
+        get_slf_dir(),
+        "Social_care",
+        "processed_sc_client_lookup"
+      ),
+      file_name = stringr::str_glue("anon-sc_client_lookup_{year}_{update}.parquet"),
+      ...
+    )
+    return(sc_client_lookup_path)
+  }
 }

R/join_deaths_data.R

@@ -15,7 +15,8 @@ join_deaths_data <- function(
   return(
     data %>%
       dplyr::left_join(
-        slf_deaths_lookup,
+        slf_deaths_lookup %>%
+          dplyr::distinct(chi, .keep_all = TRUE),
         by = "chi",
         na_matches = "never",
         relationship = "many-to-one"

R/link_delayed_discharge_eps.R

@@ -14,6 +14,12 @@ link_delayed_discharge_eps <- function(
     dd_data = read_file(get_source_extract_path(year, "dd")) %>% slfhelper::get_chi()) {
   cli::cli_alert_info("Link delayed discharge to episode file function started at {Sys.time()}")
 
+  if (!check_year_valid(year, type = "dd")) {
+    episode_file <- episode_file
+    return(episode_file)
+  }
+
+  names_ep <- names(episode_file)
   episode_file <- episode_file %>%
     dplyr::mutate(
       # remember to revoke the cij_end_date with dummy_cij_end
@@ -286,6 +292,11 @@ link_delayed_discharge_eps <- function(
     )) %>%
     dplyr::group_by(.data$chi, .data$cij_marker) %>%
     dplyr::mutate(cij_delay = max(.data$has_delay)) %>%
+    dplyr::mutate(cij_delay = dplyr::if_else(cij_delay == "0",
+      FALSE,
+      TRUE,
+      missing = NA
+    )) %>%
     dplyr::ungroup() %>%
     # add yearstay and monthly beddays
     # count_last = TRUE because DD counts last day and not the first
@@ -299,37 +310,43 @@ link_delayed_discharge_eps <- function(
       yearstay = rowSums(dplyr::pick(dplyr::ends_with("_beddays")))
     ) %>%
     # tidy up and rename columns to match the format of episode files
+    # keep variables from ep files
     dplyr::select(
+      -c(
+        "ep_file_row_id",
+        "year",
+        "recid",
+        "record_keydate1",
+        "record_keydate2",
+        "postcode",
+        "hbtreatcode",
+        "location",
+        "spec",
+        ## following are dummy variables
+        "cij_start_date_lower",
+        "cij_end_date_upper",
+        "cij_end_month",
+        "is_dummy_cij_start",
+        "dummy_cij_start",
+        "is_dummy_cij_end",
+        "dummy_cij_end",
+        "datediff_start",
+        "datediff_end",
+        "has_delay",
+        "is_dummy_keydate2",
+        "dummy_keydate2",
+        "dummy_id"
+      )
+    ) %>%
+    dplyr::rename(
       "year" = "year_dd",
       "recid" = "recid_dd",
       "record_keydate1" = "record_keydate1_dd",
       "record_keydate2" = "record_keydate2_dd",
-      "smrtype",
-      "chi",
-      "gender",
-      "dob",
-      "age",
-      "gpprac",
       "postcode" = "postcode_dd",
-      "dd_responsible_lca",
       "hbtreatcode" = "hbtreatcode_dd",
-      "delay_end_reason",
-      "primary_delay_reason",
-      "secondary_delay_reason",
-      "cij_marker",
-      "cij_start_date",
-      "cij_end_date",
-      "cij_pattype_code",
-      "cij_ipdc",
-      "cij_admtype",
-      "cij_adm_spec",
-      "cij_dis_spec",
-      "cij_delay",
-      "location",
       "spec" = "spec_dd",
-      "dd_quality",
-      dplyr::ends_with("_beddays"),
-      "yearstay"
+      "location" = "location_dd"
     ) %>%
     # Combine DD with episode data
     dplyr::bind_rows(
@@ -345,7 +362,19 @@ link_delayed_discharge_eps <- function(
             "dummy_cij_end"
           )
         )
-    )
+    ) %>%
+    # populate cij_delay dd details back to ep
+    dplyr::group_by(chi, cij_marker) %>%
+    dplyr::mutate(
+      has_dd = any(recid == "DD"),
+      delay_dd = any(cij_delay)
+    ) %>%
+    dplyr::ungroup() %>%
+    dplyr::mutate(cij_delay = dplyr::if_else(has_dd,
+      delay_dd,
+      cij_delay
+    )) %>%
+    dplyr::select(-c("has_dd", "delay_dd", "original_admission_date", "amended_dates"))
 
   return(linked_data)
 }