From 08b9f8085d0a5d7e989736fd76e42fc838db3286 Mon Sep 17 00:00:00 2001
From: Jennit07 <67372904+Jennit07@users.noreply.github.com>
Date: Mon, 13 May 2024 12:32:56 +0100
Subject: [PATCH 01/39] Update NEWS.md

---
 NEWS.md | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 52 insertions(+), 1 deletion(-)

diff --git a/NEWS.md b/NEWS.md
index fdbb64c9c..a2e0aeeea 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,4 +1,55 @@
-# September 2023 Update - Unreleased
+# June 2024 Update - Unreleased
+* Update of 2017/18 onwards to include bug fixes within the files.
+* Removal of extra variable caused by the LTCs not matching properly.
+* Homelessness improvements:
+   * Removal of filtering the data in SLFs according to completeness levels.
+   * New variables:
+      * `hl1_completeness` - a data quality indicator by percentage compared to SG annual publication. 
+      * `hl1_12_months_pre_app`- date variable
+      * `hl1_12_momths_post_app` - date variable 
+* Potential inclusions
+* Activity after death flag?
+* New care home methodology? - potentially this is on hold until September update.
+* Additional Documentation?
+
+# March 2024 Update - Released 20-Mar-2024
+* Update of 2017/18 onwards to include bug fixes within the files.
+* 2023/24 file now includes social care data.
+* Geography files updated - SPD and SIMD
+* Variable `property_type` in homelessness has been updated to include further description
+* Bug fixes:
+  * Service use cohort wrongly assigning Non-Service Users (NSU) as `psychiatry`
+  * Not Applicable (NA) introduced for variable `high_cc` in Demographic cohort
+  * Issue with delayed discharges data not linking to admissions
+  * Person ID available in self-directed support (SDS) data
+  * Issue with Social Care ID - missing sc id were all being set to one sc id.
+ * Improvements to social care methodology
+   * Demographics 
+    * person_id will now be consistent across social care cases for an individual. The social care ID for a CHI will also be consistent across all areas, not just the latest ID used in AT/SDS/CH/HC.
+ * Self-directed Support (SDS) and Alarms Telecare (AT) data
+    * Our tests show this is now in line with the social care team’s publications and therefore, the data may have changed slightly. 
+ * New Social Care methodology 
+    * The new methodology impacts how we match the demographics file and how we select the latest social care ID.
+    * Previously we used the `latest_flag` but this isn’t accurate as some IDs have none flagged, and some have more than one flagged. We now have one social care ID flagged for each CHI. This issue mostly affects Edinburgh, Falkirk, Western Isles, and Renfrewshire.
+    * Previously, in cases where a social care ID had multiple CHIs associated only one of the CHIs was chosen.
+    * The new methodology keeps all CHIs in as there is no way to tell which CHI the activity is for. The new methodology will show duplicate activity but for the different CHIs. The main areas this affects are Midlothian, Western Isles, and Renfrewshire.
+      
+
+# December 2023 Update - Released 20-Dec-2023
+* Update of 2017/18 onwards to include bug fixes within the files.
+* 2023/24 file contain data from 1st April 2023 up to the end of September 2023.
+  * No social care data available.
+* Re-addition of keep population flag.
+* SPARRA update
+* NA's introduced for variable `ch_provider` - now fixed.
+* Future improvements
+  * Activity after death flag
+  * Review of social care methodology.
+* SLFhelper updated to version 10.1.1.
+  * Includes a fix for speeding up function `get_chi()`
+
+
+# September 2023 Update - Released 22-Sep-2023
 * Update of 2017/18 onwards to include bug fixes within the files. 
 * New 2023/24 files.
   *No social care data available for new 2023/24 file.

From 7f569d3481f301689f0fe8a19142924cdabff9ea Mon Sep 17 00:00:00 2001
From: Zihao Li <lizihao_anu@outlook.com>
Date: Fri, 17 May 2024 16:53:22 +0100
Subject: [PATCH 02/39] link GP-OoH with CUP markers

---
 R/get_boxi_extract_path.R  |  1 +
 R/process_extract_gp_ooh.R | 37 +++++++++++++++++++++++++++++++++++--
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/R/get_boxi_extract_path.R b/R/get_boxi_extract_path.R
index 3c2b4acdc..c92ffdc47 100644
--- a/R/get_boxi_extract_path.R
+++ b/R/get_boxi_extract_path.R
@@ -49,6 +49,7 @@ get_boxi_extract_path <- function(
     "gp_ooh-c" ~ "GP-OoH-consultations-extract",
     "gp_ooh-d" ~ "GP-OoH-diagnosis-extract",
     "gp_ooh-o" ~ "GP-OoH-outcomes-extract",
+    "gp_ooh_cup" ~ "GP-OoH-cup-extract",
     "homelessness" ~ "Homelessness-extract",
     "maternity" ~ "Maternity-episode-level-extract",
     "mh" ~ "Mental-Health-episode-level-extract",
diff --git a/R/process_extract_gp_ooh.R b/R/process_extract_gp_ooh.R
index 37cfc8f3f..47c0ee7e5 100644
--- a/R/process_extract_gp_ooh.R
+++ b/R/process_extract_gp_ooh.R
@@ -93,10 +93,41 @@ process_extract_gp_ooh <- function(year, data_list, write_to_disk = TRUE) {
     ) %>%
     dplyr::ungroup()
 
+  ## Link CUP Marker -----
+  gp_ooh_cup_file <- read_file(
+    # path = get_boxi_extract_path(year, "gp_ooh_cup"),
+    path = file_name,
+    col_type = readr::cols(
+      "GP OOH Consultation Start Date" = readr::col_date(format = "%Y/%m/%d %T"),
+      "GP OOH Consultation Start Time" = readr::col_time(""),
+      "GUID" = readr::col_character(),
+      "CUP Marker" = readr::col_integer(),
+      "CUP Pathway Name" = readr::col_character()
+    )
+  ) %>%
+    dplyr::select(
+      record_keydate1 = "GP OOH Consultation Start Date",
+      keytime1 = "GP OOH Consultation Start Time",
+      ooh_case_id = "GUID",
+      cup_marker = "CUP Marker",
+      cup_pathway = "CUP Pathway Name"
+    ) %>%
+    dplyr::distinct(
+      .data$record_keydate1,
+      .data$keytime1,
+      .data$ooh_case_id,
+      .keep_all = TRUE
+    )
+
+  ooh_clean2 = ooh_clean %>%
+    dplyr::left_join(gp_ooh_cup_file,
+                     by = dplyr::join_by("ooh_case_id",
+                                         "record_keydate1",
+                                         "keytime1"))
 
   ## Save Outfile -------------------------------------
 
-  final_data <- ooh_clean %>%
+  final_data <- ooh_clean2 %>%
     dplyr::select(
       "year",
       "recid",
@@ -122,7 +153,9 @@ process_extract_gp_ooh <- function(year, data_list, write_to_disk = TRUE) {
       tidyselect::starts_with("ooh_outcome"),
       "cost_total_net",
       tidyselect::ends_with("_cost"),
-      "ooh_case_id"
+      "ooh_case_id",
+      cup_marker,
+      cup_pathway
     )
 
   if (write_to_disk) {

From 20cdf4d0abf3083a8c8113b4178e17fb3559eb18 Mon Sep 17 00:00:00 2001
From: lizihao-anu <lizihao-anu@users.noreply.github.com>
Date: Fri, 17 May 2024 15:57:19 +0000
Subject: [PATCH 03/39] Style code

---
 R/process_extract_gp_ooh.R | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/R/process_extract_gp_ooh.R b/R/process_extract_gp_ooh.R
index 47c0ee7e5..a1f77822c 100644
--- a/R/process_extract_gp_ooh.R
+++ b/R/process_extract_gp_ooh.R
@@ -119,11 +119,14 @@ process_extract_gp_ooh <- function(year, data_list, write_to_disk = TRUE) {
       .keep_all = TRUE
     )
 
-  ooh_clean2 = ooh_clean %>%
+  ooh_clean2 <- ooh_clean %>%
     dplyr::left_join(gp_ooh_cup_file,
-                     by = dplyr::join_by("ooh_case_id",
-                                         "record_keydate1",
-                                         "keytime1"))
+      by = dplyr::join_by(
+        "ooh_case_id",
+        "record_keydate1",
+        "keytime1"
+      )
+    )
 
   ## Save Outfile -------------------------------------
 

From 3869b098a6e0bdbe7f7bb16fcad91066aa9ba23f Mon Sep 17 00:00:00 2001
From: Zihao Li <lizihao_anu@outlook.com>
Date: Thu, 23 May 2024 11:28:34 +0100
Subject: [PATCH 04/39] update gp ooh cup

---
 R/process_extract_gp_ooh.R | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/R/process_extract_gp_ooh.R b/R/process_extract_gp_ooh.R
index a1f77822c..7fe4431b4 100644
--- a/R/process_extract_gp_ooh.R
+++ b/R/process_extract_gp_ooh.R
@@ -12,7 +12,11 @@
 #' @return the final data as a [tibble][tibble::tibble-package].
 #' @export
 #' @family process extracts
-process_extract_gp_ooh <- function(year, data_list, write_to_disk = TRUE) {
+process_extract_gp_ooh <- function(year,
+                                   data_list,
+                                   gp_ooh_cup_path = get_boxi_extract_path(year, "gp_ooh_cup"),
+                                   write_to_disk = TRUE) {
+
   diagnosis_extract <- process_extract_ooh_diagnosis(data_list[["diagnosis"]], year)
   outcomes_extract <- process_extract_ooh_outcomes(data_list[["outcomes"]], year)
   consultations_extract <- process_extract_ooh_consultations(data_list[["consultations"]], year)
@@ -95,8 +99,7 @@ process_extract_gp_ooh <- function(year, data_list, write_to_disk = TRUE) {
 
   ## Link CUP Marker -----
   gp_ooh_cup_file <- read_file(
-    # path = get_boxi_extract_path(year, "gp_ooh_cup"),
-    path = file_name,
+    path = gp_ooh_cup_path,
     col_type = readr::cols(
       "GP OOH Consultation Start Date" = readr::col_date(format = "%Y/%m/%d %T"),
       "GP OOH Consultation Start Time" = readr::col_time(""),
@@ -119,7 +122,7 @@ process_extract_gp_ooh <- function(year, data_list, write_to_disk = TRUE) {
       .keep_all = TRUE
     )
 
-  ooh_clean2 <- ooh_clean %>%
+  ooh_clean <- ooh_clean %>%
     dplyr::left_join(gp_ooh_cup_file,
       by = dplyr::join_by(
         "ooh_case_id",
@@ -130,7 +133,7 @@ process_extract_gp_ooh <- function(year, data_list, write_to_disk = TRUE) {
 
   ## Save Outfile -------------------------------------
 
-  final_data <- ooh_clean2 %>%
+  final_data <- ooh_clean %>%
     dplyr::select(
       "year",
       "recid",

From a967d5c6917535f6c2012b352a59fa665b586c29 Mon Sep 17 00:00:00 2001
From: Zihao Li <lizihao_anu@outlook.com>
Date: Thu, 23 May 2024 11:31:11 +0100
Subject: [PATCH 05/39] link cup to acute

---
 R/get_boxi_extract_path.R |  2 ++
 R/process_extract_acute.R | 49 +++++++++++++++++++++++++++++++++++----
 R/read_extract_acute.R    |  1 +
 _targets.R                | 12 ++++++++++
 4 files changed, 60 insertions(+), 4 deletions(-)

diff --git a/R/get_boxi_extract_path.R b/R/get_boxi_extract_path.R
index c92ffdc47..3a4e80c59 100644
--- a/R/get_boxi_extract_path.R
+++ b/R/get_boxi_extract_path.R
@@ -16,6 +16,7 @@ get_boxi_extract_path <- function(
       "ae",
       "ae_cup",
       "acute",
+      "acute_cup",
       "cmh",
       "deaths",
       "dn",
@@ -44,6 +45,7 @@ get_boxi_extract_path <- function(
     "ae" ~ "A&E-episode-level-extract",
     "ae_cup" ~ "A&E-UCD-CUP-extract",
     "acute" ~ "Acute-episode-level-extract",
+    "acute_cup" ~ "Actue-cup-extract",
     "cmh" ~ "Community-MH-contact-level-extract",
     "dn" ~ "District-Nursing-contact-level-extract",
     "gp_ooh-c" ~ "GP-OoH-consultations-extract",
diff --git a/R/process_extract_acute.R b/R/process_extract_acute.R
index dcfdb47c0..da4306bcc 100644
--- a/R/process_extract_acute.R
+++ b/R/process_extract_acute.R
@@ -12,7 +12,11 @@
 #' @return the final data as a [tibble][tibble::tibble-package].
 #' @export
 #' @family process extracts
-process_extract_acute <- function(data, year, write_to_disk = TRUE) {
+process_extract_acute <- function(data,
+                                  year,
+                                  acute_cup_path = get_boxi_extract_path(year, "acute_cup"),
+                                  write_to_disk = TRUE) {
+
   # Only run for a single year
   stopifnot(length(year) == 1L)
 
@@ -58,8 +62,43 @@ process_extract_acute <- function(data, year, write_to_disk = TRUE) {
     ) %>%
     # Add oldtadm as a factor with labels
     dplyr::mutate(oldtadm = factor(.data$oldtadm,
-      levels = 0L:8L
-    ))
+                                   levels = 0L:8L
+    )) %>%
+    dplyr::mutate(
+      unique_row_num = dplyr::row_number()
+    )
+
+  acute_cup <- read_file(
+    path = acute_cup_path,
+    col_type = readr::cols(
+      "Acute Admission Date" = readr::col_date(format = "%Y/%m/%d %T"),
+      "Acute Discharge Date" = readr::col_date(format = "%Y/%m/%d %T"),
+      "Acute Admission Type Code" = readr::col_character(),
+      "Acute Discharge Type Code" = readr::col_character(),
+      "Case Reference Number [C]" = readr::col_character(),
+      "UPI Number [C]" = readr::col_character(),
+      "CUP Marker" = readr::col_integer(),
+      "CUP Pathway Name" = readr::col_character()
+    )
+  ) %>% dplyr::select(
+    chi = "UPI Number [C]",
+    case_reference_number = "Case Reference Number [C]",
+    record_keydate1 = "Acute Admission Date",
+    record_keydate2 = "Acute Discharge Date",
+    tadm = "Acute Admission Type Code",
+    disch = "Acute Discharge Type Code",
+    cup_marker = "CUP Marker",
+    cup_pathway = "CUP Pathway Name"
+  ) %>% dplyr::distinct()
+
+  acute_clean <- acute_clean %>%
+    dplyr::left_join(acute_cup,
+                     by = c("record_keydate1",
+                            "record_keydate2",
+                            "case_reference_number",
+                            "chi",
+                            "tadm",
+                            "disch"))
 
   acute_processed <- acute_clean %>%
     dplyr::select(
@@ -106,7 +145,9 @@ process_extract_acute <- function(data, year, write_to_disk = TRUE) {
       "cost_total_net",
       tidyselect::ends_with("_beddays"),
       tidyselect::ends_with("_cost"),
-      "uri"
+      "uri",
+      "cup_marker",
+      "cup_pathway"
     ) %>%
     dplyr::arrange(.data$chi, .data$record_keydate1)
 
diff --git a/R/read_extract_acute.R b/R/read_extract_acute.R
index 7a227db73..6ed66b92c 100644
--- a/R/read_extract_acute.R
+++ b/R/read_extract_acute.R
@@ -149,6 +149,7 @@ read_extract_acute <- function(year, file_path = get_boxi_extract_path(year = ye
       ipdc = "Inpatient Day Case Identifier Code",
       cij_ipdc = "CIJ Inpatient Day Case Identifier Code (01)",
       lineno = "Line Number (01)",
+      case_reference_number = "Case Reference Number [C]",
       GLS_record = "GLS Record"
     ) %>%
     # replace NA in cost_total_net by 0
diff --git a/_targets.R b/_targets.R
index 15d2584bb..ae0cc5f5f 100644
--- a/_targets.R
+++ b/_targets.R
@@ -281,6 +281,16 @@ list(
       get_boxi_extract_path(year = year, type = "gp_ooh-c"),
       format = "file"
     ),
+    tar_target(
+      acute_cup_path,
+      get_boxi_extract_path(year, type = "acute_cup"),
+      format = "file"
+    ),
+    tar_target(
+      gp_ooh_cup_path,
+      get_boxi_extract_path(year, type = "gp_ooh_cup"),
+      format = "file"
+    ),
     tar_qs(
       ooh_data,
       read_extract_gp_ooh(
@@ -294,6 +304,7 @@ list(
     tar_target(source_acute_extract, process_extract_acute(
       acute_data,
       year,
+      acute_cup_path,
       write_to_disk = write_to_disk
     )),
     tar_target(
@@ -419,6 +430,7 @@ list(
     tar_target(source_ooh_extract, process_extract_gp_ooh(
       year,
       ooh_data,
+      gp_ooh_cup_path,
       write_to_disk = write_to_disk
     )),
     tar_target(

From a989b3f7d9c230b85ad21bfe4c250509576def2f Mon Sep 17 00:00:00 2001
From: lizihao-anu <lizihao-anu@users.noreply.github.com>
Date: Thu, 23 May 2024 10:32:53 +0000
Subject: [PATCH 06/39] Update documentation

---
 man/get_boxi_extract_path.Rd  | 4 ++--
 man/process_extract_acute.Rd  | 7 ++++++-
 man/process_extract_gp_ooh.Rd | 7 ++++++-
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/man/get_boxi_extract_path.Rd b/man/get_boxi_extract_path.Rd
index c012ac3ef..69bb886c4 100644
--- a/man/get_boxi_extract_path.Rd
+++ b/man/get_boxi_extract_path.Rd
@@ -6,8 +6,8 @@
 \usage{
 get_boxi_extract_path(
   year,
-  type = c("ae", "ae_cup", "acute", "cmh", "deaths", "dn", "gp_ooh-c", "gp_ooh-d",
-    "gp_ooh-o", "homelessness", "maternity", "mh", "outpatients")
+  type = c("ae", "ae_cup", "acute", "acute_cup", "cmh", "deaths", "dn", "gp_ooh-c",
+    "gp_ooh-d", "gp_ooh-o", "homelessness", "maternity", "mh", "outpatients")
 )
 }
 \arguments{
diff --git a/man/process_extract_acute.Rd b/man/process_extract_acute.Rd
index 77a99cef3..22ff164c8 100644
--- a/man/process_extract_acute.Rd
+++ b/man/process_extract_acute.Rd
@@ -4,7 +4,12 @@
 \alias{process_extract_acute}
 \title{Process the Acute extract}
 \usage{
-process_extract_acute(data, year, write_to_disk = TRUE)
+process_extract_acute(
+  data,
+  year,
+  acute_cup_path = get_boxi_extract_path(year, "acute_cup"),
+  write_to_disk = TRUE
+)
 }
 \arguments{
 \item{data}{The extract to process}
diff --git a/man/process_extract_gp_ooh.Rd b/man/process_extract_gp_ooh.Rd
index ddec006fe..b137f581c 100644
--- a/man/process_extract_gp_ooh.Rd
+++ b/man/process_extract_gp_ooh.Rd
@@ -4,7 +4,12 @@
 \alias{process_extract_gp_ooh}
 \title{Process the GP OoH extract}
 \usage{
-process_extract_gp_ooh(year, data_list, write_to_disk = TRUE)
+process_extract_gp_ooh(
+  year,
+  data_list,
+  gp_ooh_cup_path = get_boxi_extract_path(year, "gp_ooh_cup"),
+  write_to_disk = TRUE
+)
 }
 \arguments{
 \item{year}{The year to process, in FY format.}

From 5beee761d01576c6595a7f3d297c58b0df616ea0 Mon Sep 17 00:00:00 2001
From: Jennifer Thom <jennifer.thom@phs.scot>
Date: Tue, 25 Jun 2024 11:08:41 +0100
Subject: [PATCH 07/39] Update slf deaths lookup function name

---
 NAMESPACE                                              |  2 +-
 R/add_activity_after_death_flag.R                      |  2 +-
 R/create_episode_file.R                                |  2 +-
 R/get_slf_lookup_paths.R                               | 10 +++++-----
 man/add_activity_after_death_flag.Rd                   |  3 ++-
 ..._path.Rd => get_combined_slf_deaths_lookup_path.Rd} |  6 +++---
 man/get_slf_ch_name_lookup_path.Rd                     |  2 +-
 man/get_slf_chi_deaths_path.Rd                         |  2 +-
 man/get_slf_deaths_lookup_path.Rd                      |  2 +-
 man/get_slf_gpprac_path.Rd                             |  2 +-
 man/get_slf_postcode_path.Rd                           |  2 +-
 11 files changed, 18 insertions(+), 17 deletions(-)
 rename man/{get_all_slf_deaths_lookup_path.Rd => get_combined_slf_deaths_lookup_path.Rd} (83%)

diff --git a/NAMESPACE b/NAMESPACE
index 0475603bf..c6bba9b38 100644
--- a/NAMESPACE
+++ b/NAMESPACE
@@ -25,9 +25,9 @@ export(end_fy_quarter)
 export(end_next_fy_quarter)
 export(find_latest_file)
 export(fy_interval)
-export(get_all_slf_deaths_lookup_path)
 export(get_boxi_extract_path)
 export(get_ch_costs_path)
+export(get_combined_slf_deaths_lookup_path)
 export(get_dd_path)
 export(get_dd_period)
 export(get_demographic_cohorts_path)
diff --git a/R/add_activity_after_death_flag.R b/R/add_activity_after_death_flag.R
index 78131d941..b4e0bfd18 100644
--- a/R/add_activity_after_death_flag.R
+++ b/R/add_activity_after_death_flag.R
@@ -11,7 +11,7 @@
 add_activity_after_death_flag <- function(
     data,
     year,
-    deaths_data = read_file(get_all_slf_deaths_lookup_path()) %>%
+    deaths_data = read_file(get_combined_slf_deaths_lookup_path()) %>%
       slfhelper::get_chi()) {
   cli::cli_alert_info("Add activity after death flag function started at {Sys.time()}")
 
diff --git a/R/create_episode_file.R b/R/create_episode_file.R
index 34d2ddf9e..7909e2e7f 100644
--- a/R/create_episode_file.R
+++ b/R/create_episode_file.R
@@ -141,7 +141,7 @@ create_episode_file <- function(
       slf_deaths_lookup
     ) %>%
     add_activity_after_death_flag(year,
-      deaths_data = read_file(get_all_slf_deaths_lookup_path()) %>%
+      deaths_data = read_file(get_combined_slf_deaths_lookup_path()) %>%
         slfhelper::get_chi()
     ) %>%
     load_ep_file_vars(year)
diff --git a/R/get_slf_lookup_paths.R b/R/get_slf_lookup_paths.R
index 390a27a5a..2455be768 100644
--- a/R/get_slf_lookup_paths.R
+++ b/R/get_slf_lookup_paths.R
@@ -81,16 +81,16 @@ get_slf_deaths_lookup_path <- function(year, ...) {
 #' @family slf lookup file path
 #' @seealso [get_file_path()] for the generic function.
 
-get_all_slf_deaths_lookup_path <- function(update = latest_update(), ...) {
-  # Note this name is very similar to the existing slf_deaths_lookup_path which returnsthe path for
+get_combined_slf_deaths_lookup_path <- function(update = latest_update(), ...) {
+  # Note this name is very similar to the existing slf_deaths_lookup_path which returns the path for
   # the processed BOXI extract for each financial year. This function will return the combined financial
   # years lookup i.e. all years put together.
-  all_slf_deaths_lookup_path <- get_file_path(
+  combined_slf_deaths_lookup_path <- get_file_path(
     directory = fs::path(get_slf_dir(), "Deaths"),
-    file_name = stringr::str_glue("anon-all_slf_deaths_lookup_{update}.parquet"),
+    file_name = stringr::str_glue("anon-combined_slf_deaths_lookup_{update}.parquet"),
     ...
   )
-  return(all_slf_deaths_lookup_path)
+  return(combined_slf_deaths_lookup_path)
 }
 
 
diff --git a/man/add_activity_after_death_flag.Rd b/man/add_activity_after_death_flag.Rd
index 36eafe7bd..76a4ca64a 100644
--- a/man/add_activity_after_death_flag.Rd
+++ b/man/add_activity_after_death_flag.Rd
@@ -7,7 +7,8 @@
 add_activity_after_death_flag(
   data,
   year,
-  deaths_data = read_file(get_all_slf_deaths_lookup_path()) \%>\% slfhelper::get_chi()
+  deaths_data = read_file(get_combined_slf_deaths_lookup_path()) \%>\%
+    slfhelper::get_chi()
 )
 }
 \arguments{
diff --git a/man/get_all_slf_deaths_lookup_path.Rd b/man/get_combined_slf_deaths_lookup_path.Rd
similarity index 83%
rename from man/get_all_slf_deaths_lookup_path.Rd
rename to man/get_combined_slf_deaths_lookup_path.Rd
index 2f06b64d3..dd03a0541 100644
--- a/man/get_all_slf_deaths_lookup_path.Rd
+++ b/man/get_combined_slf_deaths_lookup_path.Rd
@@ -1,10 +1,10 @@
 % Generated by roxygen2: do not edit by hand
 % Please edit documentation in R/get_slf_lookup_paths.R
-\name{get_all_slf_deaths_lookup_path}
-\alias{get_all_slf_deaths_lookup_path}
+\name{get_combined_slf_deaths_lookup_path}
+\alias{get_combined_slf_deaths_lookup_path}
 \title{SLF death dates File Path}
 \usage{
-get_all_slf_deaths_lookup_path(update = latest_update(), ...)
+get_combined_slf_deaths_lookup_path(update = latest_update(), ...)
 }
 \arguments{
 \item{update}{the update month (defaults to use \code{\link[=latest_update]{latest_update()}})}
diff --git a/man/get_slf_ch_name_lookup_path.Rd b/man/get_slf_ch_name_lookup_path.Rd
index 2660bbeab..1b0c3f3bb 100644
--- a/man/get_slf_ch_name_lookup_path.Rd
+++ b/man/get_slf_ch_name_lookup_path.Rd
@@ -22,7 +22,7 @@ has official Care Home names and addresses provided by the Care Inspectorate.
 \code{\link[=get_file_path]{get_file_path()}} for the generic function.
 
 Other slf lookup file path: 
-\code{\link{get_all_slf_deaths_lookup_path}()},
+\code{\link{get_combined_slf_deaths_lookup_path}()},
 \code{\link{get_slf_chi_deaths_path}()},
 \code{\link{get_slf_deaths_lookup_path}()},
 \code{\link{get_slf_gpprac_path}()},
diff --git a/man/get_slf_chi_deaths_path.Rd b/man/get_slf_chi_deaths_path.Rd
index 0db72d9d3..8ba115dfe 100644
--- a/man/get_slf_chi_deaths_path.Rd
+++ b/man/get_slf_chi_deaths_path.Rd
@@ -22,7 +22,7 @@ Get the full path to the CHI deaths file
 \code{\link[=get_file_path]{get_file_path()}} for the generic function.
 
 Other slf lookup file path: 
-\code{\link{get_all_slf_deaths_lookup_path}()},
+\code{\link{get_combined_slf_deaths_lookup_path}()},
 \code{\link{get_slf_ch_name_lookup_path}()},
 \code{\link{get_slf_deaths_lookup_path}()},
 \code{\link{get_slf_gpprac_path}()},
diff --git a/man/get_slf_deaths_lookup_path.Rd b/man/get_slf_deaths_lookup_path.Rd
index 307c38ad3..ae64e2371 100644
--- a/man/get_slf_deaths_lookup_path.Rd
+++ b/man/get_slf_deaths_lookup_path.Rd
@@ -21,7 +21,7 @@ Get the full path to the SLF deaths lookup file
 \code{\link[=get_file_path]{get_file_path()}} for the generic function.
 
 Other slf lookup file path: 
-\code{\link{get_all_slf_deaths_lookup_path}()},
+\code{\link{get_combined_slf_deaths_lookup_path}()},
 \code{\link{get_slf_ch_name_lookup_path}()},
 \code{\link{get_slf_chi_deaths_path}()},
 \code{\link{get_slf_gpprac_path}()},
diff --git a/man/get_slf_gpprac_path.Rd b/man/get_slf_gpprac_path.Rd
index 1fb23116f..1371f758e 100644
--- a/man/get_slf_gpprac_path.Rd
+++ b/man/get_slf_gpprac_path.Rd
@@ -21,7 +21,7 @@ Get the full path to the SLF GP practice lookup
 \code{\link[=get_file_path]{get_file_path()}} for the generic function.
 
 Other slf lookup file path: 
-\code{\link{get_all_slf_deaths_lookup_path}()},
+\code{\link{get_combined_slf_deaths_lookup_path}()},
 \code{\link{get_slf_ch_name_lookup_path}()},
 \code{\link{get_slf_chi_deaths_path}()},
 \code{\link{get_slf_deaths_lookup_path}()},
diff --git a/man/get_slf_postcode_path.Rd b/man/get_slf_postcode_path.Rd
index f37678695..ee9bc65a9 100644
--- a/man/get_slf_postcode_path.Rd
+++ b/man/get_slf_postcode_path.Rd
@@ -21,7 +21,7 @@ Get the full path to the SLF Postcode lookup
 \code{\link[=get_file_path]{get_file_path()}} for the generic function.
 
 Other slf lookup file path: 
-\code{\link{get_all_slf_deaths_lookup_path}()},
+\code{\link{get_combined_slf_deaths_lookup_path}()},
 \code{\link{get_slf_ch_name_lookup_path}()},
 \code{\link{get_slf_chi_deaths_path}()},
 \code{\link{get_slf_deaths_lookup_path}()},

From 33278ef2dfdb4cf8bf6b936ae33e1ab1ffb071dc Mon Sep 17 00:00:00 2001
From: Jennifer Thom <jennifer.thom@phs.scot>
Date: Wed, 26 Jun 2024 11:34:55 +0100
Subject: [PATCH 08/39] automate combined deaths lookup

---
 R/add_activity_after_death_flag.R | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/R/add_activity_after_death_flag.R b/R/add_activity_after_death_flag.R
index b4e0bfd18..5f973eb1e 100644
--- a/R/add_activity_after_death_flag.R
+++ b/R/add_activity_after_death_flag.R
@@ -127,17 +127,16 @@ add_activity_after_death_flag <- function(
 # Read data------------------------------------------------
 process_deaths_lookup <- function(update = latest_update(),
                                   write_to_disk = TRUE, ...) {
-  all_boxi_deaths <- read_file(get_slf_deaths_lookup_path("1415")) %>%
-    rbind(read_file(get_slf_deaths_lookup_path("1516"))) %>%
-    rbind(read_file(get_slf_deaths_lookup_path("1617"))) %>%
-    rbind(read_file(get_slf_deaths_lookup_path("1718"))) %>%
-    rbind(read_file(get_slf_deaths_lookup_path("1819"))) %>%
-    rbind(read_file(get_slf_deaths_lookup_path("1920"))) %>%
-    rbind(read_file(get_slf_deaths_lookup_path("2021"))) %>%
-    rbind(read_file(get_slf_deaths_lookup_path("2122"))) %>%
-    rbind(read_file(get_slf_deaths_lookup_path("2223"))) %>%
-    rbind(read_file(get_slf_deaths_lookup_path("2324"))) %>%
-    # TODO: make this automated to pick up files starting with name "get_slf_deaths_lookup_path"
+  dir_folder <- "/conf/hscdiip/SLF_Extracts/Deaths"
+  file_names <- list.files(dir_folder,
+    pattern = "^anon-slf_deaths_lookup_.*parquet",
+    full.names = TRUE
+  )
+
+  # read all year specific deaths lookups and bind them together
+  all_boxi_deaths <- lapply(file_names, arrow::read_parquet) %>%
+    data.table::rbindlist() %>%
+    # convert to chi for processing
     slfhelper::get_chi() %>%
     # Remove rows with missing or blank CHI number - could also use na.omit?
     # na.omit(all_boxi_deaths)
@@ -185,7 +184,7 @@ process_deaths_lookup <- function(update = latest_update(),
     write_file(
       all_boxi_deaths,
       fs::path(get_slf_dir(), "Deaths",
-        file_name = stringr::str_glue("anon-all_slf_deaths_lookup_{update}.parquet")
+        file_name = stringr::str_glue("anon-combined_slf_deaths_lookup_{update}.parquet")
       )
     )
   }

From 16f5852cf89f1170dac0751e71c359b91e207c3e Mon Sep 17 00:00:00 2001
From: Jennit07 <Jennit07@users.noreply.github.com>
Date: Tue, 2 Jul 2024 12:39:38 +0000
Subject: [PATCH 09/39] Update documentation

---
 DESCRIPTION | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/DESCRIPTION b/DESCRIPTION
index 3d731a0af..a25794864 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -73,5 +73,4 @@ Encoding: UTF-8
 Language: en-GB
 LazyData: true
 Roxygen: list(markdown = TRUE)
-RoxygenNote: 7.3.1
-
+RoxygenNote: 7.3.2

From dbb118aa5e90ecd695df13249f95660645857261 Mon Sep 17 00:00:00 2001
From: Jennifer Thom <jennifer.thom@phs.scot>
Date: Tue, 2 Jul 2024 16:17:15 +0100
Subject: [PATCH 10/39] Update targets script

---
 _targets.R | 95 ++++++++++++++++++++++++++++++------------------------
 1 file changed, 52 insertions(+), 43 deletions(-)

diff --git a/_targets.R b/_targets.R
index 4ea32c179..3c788438e 100644
--- a/_targets.R
+++ b/_targets.R
@@ -19,7 +19,7 @@ tar_option_set(
   memory = "persistent" # default option
 )
 
-years_to_run <- c("1718", "1819", "1920", "2021", "2122", "2223", "2324")
+years_to_run <- c("1718", "1819", "1920") # , "2021", "2122", "2223", "2324")
 
 list(
   tar_rds(write_to_disk, TRUE),
@@ -542,26 +542,6 @@ list(
         write_to_disk = write_to_disk
       )
     ),
-    tar_qs(
-      processed_data_list,
-      list(
-        source_acute_extract,
-        source_ae_extract,
-        source_cmh_extract,
-        source_dn_extract,
-        source_homelessness_extract,
-        source_maternity_extract,
-        source_mental_health_extract,
-        source_nrs_deaths_extract,
-        source_ooh_extract,
-        source_outpatients_extract,
-        source_prescribing_extract,
-        source_sc_care_home,
-        source_sc_home_care,
-        source_sc_sds,
-        source_sc_alarms_tele
-      )
-    ),
     tar_file_read(nsu_cohort, get_nsu_path(year), read_file(!!.x)),
     tar_target(
       homelessness_lookup,
@@ -571,28 +551,57 @@ list(
       )
     ),
     tar_target(
-      episode_file,
-      create_episode_file(
-        processed_data_list,
-        year,
-        homelessness_lookup = homelessness_lookup,
-        dd_data = source_dd_extract %>% slfhelper::get_chi(),
-        nsu_cohort = nsu_cohort %>% slfhelper::get_chi(),
-        ltc_data = source_ltc_lookup %>% slfhelper::get_chi(),
-        slf_pc_lookup = source_pc_lookup,
-        slf_gpprac_lookup = source_gp_lookup,
-        slf_deaths_lookup = slf_deaths_lookup %>% slfhelper::get_chi(),
-        sc_client = sc_client_lookup %>% slfhelper::get_chi(),
-        write_to_disk
-      )
-    ),
-    tar_target(
-      episode_file_tests,
-      process_tests_episode_file(
-        data = episode_file,
-        year = year
-      )
-    ) # ,
+      combined_deaths_lookup,
+      process_combined_deaths_lookup()
+    )
+
+    ## End of Targets pipeline ##
+
+    ################################################################################
+    ## Redundant code which may still be useful for including ep/indiv files.
+    # tar_qs(
+    #   processed_data_list,
+    #   list(
+    #     source_acute_extract,
+    #     source_ae_extract,
+    #     source_cmh_extract,
+    #     source_dn_extract,
+    #     source_homelessness_extract,
+    #     source_maternity_extract,
+    #     source_mental_health_extract,
+    #     source_nrs_deaths_extract,
+    #     source_ooh_extract,
+    #     source_outpatients_extract,
+    #     source_prescribing_extract,
+    #     source_sc_care_home,
+    #     source_sc_home_care,
+    #     source_sc_sds,
+    #     source_sc_alarms_tele
+    #   )
+    # ),
+    # tar_target(
+    #   episode_file,
+    #   create_episode_file(
+    #     processed_data_list,
+    #     year,
+    #     homelessness_lookup = homelessness_lookup,
+    #     dd_data = source_dd_extract %>% slfhelper::get_chi(),
+    #     nsu_cohort = nsu_cohort %>% slfhelper::get_chi(),
+    #     ltc_data = source_ltc_lookup %>% slfhelper::get_chi(),
+    #     slf_pc_lookup = source_pc_lookup,
+    #     slf_gpprac_lookup = source_gp_lookup,
+    #     slf_deaths_lookup = slf_deaths_lookup %>% slfhelper::get_chi(),
+    #     sc_client = sc_client_lookup %>% slfhelper::get_chi(),
+    #     write_to_disk
+    #   )
+    # ),
+    # tar_target(
+    #   episode_file_tests,
+    #   process_tests_episode_file(
+    #     data = episode_file,
+    #     year = year
+    #   )
+    # ) # ,
     # tar_target(
     #   cross_year_tests,
     #   process_tests_cross_year(year = year)

From 623b2224a2e8c1fc321bf79c5a4e579d6e3a059a Mon Sep 17 00:00:00 2001
From: Jennifer Thom <jennifer.thom@phs.scot>
Date: Tue, 2 Jul 2024 16:18:03 +0100
Subject: [PATCH 11/39] Update years

---
 _targets.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_targets.R b/_targets.R
index 3c788438e..7511ff03d 100644
--- a/_targets.R
+++ b/_targets.R
@@ -19,7 +19,7 @@ tar_option_set(
   memory = "persistent" # default option
 )
 
-years_to_run <- c("1718", "1819", "1920") # , "2021", "2122", "2223", "2324")
+years_to_run <- c("1718", "1819", "1920", "2021", "2122", "2223", "2324")
 
 list(
   tar_rds(write_to_disk, TRUE),

From d319c8543bf24440d8aa593147095638a7ced783 Mon Sep 17 00:00:00 2001
From: Jennifer Thom <jennifer.thom@phs.scot>
Date: Tue, 2 Jul 2024 16:41:52 +0100
Subject: [PATCH 12/39] Update running process manually

---
 .../run_episode_file_1718.R                   | 71 ++++++++++++++++++-
 .../run_episode_file_1819.R                   | 71 ++++++++++++++++++-
 .../run_episode_file_1920.R                   | 71 ++++++++++++++++++-
 .../run_episode_file_2021.R                   | 71 ++++++++++++++++++-
 .../run_episode_file_2122.R                   | 71 ++++++++++++++++++-
 .../run_episode_file_2223.R                   | 71 ++++++++++++++++++-
 .../run_episode_file_2324.R                   | 71 ++++++++++++++++++-
 7 files changed, 483 insertions(+), 14 deletions(-)

diff --git a/Run_SLF_Files_manually/run_episode_file_1718.R b/Run_SLF_Files_manually/run_episode_file_1718.R
index ab75b94d7..f679ea669 100644
--- a/Run_SLF_Files_manually/run_episode_file_1718.R
+++ b/Run_SLF_Files_manually/run_episode_file_1718.R
@@ -3,10 +3,77 @@ library(createslf)
 
 year <- "1718"
 
-processed_data_list <- targets::tar_read("processed_data_list_1718",
-  store = fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+targets_store <- fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+
+processed_data_list <- list(
+  acute = targets::tar_read(
+    "source_acute_extract_1718",
+    store = targets_store
+  ),
+  ae = targets::tar_read(
+    "source_ae_extract_1718",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_1718",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_1718",
+    store = targets_store
+  ),
+  dn = targets::tar_read(
+    "source_dn_extract_1718",
+    store = targets_store
+  ),
+  deaths = targets::tar_read(
+    "source_nrs_deaths_extract_1718",
+    store = targets_store
+  ),
+  homelessness = targets::tar_read(
+    "source_homelessness_extract_1718",
+    store = targets_store
+  ),
+  maternity = targets::tar_read(
+    "source_maternity_extract_1718",
+    store = targets_store
+  ),
+  mental_health = targets::tar_read(
+    "source_mental_health_extract_1718",
+    store = targets_store
+  ),
+  outpatients = targets::tar_read(
+    "source_outpatients_extract_1718",
+    store = targets_store
+  ),
+  gp_ooh = targets::tar_read(
+    "source_ooh_extract_1718",
+    store = targets_store
+  ),
+  prescribing = targets::tar_read(
+    "source_prescribing_extract_1718",
+    store = targets_store
+  ),
+  care_home = targets::tar_read(
+    "source_sc_care_home_1718",
+    store = targets_store
+  ),
+  home_care = targets::tar_read(
+    "source_sc_home_care_1718",
+    store = targets_store
+  ),
+  at = targets::tar_read(
+    "source_sc_alarms_tele_1718",
+    store = targets_store
+  ),
+  sds = targets::tar_read(
+    "source_sc_sds_1718",
+    store = targets_store
+  )
 )
 
 # Run episode file
 create_episode_file(processed_data_list, year = year) %>%
   process_tests_episode_file(year = year)
+
+## End of Script ##
diff --git a/Run_SLF_Files_manually/run_episode_file_1819.R b/Run_SLF_Files_manually/run_episode_file_1819.R
index cd5a7435f..d7a65690e 100644
--- a/Run_SLF_Files_manually/run_episode_file_1819.R
+++ b/Run_SLF_Files_manually/run_episode_file_1819.R
@@ -3,10 +3,77 @@ library(createslf)
 
 year <- "1819"
 
-processed_data_list <- targets::tar_read("processed_data_list_1819",
-  store = fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+targets_store <- fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+
+processed_data_list <- list(
+  acute = targets::tar_read(
+    "source_acute_extract_1819",
+    store = targets_store
+  ),
+  ae = targets::tar_read(
+    "source_ae_extract_1819",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_1819",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_1819",
+    store = targets_store
+  ),
+  dn = targets::tar_read(
+    "source_dn_extract_1819",
+    store = targets_store
+  ),
+  deaths = targets::tar_read(
+    "source_nrs_deaths_extract_1819",
+    store = targets_store
+  ),
+  homelessness = targets::tar_read(
+    "source_homelessness_extract_1819",
+    store = targets_store
+  ),
+  maternity = targets::tar_read(
+    "source_maternity_extract_1819",
+    store = targets_store
+  ),
+  mental_health = targets::tar_read(
+    "source_mental_health_extract_1819",
+    store = targets_store
+  ),
+  outpatients = targets::tar_read(
+    "source_outpatients_extract_1819",
+    store = targets_store
+  ),
+  gp_ooh = targets::tar_read(
+    "source_ooh_extract_1819",
+    store = targets_store
+  ),
+  prescribing = targets::tar_read(
+    "source_prescribing_extract_1819",
+    store = targets_store
+  ),
+  care_home = targets::tar_read(
+    "source_sc_care_home_1819",
+    store = targets_store
+  ),
+  home_care = targets::tar_read(
+    "source_sc_home_care_1819",
+    store = targets_store
+  ),
+  at = targets::tar_read(
+    "source_sc_alarms_tele_1819",
+    store = targets_store
+  ),
+  sds = targets::tar_read(
+    "source_sc_sds_1819",
+    store = targets_store
+  )
 )
 
 # Run episode file
 create_episode_file(processed_data_list, year = year) %>%
   process_tests_episode_file(year = year)
+
+## End of Script ##
diff --git a/Run_SLF_Files_manually/run_episode_file_1920.R b/Run_SLF_Files_manually/run_episode_file_1920.R
index a9dc591b1..e3c2ebeb0 100644
--- a/Run_SLF_Files_manually/run_episode_file_1920.R
+++ b/Run_SLF_Files_manually/run_episode_file_1920.R
@@ -3,10 +3,77 @@ library(createslf)
 
 year <- "1920"
 
-processed_data_list <- targets::tar_read("processed_data_list_1920",
-  store = fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+targets_store <- fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+
+processed_data_list <- list(
+  acute = targets::tar_read(
+    "source_acute_extract_1920",
+    store = targets_store
+  ),
+  ae = targets::tar_read(
+    "source_ae_extract_1920",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_1920",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_1920",
+    store = targets_store
+  ),
+  dn = targets::tar_read(
+    "source_dn_extract_1920",
+    store = targets_store
+  ),
+  deaths = targets::tar_read(
+    "source_nrs_deaths_extract_1920",
+    store = targets_store
+  ),
+  homelessness = targets::tar_read(
+    "source_homelessness_extract_1920",
+    store = targets_store
+  ),
+  maternity = targets::tar_read(
+    "source_maternity_extract_1920",
+    store = targets_store
+  ),
+  mental_health = targets::tar_read(
+    "source_mental_health_extract_1920",
+    store = targets_store
+  ),
+  outpatients = targets::tar_read(
+    "source_outpatients_extract_1920",
+    store = targets_store
+  ),
+  gp_ooh = targets::tar_read(
+    "source_ooh_extract_1920",
+    store = targets_store
+  ),
+  prescribing = targets::tar_read(
+    "source_prescribing_extract_1920",
+    store = targets_store
+  ),
+  care_home = targets::tar_read(
+    "source_sc_care_home_1920",
+    store = targets_store
+  ),
+  home_care = targets::tar_read(
+    "source_sc_home_care_1920",
+    store = targets_store
+  ),
+  at = targets::tar_read(
+    "source_sc_alarms_tele_1920",
+    store = targets_store
+  ),
+  sds = targets::tar_read(
+    "source_sc_sds_1920",
+    store = targets_store
+  )
 )
 
 # Run episode file
 create_episode_file(processed_data_list, year = year) %>%
   process_tests_episode_file(year = year)
+
+## End of Script ##
diff --git a/Run_SLF_Files_manually/run_episode_file_2021.R b/Run_SLF_Files_manually/run_episode_file_2021.R
index 37708ee8b..c66f4572d 100644
--- a/Run_SLF_Files_manually/run_episode_file_2021.R
+++ b/Run_SLF_Files_manually/run_episode_file_2021.R
@@ -3,10 +3,77 @@ library(createslf)
 
 year <- "2021"
 
-processed_data_list <- targets::tar_read("processed_data_list_2021",
-  store = fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+targets_store <- fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+
+processed_data_list <- list(
+  acute = targets::tar_read(
+    "source_acute_extract_2021",
+    store = targets_store
+  ),
+  ae = targets::tar_read(
+    "source_ae_extract_2021",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_2021",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_2021",
+    store = targets_store
+  ),
+  dn = targets::tar_read(
+    "source_dn_extract_2021",
+    store = targets_store
+  ),
+  deaths = targets::tar_read(
+    "source_nrs_deaths_extract_2021",
+    store = targets_store
+  ),
+  homelessness = targets::tar_read(
+    "source_homelessness_extract_2021",
+    store = targets_store
+  ),
+  maternity = targets::tar_read(
+    "source_maternity_extract_2021",
+    store = targets_store
+  ),
+  mental_health = targets::tar_read(
+    "source_mental_health_extract_2021",
+    store = targets_store
+  ),
+  outpatients = targets::tar_read(
+    "source_outpatients_extract_2021",
+    store = targets_store
+  ),
+  gp_ooh = targets::tar_read(
+    "source_ooh_extract_2021",
+    store = targets_store
+  ),
+  prescribing = targets::tar_read(
+    "source_prescribing_extract_2021",
+    store = targets_store
+  ),
+  care_home = targets::tar_read(
+    "source_sc_care_home_2021",
+    store = targets_store
+  ),
+  home_care = targets::tar_read(
+    "source_sc_home_care_2021",
+    store = targets_store
+  ),
+  at = targets::tar_read(
+    "source_sc_alarms_tele_2021",
+    store = targets_store
+  ),
+  sds = targets::tar_read(
+    "source_sc_sds_2021",
+    store = targets_store
+  )
 )
 
 # Run episode file
 create_episode_file(processed_data_list, year = year) %>%
   process_tests_episode_file(year = year)
+
+## End of Script ##
diff --git a/Run_SLF_Files_manually/run_episode_file_2122.R b/Run_SLF_Files_manually/run_episode_file_2122.R
index 47400e2d1..cde974be2 100644
--- a/Run_SLF_Files_manually/run_episode_file_2122.R
+++ b/Run_SLF_Files_manually/run_episode_file_2122.R
@@ -3,10 +3,77 @@ library(createslf)
 
 year <- "2122"
 
-processed_data_list <- targets::tar_read("processed_data_list_2122",
-  store = fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+targets_store <- fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+
+processed_data_list <- list(
+  acute = targets::tar_read(
+    "source_acute_extract_2122",
+    store = targets_store
+  ),
+  ae = targets::tar_read(
+    "source_ae_extract_2122",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_2122",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_2122",
+    store = targets_store
+  ),
+  dn = targets::tar_read(
+    "source_dn_extract_2122",
+    store = targets_store
+  ),
+  deaths = targets::tar_read(
+    "source_nrs_deaths_extract_2122",
+    store = targets_store
+  ),
+  homelessness = targets::tar_read(
+    "source_homelessness_extract_2122",
+    store = targets_store
+  ),
+  maternity = targets::tar_read(
+    "source_maternity_extract_2122",
+    store = targets_store
+  ),
+  mental_health = targets::tar_read(
+    "source_mental_health_extract_2122",
+    store = targets_store
+  ),
+  outpatients = targets::tar_read(
+    "source_outpatients_extract_2122",
+    store = targets_store
+  ),
+  gp_ooh = targets::tar_read(
+    "source_ooh_extract_2122",
+    store = targets_store
+  ),
+  prescribing = targets::tar_read(
+    "source_prescribing_extract_2122",
+    store = targets_store
+  ),
+  care_home = targets::tar_read(
+    "source_sc_care_home_2122",
+    store = targets_store
+  ),
+  home_care = targets::tar_read(
+    "source_sc_home_care_2122",
+    store = targets_store
+  ),
+  at = targets::tar_read(
+    "source_sc_alarms_tele_2122",
+    store = targets_store
+  ),
+  sds = targets::tar_read(
+    "source_sc_sds_2122",
+    store = targets_store
+  )
 )
 
 # Run episode file
 create_episode_file(processed_data_list, year = year) %>%
   process_tests_episode_file(year = year)
+
+## End of Script ##
diff --git a/Run_SLF_Files_manually/run_episode_file_2223.R b/Run_SLF_Files_manually/run_episode_file_2223.R
index e64a57f32..ee83082f1 100644
--- a/Run_SLF_Files_manually/run_episode_file_2223.R
+++ b/Run_SLF_Files_manually/run_episode_file_2223.R
@@ -3,10 +3,77 @@ library(createslf)
 
 year <- "2223"
 
-processed_data_list <- targets::tar_read("processed_data_list_2223",
-  store = fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+targets_store <- fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+
+processed_data_list <- list(
+  acute = targets::tar_read(
+    "source_acute_extract_2223",
+    store = targets_store
+  ),
+  ae = targets::tar_read(
+    "source_ae_extract_2223",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_2223",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_2223",
+    store = targets_store
+  ),
+  dn = targets::tar_read(
+    "source_dn_extract_2223",
+    store = targets_store
+  ),
+  deaths = targets::tar_read(
+    "source_nrs_deaths_extract_2223",
+    store = targets_store
+  ),
+  homelessness = targets::tar_read(
+    "source_homelessness_extract_2223",
+    store = targets_store
+  ),
+  maternity = targets::tar_read(
+    "source_maternity_extract_2223",
+    store = targets_store
+  ),
+  mental_health = targets::tar_read(
+    "source_mental_health_extract_2223",
+    store = targets_store
+  ),
+  outpatients = targets::tar_read(
+    "source_outpatients_extract_2223",
+    store = targets_store
+  ),
+  gp_ooh = targets::tar_read(
+    "source_ooh_extract_2223",
+    store = targets_store
+  ),
+  prescribing = targets::tar_read(
+    "source_prescribing_extract_2223",
+    store = targets_store
+  ),
+  care_home = targets::tar_read(
+    "source_sc_care_home_2223",
+    store = targets_store
+  ),
+  home_care = targets::tar_read(
+    "source_sc_home_care_2223",
+    store = targets_store
+  ),
+  at = targets::tar_read(
+    "source_sc_alarms_tele_2223",
+    store = targets_store
+  ),
+  sds = targets::tar_read(
+    "source_sc_sds_2223",
+    store = targets_store
+  )
 )
 
 # Run episode file
 create_episode_file(processed_data_list, year = year) %>%
   process_tests_episode_file(year = year)
+
+## End of Script ##
diff --git a/Run_SLF_Files_manually/run_episode_file_2324.R b/Run_SLF_Files_manually/run_episode_file_2324.R
index 4a7f0ad29..508689f6d 100644
--- a/Run_SLF_Files_manually/run_episode_file_2324.R
+++ b/Run_SLF_Files_manually/run_episode_file_2324.R
@@ -3,10 +3,77 @@ library(createslf)
 
 year <- "2324"
 
-processed_data_list <- targets::tar_read("processed_data_list_2324",
-  store = fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+targets_store <- fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+
+processed_data_list <- list(
+  acute = targets::tar_read(
+    "source_acute_extract_2324",
+    store = targets_store
+  ),
+  ae = targets::tar_read(
+    "source_ae_extract_2324",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_2324",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_2324",
+    store = targets_store
+  ),
+  dn = targets::tar_read(
+    "source_dn_extract_2324",
+    store = targets_store
+  ),
+  deaths = targets::tar_read(
+    "source_nrs_deaths_extract_2324",
+    store = targets_store
+  ),
+  homelessness = targets::tar_read(
+    "source_homelessness_extract_2324",
+    store = targets_store
+  ),
+  maternity = targets::tar_read(
+    "source_maternity_extract_2324",
+    store = targets_store
+  ),
+  mental_health = targets::tar_read(
+    "source_mental_health_extract_2324",
+    store = targets_store
+  ),
+  outpatients = targets::tar_read(
+    "source_outpatients_extract_2324",
+    store = targets_store
+  ),
+  gp_ooh = targets::tar_read(
+    "source_ooh_extract_2324",
+    store = targets_store
+  ),
+  prescribing = targets::tar_read(
+    "source_prescribing_extract_2324",
+    store = targets_store
+  ),
+  care_home = targets::tar_read(
+    "source_sc_care_home_2324",
+    store = targets_store
+  ),
+  home_care = targets::tar_read(
+    "source_sc_home_care_2324",
+    store = targets_store
+  ),
+  at = targets::tar_read(
+    "source_sc_alarms_tele_2324",
+    store = targets_store
+  ),
+  sds = targets::tar_read(
+    "source_sc_sds_2324",
+    store = targets_store
+  )
 )
 
 # Run episode file
 create_episode_file(processed_data_list, year = year) %>%
   process_tests_episode_file(year = year)
+
+## End of Script ##

From ca2d033dd156df0832d257202b31a82b102a62ac Mon Sep 17 00:00:00 2001
From: Jennifer Thom <jennifer.thom@phs.scot>
Date: Wed, 3 Jul 2024 09:32:23 +0100
Subject: [PATCH 13/39] re-arrange brackets

---
 _targets.R | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/_targets.R b/_targets.R
index 7511ff03d..4583f21fb 100644
--- a/_targets.R
+++ b/_targets.R
@@ -554,7 +554,8 @@ list(
       combined_deaths_lookup,
       process_combined_deaths_lookup()
     )
-
+  )
+)
     ## End of Targets pipeline ##
 
     ################################################################################
@@ -652,5 +653,4 @@ list(
     #     version = "latest"
     #   )
     # )
-  )
-)
+

From f15ca4e6c3f948ce610ac919a768b0518fefc55f Mon Sep 17 00:00:00 2001
From: Jennifer Thom <jennifer.thom@phs.scot>
Date: Wed, 3 Jul 2024 09:33:50 +0100
Subject: [PATCH 14/39] Style code

---
 _targets.R | 201 ++++++++++++++++++++++++++---------------------------
 1 file changed, 100 insertions(+), 101 deletions(-)

diff --git a/_targets.R b/_targets.R
index 4583f21fb..80c56701e 100644
--- a/_targets.R
+++ b/_targets.R
@@ -549,108 +549,107 @@ list(
         year,
         homelessness_data = source_homelessness_extract %>% slfhelper::get_chi()
       )
-    ),
-    tar_target(
-      combined_deaths_lookup,
-      process_combined_deaths_lookup()
     )
+  ),
+  tar_target(
+    combined_deaths_lookup,
+    process_combined_deaths_lookup()
   )
 )
-    ## End of Targets pipeline ##
-
-    ################################################################################
-    ## Redundant code which may still be useful for including ep/indiv files.
-    # tar_qs(
-    #   processed_data_list,
-    #   list(
-    #     source_acute_extract,
-    #     source_ae_extract,
-    #     source_cmh_extract,
-    #     source_dn_extract,
-    #     source_homelessness_extract,
-    #     source_maternity_extract,
-    #     source_mental_health_extract,
-    #     source_nrs_deaths_extract,
-    #     source_ooh_extract,
-    #     source_outpatients_extract,
-    #     source_prescribing_extract,
-    #     source_sc_care_home,
-    #     source_sc_home_care,
-    #     source_sc_sds,
-    #     source_sc_alarms_tele
-    #   )
-    # ),
-    # tar_target(
-    #   episode_file,
-    #   create_episode_file(
-    #     processed_data_list,
-    #     year,
-    #     homelessness_lookup = homelessness_lookup,
-    #     dd_data = source_dd_extract %>% slfhelper::get_chi(),
-    #     nsu_cohort = nsu_cohort %>% slfhelper::get_chi(),
-    #     ltc_data = source_ltc_lookup %>% slfhelper::get_chi(),
-    #     slf_pc_lookup = source_pc_lookup,
-    #     slf_gpprac_lookup = source_gp_lookup,
-    #     slf_deaths_lookup = slf_deaths_lookup %>% slfhelper::get_chi(),
-    #     sc_client = sc_client_lookup %>% slfhelper::get_chi(),
-    #     write_to_disk
-    #   )
-    # ),
-    # tar_target(
-    #   episode_file_tests,
-    #   process_tests_episode_file(
-    #     data = episode_file,
-    #     year = year
-    #   )
-    # ) # ,
-    # tar_target(
-    #   cross_year_tests,
-    #   process_tests_cross_year(year = year)
-    # ), # ,
-    # tar_target(
-    #   individual_file,
-    #   create_individual_file(
-    #     episode_file = episode_file,
-    #     year = year,
-    #     homelessness_lookup = homelessness_lookup,
-    #     write_to_disk = write_to_disk
-    #   )
-    # ),
-    # tar_target(
-    #   individual_file_tests,
-    #   process_tests_individual_file(
-    #     data = individual_file,
-    #     year = year
-    #   )
-    # ) # ,
-    # tar_target(
-    #   episode_file_dataset,
-    #   arrow::write_dataset(
-    #     dataset = episode_file,
-    #     path = fs::path(
-    #       get_year_dir(year),
-    #       stringr::str_glue("source-episode-file-{year}")
-    #     ),
-    #     format = "parquet",
-    #     # Should correspond to the available slfhelper filters
-    #     partitioning = c("recid", "hscp2018"),
-    #     compression = "zstd",
-    #     version = "latest"
-    #   )
-    # ),
-    # tar_target(
-    #   individual_file_dataset,
-    #   arrow::write_dataset(
-    #     dataset = individual_file,
-    #     path = fs::path(
-    #       get_year_dir(year),
-    #       stringr::str_glue("source-individual-file-{year}")
-    #     ),
-    #     format = "parquet",
-    #     # Should correspond to the available slfhelper filters
-    #     partitioning = c("hscp2018"),
-    #     compression = "zstd",
-    #     version = "latest"
-    #   )
-    # )
+## End of Targets pipeline ##
 
+################################################################################
+## Redundant code which may still be useful for including ep/indiv files.
+# tar_qs(
+#   processed_data_list,
+#   list(
+#     source_acute_extract,
+#     source_ae_extract,
+#     source_cmh_extract,
+#     source_dn_extract,
+#     source_homelessness_extract,
+#     source_maternity_extract,
+#     source_mental_health_extract,
+#     source_nrs_deaths_extract,
+#     source_ooh_extract,
+#     source_outpatients_extract,
+#     source_prescribing_extract,
+#     source_sc_care_home,
+#     source_sc_home_care,
+#     source_sc_sds,
+#     source_sc_alarms_tele
+#   )
+# ),
+# tar_target(
+#   episode_file,
+#   create_episode_file(
+#     processed_data_list,
+#     year,
+#     homelessness_lookup = homelessness_lookup,
+#     dd_data = source_dd_extract %>% slfhelper::get_chi(),
+#     nsu_cohort = nsu_cohort %>% slfhelper::get_chi(),
+#     ltc_data = source_ltc_lookup %>% slfhelper::get_chi(),
+#     slf_pc_lookup = source_pc_lookup,
+#     slf_gpprac_lookup = source_gp_lookup,
+#     slf_deaths_lookup = slf_deaths_lookup %>% slfhelper::get_chi(),
+#     sc_client = sc_client_lookup %>% slfhelper::get_chi(),
+#     write_to_disk
+#   )
+# ),
+# tar_target(
+#   episode_file_tests,
+#   process_tests_episode_file(
+#     data = episode_file,
+#     year = year
+#   )
+# ) # ,
+# tar_target(
+#   cross_year_tests,
+#   process_tests_cross_year(year = year)
+# ), # ,
+# tar_target(
+#   individual_file,
+#   create_individual_file(
+#     episode_file = episode_file,
+#     year = year,
+#     homelessness_lookup = homelessness_lookup,
+#     write_to_disk = write_to_disk
+#   )
+# ),
+# tar_target(
+#   individual_file_tests,
+#   process_tests_individual_file(
+#     data = individual_file,
+#     year = year
+#   )
+# ) # ,
+# tar_target(
+#   episode_file_dataset,
+#   arrow::write_dataset(
+#     dataset = episode_file,
+#     path = fs::path(
+#       get_year_dir(year),
+#       stringr::str_glue("source-episode-file-{year}")
+#     ),
+#     format = "parquet",
+#     # Should correspond to the available slfhelper filters
+#     partitioning = c("recid", "hscp2018"),
+#     compression = "zstd",
+#     version = "latest"
+#   )
+# ),
+# tar_target(
+#   individual_file_dataset,
+#   arrow::write_dataset(
+#     dataset = individual_file,
+#     path = fs::path(
+#       get_year_dir(year),
+#       stringr::str_glue("source-individual-file-{year}")
+#     ),
+#     format = "parquet",
+#     # Should correspond to the available slfhelper filters
+#     partitioning = c("hscp2018"),
+#     compression = "zstd",
+#     version = "latest"
+#   )
+# )

From 61baf029f359b41529972b2755a44877d1b2e077 Mon Sep 17 00:00:00 2001
From: Jennifer Thom <jennifer.thom@phs.scot>
Date: Wed, 3 Jul 2024 10:27:54 +0100
Subject: [PATCH 15/39] Update run targets scripts

---
 Run_SLF_Files_targets/run_targets_1718.R | 8 --------
 Run_SLF_Files_targets/run_targets_1819.R | 8 --------
 Run_SLF_Files_targets/run_targets_1920.R | 8 --------
 Run_SLF_Files_targets/run_targets_2021.R | 8 --------
 Run_SLF_Files_targets/run_targets_2122.R | 8 --------
 Run_SLF_Files_targets/run_targets_2223.R | 8 --------
 Run_SLF_Files_targets/run_targets_2324.R | 8 --------
 7 files changed, 56 deletions(-)

diff --git a/Run_SLF_Files_targets/run_targets_1718.R b/Run_SLF_Files_targets/run_targets_1718.R
index ac03edd3f..a0dc17cf0 100644
--- a/Run_SLF_Files_targets/run_targets_1718.R
+++ b/Run_SLF_Files_targets/run_targets_1718.R
@@ -8,11 +8,3 @@ tar_make_future(
   names = (targets::contains("1718"))
 )
 
-# use targets to create individual files due to RAM limit
-library(createslf)
-
-episode_file <- arrow::read_parquet(get_slf_episode_path(year))
-
-# Run individual file
-create_individual_file(episode_file, year = year) %>%
-  process_tests_individual_file(year = year)
diff --git a/Run_SLF_Files_targets/run_targets_1819.R b/Run_SLF_Files_targets/run_targets_1819.R
index b60728359..981b6ee33 100644
--- a/Run_SLF_Files_targets/run_targets_1819.R
+++ b/Run_SLF_Files_targets/run_targets_1819.R
@@ -8,11 +8,3 @@ tar_make_future(
   names = (targets::contains("1819"))
 )
 
-# use targets to create individual files due to RAM limit
-library(createslf)
-
-episode_file <- arrow::read_parquet(get_slf_episode_path(year))
-
-# Run individual file
-create_individual_file(episode_file, year = year) %>%
-  process_tests_individual_file(year = year)
diff --git a/Run_SLF_Files_targets/run_targets_1920.R b/Run_SLF_Files_targets/run_targets_1920.R
index 897ee0b7a..79ee50644 100644
--- a/Run_SLF_Files_targets/run_targets_1920.R
+++ b/Run_SLF_Files_targets/run_targets_1920.R
@@ -8,11 +8,3 @@ tar_make_future(
   names = (targets::contains("1920"))
 )
 
-# use targets to create individual files due to RAM limit
-library(createslf)
-
-episode_file <- arrow::read_parquet(get_slf_episode_path(year))
-
-# Run individual file
-create_individual_file(episode_file, year = year) %>%
-  process_tests_individual_file(year = year)
diff --git a/Run_SLF_Files_targets/run_targets_2021.R b/Run_SLF_Files_targets/run_targets_2021.R
index 53333c014..c0433117d 100644
--- a/Run_SLF_Files_targets/run_targets_2021.R
+++ b/Run_SLF_Files_targets/run_targets_2021.R
@@ -8,11 +8,3 @@ tar_make_future(
   names = (targets::contains("2021"))
 )
 
-# use targets to create individual files due to RAM limit
-library(createslf)
-
-episode_file <- arrow::read_parquet(get_slf_episode_path(year))
-
-# Run individual file
-create_individual_file(episode_file, year = year) %>%
-  process_tests_individual_file(year = year)
diff --git a/Run_SLF_Files_targets/run_targets_2122.R b/Run_SLF_Files_targets/run_targets_2122.R
index 457fe33e7..3424daf08 100644
--- a/Run_SLF_Files_targets/run_targets_2122.R
+++ b/Run_SLF_Files_targets/run_targets_2122.R
@@ -8,11 +8,3 @@ tar_make_future(
   names = (targets::contains("2122"))
 )
 
-# use targets to create individual files due to RAM limit
-library(createslf)
-
-episode_file <- arrow::read_parquet(get_slf_episode_path(year))
-
-# Run individual file
-create_individual_file(episode_file, year = year) %>%
-  process_tests_individual_file(year = year)
diff --git a/Run_SLF_Files_targets/run_targets_2223.R b/Run_SLF_Files_targets/run_targets_2223.R
index fc851f3f7..052daed3e 100644
--- a/Run_SLF_Files_targets/run_targets_2223.R
+++ b/Run_SLF_Files_targets/run_targets_2223.R
@@ -8,11 +8,3 @@ tar_make_future(
   names = (targets::contains("2223"))
 )
 
-# use targets to create individual files due to RAM limit
-library(createslf)
-
-episode_file <- arrow::read_parquet(get_slf_episode_path(year))
-
-# Run individual file
-create_individual_file(episode_file, year = year) %>%
-  process_tests_individual_file(year = year)
diff --git a/Run_SLF_Files_targets/run_targets_2324.R b/Run_SLF_Files_targets/run_targets_2324.R
index 3b4c9b240..c34ae7061 100644
--- a/Run_SLF_Files_targets/run_targets_2324.R
+++ b/Run_SLF_Files_targets/run_targets_2324.R
@@ -8,11 +8,3 @@ tar_make_future(
   names = (targets::contains("2324"))
 )
 
-# use targets to create individual files due to RAM limit
-library(createslf)
-
-episode_file <- arrow::read_parquet(get_slf_episode_path(year))
-
-# Run individual file
-create_individual_file(episode_file, year = year) %>%
-  process_tests_individual_file(year = year)

From 379b13752660909c0daeb8e94840648486d3d5e9 Mon Sep 17 00:00:00 2001
From: Jennifer Thom <jennifer.thom@phs.scot>
Date: Wed, 3 Jul 2024 10:44:03 +0100
Subject: [PATCH 16/39] Set up all-targets script

---
 Run_SLF_Files_targets/run_all_targets.R | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 Run_SLF_Files_targets/run_all_targets.R

diff --git a/Run_SLF_Files_targets/run_all_targets.R b/Run_SLF_Files_targets/run_all_targets.R
new file mode 100644
index 000000000..fb5b94fab
--- /dev/null
+++ b/Run_SLF_Files_targets/run_all_targets.R
@@ -0,0 +1,5 @@
+library(targets)
+
+# use tar_make_future() to run targets for all years
+# This will run everything needed for creating the episode file.
+tar_make_future()

From 3fb27ae732ff72a2438efca850527f9d57359094 Mon Sep 17 00:00:00 2001
From: Jennit07 <Jennit07@users.noreply.github.com>
Date: Wed, 3 Jul 2024 09:45:29 +0000
Subject: [PATCH 17/39] Style code

---
 Run_SLF_Files_targets/run_targets_1718.R | 1 -
 Run_SLF_Files_targets/run_targets_1819.R | 1 -
 Run_SLF_Files_targets/run_targets_1920.R | 1 -
 Run_SLF_Files_targets/run_targets_2021.R | 1 -
 Run_SLF_Files_targets/run_targets_2122.R | 1 -
 Run_SLF_Files_targets/run_targets_2223.R | 1 -
 Run_SLF_Files_targets/run_targets_2324.R | 1 -
 7 files changed, 7 deletions(-)

diff --git a/Run_SLF_Files_targets/run_targets_1718.R b/Run_SLF_Files_targets/run_targets_1718.R
index a0dc17cf0..e85a89ff8 100644
--- a/Run_SLF_Files_targets/run_targets_1718.R
+++ b/Run_SLF_Files_targets/run_targets_1718.R
@@ -7,4 +7,3 @@ tar_make_future(
   # it does not recognise `contains(year)`
   names = (targets::contains("1718"))
 )
-
diff --git a/Run_SLF_Files_targets/run_targets_1819.R b/Run_SLF_Files_targets/run_targets_1819.R
index 981b6ee33..6957054c8 100644
--- a/Run_SLF_Files_targets/run_targets_1819.R
+++ b/Run_SLF_Files_targets/run_targets_1819.R
@@ -7,4 +7,3 @@ tar_make_future(
   # it does not recognise `contains(year)`
   names = (targets::contains("1819"))
 )
-
diff --git a/Run_SLF_Files_targets/run_targets_1920.R b/Run_SLF_Files_targets/run_targets_1920.R
index 79ee50644..ef6272fca 100644
--- a/Run_SLF_Files_targets/run_targets_1920.R
+++ b/Run_SLF_Files_targets/run_targets_1920.R
@@ -7,4 +7,3 @@ tar_make_future(
   # it does not recognise `contains(year)`
   names = (targets::contains("1920"))
 )
-
diff --git a/Run_SLF_Files_targets/run_targets_2021.R b/Run_SLF_Files_targets/run_targets_2021.R
index c0433117d..237078e05 100644
--- a/Run_SLF_Files_targets/run_targets_2021.R
+++ b/Run_SLF_Files_targets/run_targets_2021.R
@@ -7,4 +7,3 @@ tar_make_future(
   # it does not recognise `contains(year)`
   names = (targets::contains("2021"))
 )
-
diff --git a/Run_SLF_Files_targets/run_targets_2122.R b/Run_SLF_Files_targets/run_targets_2122.R
index 3424daf08..dc64b625f 100644
--- a/Run_SLF_Files_targets/run_targets_2122.R
+++ b/Run_SLF_Files_targets/run_targets_2122.R
@@ -7,4 +7,3 @@ tar_make_future(
   # it does not recognise `contains(year)`
   names = (targets::contains("2122"))
 )
-
diff --git a/Run_SLF_Files_targets/run_targets_2223.R b/Run_SLF_Files_targets/run_targets_2223.R
index 052daed3e..7d8677a80 100644
--- a/Run_SLF_Files_targets/run_targets_2223.R
+++ b/Run_SLF_Files_targets/run_targets_2223.R
@@ -7,4 +7,3 @@ tar_make_future(
   # it does not recognise `contains(year)`
   names = (targets::contains("2223"))
 )
-
diff --git a/Run_SLF_Files_targets/run_targets_2324.R b/Run_SLF_Files_targets/run_targets_2324.R
index c34ae7061..bcebe5fb8 100644
--- a/Run_SLF_Files_targets/run_targets_2324.R
+++ b/Run_SLF_Files_targets/run_targets_2324.R
@@ -7,4 +7,3 @@ tar_make_future(
   # it does not recognise `contains(year)`
   names = (targets::contains("2324"))
 )
-

From ceb526a1260972da21c4103929e4ff0b4772d74c Mon Sep 17 00:00:00 2001
From: Jennit07 <Jennit07@users.noreply.github.com>
Date: Wed, 3 Jul 2024 10:19:45 +0000
Subject: [PATCH 18/39] Update documentation

---
 DESCRIPTION | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/DESCRIPTION b/DESCRIPTION
index 3d731a0af..a25794864 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -73,5 +73,4 @@ Encoding: UTF-8
 Language: en-GB
 LazyData: true
 Roxygen: list(markdown = TRUE)
-RoxygenNote: 7.3.1
-
+RoxygenNote: 7.3.2

From a521206271345728e155d66b00789698ce106261 Mon Sep 17 00:00:00 2001
From: Jennifer Thom <jennifer.thom@phs.scot>
Date: Tue, 2 Jul 2024 16:17:15 +0100
Subject: [PATCH 19/39] Update targets script

---
 _targets.R | 95 ++++++++++++++++++++++++++++++------------------------
 1 file changed, 52 insertions(+), 43 deletions(-)

diff --git a/_targets.R b/_targets.R
index 4ea32c179..3c788438e 100644
--- a/_targets.R
+++ b/_targets.R
@@ -19,7 +19,7 @@ tar_option_set(
   memory = "persistent" # default option
 )
 
-years_to_run <- c("1718", "1819", "1920", "2021", "2122", "2223", "2324")
+years_to_run <- c("1718", "1819", "1920") # , "2021", "2122", "2223", "2324")
 
 list(
   tar_rds(write_to_disk, TRUE),
@@ -542,26 +542,6 @@ list(
         write_to_disk = write_to_disk
       )
     ),
-    tar_qs(
-      processed_data_list,
-      list(
-        source_acute_extract,
-        source_ae_extract,
-        source_cmh_extract,
-        source_dn_extract,
-        source_homelessness_extract,
-        source_maternity_extract,
-        source_mental_health_extract,
-        source_nrs_deaths_extract,
-        source_ooh_extract,
-        source_outpatients_extract,
-        source_prescribing_extract,
-        source_sc_care_home,
-        source_sc_home_care,
-        source_sc_sds,
-        source_sc_alarms_tele
-      )
-    ),
     tar_file_read(nsu_cohort, get_nsu_path(year), read_file(!!.x)),
     tar_target(
       homelessness_lookup,
@@ -571,28 +551,57 @@ list(
       )
     ),
     tar_target(
-      episode_file,
-      create_episode_file(
-        processed_data_list,
-        year,
-        homelessness_lookup = homelessness_lookup,
-        dd_data = source_dd_extract %>% slfhelper::get_chi(),
-        nsu_cohort = nsu_cohort %>% slfhelper::get_chi(),
-        ltc_data = source_ltc_lookup %>% slfhelper::get_chi(),
-        slf_pc_lookup = source_pc_lookup,
-        slf_gpprac_lookup = source_gp_lookup,
-        slf_deaths_lookup = slf_deaths_lookup %>% slfhelper::get_chi(),
-        sc_client = sc_client_lookup %>% slfhelper::get_chi(),
-        write_to_disk
-      )
-    ),
-    tar_target(
-      episode_file_tests,
-      process_tests_episode_file(
-        data = episode_file,
-        year = year
-      )
-    ) # ,
+      combined_deaths_lookup,
+      process_combined_deaths_lookup()
+    )
+
+    ## End of Targets pipeline ##
+
+    ################################################################################
+    ## Redundant code which may still be useful for including ep/indiv files.
+    # tar_qs(
+    #   processed_data_list,
+    #   list(
+    #     source_acute_extract,
+    #     source_ae_extract,
+    #     source_cmh_extract,
+    #     source_dn_extract,
+    #     source_homelessness_extract,
+    #     source_maternity_extract,
+    #     source_mental_health_extract,
+    #     source_nrs_deaths_extract,
+    #     source_ooh_extract,
+    #     source_outpatients_extract,
+    #     source_prescribing_extract,
+    #     source_sc_care_home,
+    #     source_sc_home_care,
+    #     source_sc_sds,
+    #     source_sc_alarms_tele
+    #   )
+    # ),
+    # tar_target(
+    #   episode_file,
+    #   create_episode_file(
+    #     processed_data_list,
+    #     year,
+    #     homelessness_lookup = homelessness_lookup,
+    #     dd_data = source_dd_extract %>% slfhelper::get_chi(),
+    #     nsu_cohort = nsu_cohort %>% slfhelper::get_chi(),
+    #     ltc_data = source_ltc_lookup %>% slfhelper::get_chi(),
+    #     slf_pc_lookup = source_pc_lookup,
+    #     slf_gpprac_lookup = source_gp_lookup,
+    #     slf_deaths_lookup = slf_deaths_lookup %>% slfhelper::get_chi(),
+    #     sc_client = sc_client_lookup %>% slfhelper::get_chi(),
+    #     write_to_disk
+    #   )
+    # ),
+    # tar_target(
+    #   episode_file_tests,
+    #   process_tests_episode_file(
+    #     data = episode_file,
+    #     year = year
+    #   )
+    # ) # ,
     # tar_target(
     #   cross_year_tests,
     #   process_tests_cross_year(year = year)

From 585db7f28da533e35657f93117d628aa361152a2 Mon Sep 17 00:00:00 2001
From: Jennifer Thom <jennifer.thom@phs.scot>
Date: Tue, 2 Jul 2024 16:18:03 +0100
Subject: [PATCH 20/39] Update years

---
 _targets.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_targets.R b/_targets.R
index 3c788438e..7511ff03d 100644
--- a/_targets.R
+++ b/_targets.R
@@ -19,7 +19,7 @@ tar_option_set(
   memory = "persistent" # default option
 )
 
-years_to_run <- c("1718", "1819", "1920") # , "2021", "2122", "2223", "2324")
+years_to_run <- c("1718", "1819", "1920", "2021", "2122", "2223", "2324")
 
 list(
   tar_rds(write_to_disk, TRUE),

From dada9cb5676d115af9ba8b6896349ffe693836eb Mon Sep 17 00:00:00 2001
From: Jennifer Thom <jennifer.thom@phs.scot>
Date: Tue, 2 Jul 2024 16:41:52 +0100
Subject: [PATCH 21/39] Update running process manually

---
 .../run_episode_file_1718.R                   | 71 ++++++++++++++++++-
 .../run_episode_file_1819.R                   | 71 ++++++++++++++++++-
 .../run_episode_file_1920.R                   | 71 ++++++++++++++++++-
 .../run_episode_file_2021.R                   | 71 ++++++++++++++++++-
 .../run_episode_file_2122.R                   | 71 ++++++++++++++++++-
 .../run_episode_file_2223.R                   | 71 ++++++++++++++++++-
 .../run_episode_file_2324.R                   | 71 ++++++++++++++++++-
 7 files changed, 483 insertions(+), 14 deletions(-)

diff --git a/Run_SLF_Files_manually/run_episode_file_1718.R b/Run_SLF_Files_manually/run_episode_file_1718.R
index ab75b94d7..f679ea669 100644
--- a/Run_SLF_Files_manually/run_episode_file_1718.R
+++ b/Run_SLF_Files_manually/run_episode_file_1718.R
@@ -3,10 +3,77 @@ library(createslf)
 
 year <- "1718"
 
-processed_data_list <- targets::tar_read("processed_data_list_1718",
-  store = fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+targets_store <- fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+
+processed_data_list <- list(
+  acute = targets::tar_read(
+    "source_acute_extract_1718",
+    store = targets_store
+  ),
+  ae = targets::tar_read(
+    "source_ae_extract_1718",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_1718",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_1718",
+    store = targets_store
+  ),
+  dn = targets::tar_read(
+    "source_dn_extract_1718",
+    store = targets_store
+  ),
+  deaths = targets::tar_read(
+    "source_nrs_deaths_extract_1718",
+    store = targets_store
+  ),
+  homelessness = targets::tar_read(
+    "source_homelessness_extract_1718",
+    store = targets_store
+  ),
+  maternity = targets::tar_read(
+    "source_maternity_extract_1718",
+    store = targets_store
+  ),
+  mental_health = targets::tar_read(
+    "source_mental_health_extract_1718",
+    store = targets_store
+  ),
+  outpatients = targets::tar_read(
+    "source_outpatients_extract_1718",
+    store = targets_store
+  ),
+  gp_ooh = targets::tar_read(
+    "source_ooh_extract_1718",
+    store = targets_store
+  ),
+  prescribing = targets::tar_read(
+    "source_prescribing_extract_1718",
+    store = targets_store
+  ),
+  care_home = targets::tar_read(
+    "source_sc_care_home_1718",
+    store = targets_store
+  ),
+  home_care = targets::tar_read(
+    "source_sc_home_care_1718",
+    store = targets_store
+  ),
+  at = targets::tar_read(
+    "source_sc_alarms_tele_1718",
+    store = targets_store
+  ),
+  sds = targets::tar_read(
+    "source_sc_sds_1718",
+    store = targets_store
+  )
 )
 
 # Run episode file
 create_episode_file(processed_data_list, year = year) %>%
   process_tests_episode_file(year = year)
+
+## End of Script ##
diff --git a/Run_SLF_Files_manually/run_episode_file_1819.R b/Run_SLF_Files_manually/run_episode_file_1819.R
index cd5a7435f..d7a65690e 100644
--- a/Run_SLF_Files_manually/run_episode_file_1819.R
+++ b/Run_SLF_Files_manually/run_episode_file_1819.R
@@ -3,10 +3,77 @@ library(createslf)
 
 year <- "1819"
 
-processed_data_list <- targets::tar_read("processed_data_list_1819",
-  store = fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+targets_store <- fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+
+processed_data_list <- list(
+  acute = targets::tar_read(
+    "source_acute_extract_1819",
+    store = targets_store
+  ),
+  ae = targets::tar_read(
+    "source_ae_extract_1819",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_1819",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_1819",
+    store = targets_store
+  ),
+  dn = targets::tar_read(
+    "source_dn_extract_1819",
+    store = targets_store
+  ),
+  deaths = targets::tar_read(
+    "source_nrs_deaths_extract_1819",
+    store = targets_store
+  ),
+  homelessness = targets::tar_read(
+    "source_homelessness_extract_1819",
+    store = targets_store
+  ),
+  maternity = targets::tar_read(
+    "source_maternity_extract_1819",
+    store = targets_store
+  ),
+  mental_health = targets::tar_read(
+    "source_mental_health_extract_1819",
+    store = targets_store
+  ),
+  outpatients = targets::tar_read(
+    "source_outpatients_extract_1819",
+    store = targets_store
+  ),
+  gp_ooh = targets::tar_read(
+    "source_ooh_extract_1819",
+    store = targets_store
+  ),
+  prescribing = targets::tar_read(
+    "source_prescribing_extract_1819",
+    store = targets_store
+  ),
+  care_home = targets::tar_read(
+    "source_sc_care_home_1819",
+    store = targets_store
+  ),
+  home_care = targets::tar_read(
+    "source_sc_home_care_1819",
+    store = targets_store
+  ),
+  at = targets::tar_read(
+    "source_sc_alarms_tele_1819",
+    store = targets_store
+  ),
+  sds = targets::tar_read(
+    "source_sc_sds_1819",
+    store = targets_store
+  )
 )
 
 # Run episode file
 create_episode_file(processed_data_list, year = year) %>%
   process_tests_episode_file(year = year)
+
+## End of Script ##
diff --git a/Run_SLF_Files_manually/run_episode_file_1920.R b/Run_SLF_Files_manually/run_episode_file_1920.R
index a9dc591b1..e3c2ebeb0 100644
--- a/Run_SLF_Files_manually/run_episode_file_1920.R
+++ b/Run_SLF_Files_manually/run_episode_file_1920.R
@@ -3,10 +3,77 @@ library(createslf)
 
 year <- "1920"
 
-processed_data_list <- targets::tar_read("processed_data_list_1920",
-  store = fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+targets_store <- fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+
+processed_data_list <- list(
+  acute = targets::tar_read(
+    "source_acute_extract_1920",
+    store = targets_store
+  ),
+  ae = targets::tar_read(
+    "source_ae_extract_1920",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_1920",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_1920",
+    store = targets_store
+  ),
+  dn = targets::tar_read(
+    "source_dn_extract_1920",
+    store = targets_store
+  ),
+  deaths = targets::tar_read(
+    "source_nrs_deaths_extract_1920",
+    store = targets_store
+  ),
+  homelessness = targets::tar_read(
+    "source_homelessness_extract_1920",
+    store = targets_store
+  ),
+  maternity = targets::tar_read(
+    "source_maternity_extract_1920",
+    store = targets_store
+  ),
+  mental_health = targets::tar_read(
+    "source_mental_health_extract_1920",
+    store = targets_store
+  ),
+  outpatients = targets::tar_read(
+    "source_outpatients_extract_1920",
+    store = targets_store
+  ),
+  gp_ooh = targets::tar_read(
+    "source_ooh_extract_1920",
+    store = targets_store
+  ),
+  prescribing = targets::tar_read(
+    "source_prescribing_extract_1920",
+    store = targets_store
+  ),
+  care_home = targets::tar_read(
+    "source_sc_care_home_1920",
+    store = targets_store
+  ),
+  home_care = targets::tar_read(
+    "source_sc_home_care_1920",
+    store = targets_store
+  ),
+  at = targets::tar_read(
+    "source_sc_alarms_tele_1920",
+    store = targets_store
+  ),
+  sds = targets::tar_read(
+    "source_sc_sds_1920",
+    store = targets_store
+  )
 )
 
 # Run episode file
 create_episode_file(processed_data_list, year = year) %>%
   process_tests_episode_file(year = year)
+
+## End of Script ##
diff --git a/Run_SLF_Files_manually/run_episode_file_2021.R b/Run_SLF_Files_manually/run_episode_file_2021.R
index 37708ee8b..c66f4572d 100644
--- a/Run_SLF_Files_manually/run_episode_file_2021.R
+++ b/Run_SLF_Files_manually/run_episode_file_2021.R
@@ -3,10 +3,77 @@ library(createslf)
 
 year <- "2021"
 
-processed_data_list <- targets::tar_read("processed_data_list_2021",
-  store = fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+targets_store <- fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+
+processed_data_list <- list(
+  acute = targets::tar_read(
+    "source_acute_extract_2021",
+    store = targets_store
+  ),
+  ae = targets::tar_read(
+    "source_ae_extract_2021",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_2021",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_2021",
+    store = targets_store
+  ),
+  dn = targets::tar_read(
+    "source_dn_extract_2021",
+    store = targets_store
+  ),
+  deaths = targets::tar_read(
+    "source_nrs_deaths_extract_2021",
+    store = targets_store
+  ),
+  homelessness = targets::tar_read(
+    "source_homelessness_extract_2021",
+    store = targets_store
+  ),
+  maternity = targets::tar_read(
+    "source_maternity_extract_2021",
+    store = targets_store
+  ),
+  mental_health = targets::tar_read(
+    "source_mental_health_extract_2021",
+    store = targets_store
+  ),
+  outpatients = targets::tar_read(
+    "source_outpatients_extract_2021",
+    store = targets_store
+  ),
+  gp_ooh = targets::tar_read(
+    "source_ooh_extract_2021",
+    store = targets_store
+  ),
+  prescribing = targets::tar_read(
+    "source_prescribing_extract_2021",
+    store = targets_store
+  ),
+  care_home = targets::tar_read(
+    "source_sc_care_home_2021",
+    store = targets_store
+  ),
+  home_care = targets::tar_read(
+    "source_sc_home_care_2021",
+    store = targets_store
+  ),
+  at = targets::tar_read(
+    "source_sc_alarms_tele_2021",
+    store = targets_store
+  ),
+  sds = targets::tar_read(
+    "source_sc_sds_2021",
+    store = targets_store
+  )
 )
 
 # Run episode file
 create_episode_file(processed_data_list, year = year) %>%
   process_tests_episode_file(year = year)
+
+## End of Script ##
diff --git a/Run_SLF_Files_manually/run_episode_file_2122.R b/Run_SLF_Files_manually/run_episode_file_2122.R
index 47400e2d1..cde974be2 100644
--- a/Run_SLF_Files_manually/run_episode_file_2122.R
+++ b/Run_SLF_Files_manually/run_episode_file_2122.R
@@ -3,10 +3,77 @@ library(createslf)
 
 year <- "2122"
 
-processed_data_list <- targets::tar_read("processed_data_list_2122",
-  store = fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+targets_store <- fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+
+processed_data_list <- list(
+  acute = targets::tar_read(
+    "source_acute_extract_2122",
+    store = targets_store
+  ),
+  ae = targets::tar_read(
+    "source_ae_extract_2122",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_2122",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_2122",
+    store = targets_store
+  ),
+  dn = targets::tar_read(
+    "source_dn_extract_2122",
+    store = targets_store
+  ),
+  deaths = targets::tar_read(
+    "source_nrs_deaths_extract_2122",
+    store = targets_store
+  ),
+  homelessness = targets::tar_read(
+    "source_homelessness_extract_2122",
+    store = targets_store
+  ),
+  maternity = targets::tar_read(
+    "source_maternity_extract_2122",
+    store = targets_store
+  ),
+  mental_health = targets::tar_read(
+    "source_mental_health_extract_2122",
+    store = targets_store
+  ),
+  outpatients = targets::tar_read(
+    "source_outpatients_extract_2122",
+    store = targets_store
+  ),
+  gp_ooh = targets::tar_read(
+    "source_ooh_extract_2122",
+    store = targets_store
+  ),
+  prescribing = targets::tar_read(
+    "source_prescribing_extract_2122",
+    store = targets_store
+  ),
+  care_home = targets::tar_read(
+    "source_sc_care_home_2122",
+    store = targets_store
+  ),
+  home_care = targets::tar_read(
+    "source_sc_home_care_2122",
+    store = targets_store
+  ),
+  at = targets::tar_read(
+    "source_sc_alarms_tele_2122",
+    store = targets_store
+  ),
+  sds = targets::tar_read(
+    "source_sc_sds_2122",
+    store = targets_store
+  )
 )
 
 # Run episode file
 create_episode_file(processed_data_list, year = year) %>%
   process_tests_episode_file(year = year)
+
+## End of Script ##
diff --git a/Run_SLF_Files_manually/run_episode_file_2223.R b/Run_SLF_Files_manually/run_episode_file_2223.R
index e64a57f32..ee83082f1 100644
--- a/Run_SLF_Files_manually/run_episode_file_2223.R
+++ b/Run_SLF_Files_manually/run_episode_file_2223.R
@@ -3,10 +3,77 @@ library(createslf)
 
 year <- "2223"
 
-processed_data_list <- targets::tar_read("processed_data_list_2223",
-  store = fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+targets_store <- fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+
+processed_data_list <- list(
+  acute = targets::tar_read(
+    "source_acute_extract_2223",
+    store = targets_store
+  ),
+  ae = targets::tar_read(
+    "source_ae_extract_2223",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_2223",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_2223",
+    store = targets_store
+  ),
+  dn = targets::tar_read(
+    "source_dn_extract_2223",
+    store = targets_store
+  ),
+  deaths = targets::tar_read(
+    "source_nrs_deaths_extract_2223",
+    store = targets_store
+  ),
+  homelessness = targets::tar_read(
+    "source_homelessness_extract_2223",
+    store = targets_store
+  ),
+  maternity = targets::tar_read(
+    "source_maternity_extract_2223",
+    store = targets_store
+  ),
+  mental_health = targets::tar_read(
+    "source_mental_health_extract_2223",
+    store = targets_store
+  ),
+  outpatients = targets::tar_read(
+    "source_outpatients_extract_2223",
+    store = targets_store
+  ),
+  gp_ooh = targets::tar_read(
+    "source_ooh_extract_2223",
+    store = targets_store
+  ),
+  prescribing = targets::tar_read(
+    "source_prescribing_extract_2223",
+    store = targets_store
+  ),
+  care_home = targets::tar_read(
+    "source_sc_care_home_2223",
+    store = targets_store
+  ),
+  home_care = targets::tar_read(
+    "source_sc_home_care_2223",
+    store = targets_store
+  ),
+  at = targets::tar_read(
+    "source_sc_alarms_tele_2223",
+    store = targets_store
+  ),
+  sds = targets::tar_read(
+    "source_sc_sds_2223",
+    store = targets_store
+  )
 )
 
 # Run episode file
 create_episode_file(processed_data_list, year = year) %>%
   process_tests_episode_file(year = year)
+
+## End of Script ##
diff --git a/Run_SLF_Files_manually/run_episode_file_2324.R b/Run_SLF_Files_manually/run_episode_file_2324.R
index 4a7f0ad29..508689f6d 100644
--- a/Run_SLF_Files_manually/run_episode_file_2324.R
+++ b/Run_SLF_Files_manually/run_episode_file_2324.R
@@ -3,10 +3,77 @@ library(createslf)
 
 year <- "2324"
 
-processed_data_list <- targets::tar_read("processed_data_list_2324",
-  store = fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+targets_store <- fs::path("/conf/sourcedev/Source_Linkage_File_Updates/", "_targets")
+
+processed_data_list <- list(
+  acute = targets::tar_read(
+    "source_acute_extract_2324",
+    store = targets_store
+  ),
+  ae = targets::tar_read(
+    "source_ae_extract_2324",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_2324",
+    store = targets_store
+  ),
+  cmh = targets::tar_read(
+    "source_cmh_extract_2324",
+    store = targets_store
+  ),
+  dn = targets::tar_read(
+    "source_dn_extract_2324",
+    store = targets_store
+  ),
+  deaths = targets::tar_read(
+    "source_nrs_deaths_extract_2324",
+    store = targets_store
+  ),
+  homelessness = targets::tar_read(
+    "source_homelessness_extract_2324",
+    store = targets_store
+  ),
+  maternity = targets::tar_read(
+    "source_maternity_extract_2324",
+    store = targets_store
+  ),
+  mental_health = targets::tar_read(
+    "source_mental_health_extract_2324",
+    store = targets_store
+  ),
+  outpatients = targets::tar_read(
+    "source_outpatients_extract_2324",
+    store = targets_store
+  ),
+  gp_ooh = targets::tar_read(
+    "source_ooh_extract_2324",
+    store = targets_store
+  ),
+  prescribing = targets::tar_read(
+    "source_prescribing_extract_2324",
+    store = targets_store
+  ),
+  care_home = targets::tar_read(
+    "source_sc_care_home_2324",
+    store = targets_store
+  ),
+  home_care = targets::tar_read(
+    "source_sc_home_care_2324",
+    store = targets_store
+  ),
+  at = targets::tar_read(
+    "source_sc_alarms_tele_2324",
+    store = targets_store
+  ),
+  sds = targets::tar_read(
+    "source_sc_sds_2324",
+    store = targets_store
+  )
 )
 
 # Run episode file
 create_episode_file(processed_data_list, year = year) %>%
   process_tests_episode_file(year = year)
+
+## End of Script ##

From bf63d73616a6cda5e32c8a0862b0c7794f23c80a Mon Sep 17 00:00:00 2001
From: Jennifer Thom <jennifer.thom@phs.scot>
Date: Wed, 3 Jul 2024 09:32:23 +0100
Subject: [PATCH 22/39] re-arrange brackets

---
 _targets.R | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/_targets.R b/_targets.R
index 7511ff03d..4583f21fb 100644
--- a/_targets.R
+++ b/_targets.R
@@ -554,7 +554,8 @@ list(
       combined_deaths_lookup,
       process_combined_deaths_lookup()
     )
-
+  )
+)
     ## End of Targets pipeline ##
 
     ################################################################################
@@ -652,5 +653,4 @@ list(
     #     version = "latest"
     #   )
     # )
-  )
-)
+

From ba2be196ec1eb9fee627413af3ea28081fac8533 Mon Sep 17 00:00:00 2001
From: Jennifer Thom <jennifer.thom@phs.scot>
Date: Wed, 3 Jul 2024 09:33:50 +0100
Subject: [PATCH 23/39] Style code

---
 _targets.R | 201 ++++++++++++++++++++++++++---------------------------
 1 file changed, 100 insertions(+), 101 deletions(-)

diff --git a/_targets.R b/_targets.R
index 4583f21fb..80c56701e 100644
--- a/_targets.R
+++ b/_targets.R
@@ -549,108 +549,107 @@ list(
         year,
         homelessness_data = source_homelessness_extract %>% slfhelper::get_chi()
       )
-    ),
-    tar_target(
-      combined_deaths_lookup,
-      process_combined_deaths_lookup()
     )
+  ),
+  tar_target(
+    combined_deaths_lookup,
+    process_combined_deaths_lookup()
   )
 )
-    ## End of Targets pipeline ##
-
-    ################################################################################
-    ## Redundant code which may still be useful for including ep/indiv files.
-    # tar_qs(
-    #   processed_data_list,
-    #   list(
-    #     source_acute_extract,
-    #     source_ae_extract,
-    #     source_cmh_extract,
-    #     source_dn_extract,
-    #     source_homelessness_extract,
-    #     source_maternity_extract,
-    #     source_mental_health_extract,
-    #     source_nrs_deaths_extract,
-    #     source_ooh_extract,
-    #     source_outpatients_extract,
-    #     source_prescribing_extract,
-    #     source_sc_care_home,
-    #     source_sc_home_care,
-    #     source_sc_sds,
-    #     source_sc_alarms_tele
-    #   )
-    # ),
-    # tar_target(
-    #   episode_file,
-    #   create_episode_file(
-    #     processed_data_list,
-    #     year,
-    #     homelessness_lookup = homelessness_lookup,
-    #     dd_data = source_dd_extract %>% slfhelper::get_chi(),
-    #     nsu_cohort = nsu_cohort %>% slfhelper::get_chi(),
-    #     ltc_data = source_ltc_lookup %>% slfhelper::get_chi(),
-    #     slf_pc_lookup = source_pc_lookup,
-    #     slf_gpprac_lookup = source_gp_lookup,
-    #     slf_deaths_lookup = slf_deaths_lookup %>% slfhelper::get_chi(),
-    #     sc_client = sc_client_lookup %>% slfhelper::get_chi(),
-    #     write_to_disk
-    #   )
-    # ),
-    # tar_target(
-    #   episode_file_tests,
-    #   process_tests_episode_file(
-    #     data = episode_file,
-    #     year = year
-    #   )
-    # ) # ,
-    # tar_target(
-    #   cross_year_tests,
-    #   process_tests_cross_year(year = year)
-    # ), # ,
-    # tar_target(
-    #   individual_file,
-    #   create_individual_file(
-    #     episode_file = episode_file,
-    #     year = year,
-    #     homelessness_lookup = homelessness_lookup,
-    #     write_to_disk = write_to_disk
-    #   )
-    # ),
-    # tar_target(
-    #   individual_file_tests,
-    #   process_tests_individual_file(
-    #     data = individual_file,
-    #     year = year
-    #   )
-    # ) # ,
-    # tar_target(
-    #   episode_file_dataset,
-    #   arrow::write_dataset(
-    #     dataset = episode_file,
-    #     path = fs::path(
-    #       get_year_dir(year),
-    #       stringr::str_glue("source-episode-file-{year}")
-    #     ),
-    #     format = "parquet",
-    #     # Should correspond to the available slfhelper filters
-    #     partitioning = c("recid", "hscp2018"),
-    #     compression = "zstd",
-    #     version = "latest"
-    #   )
-    # ),
-    # tar_target(
-    #   individual_file_dataset,
-    #   arrow::write_dataset(
-    #     dataset = individual_file,
-    #     path = fs::path(
-    #       get_year_dir(year),
-    #       stringr::str_glue("source-individual-file-{year}")
-    #     ),
-    #     format = "parquet",
-    #     # Should correspond to the available slfhelper filters
-    #     partitioning = c("hscp2018"),
-    #     compression = "zstd",
-    #     version = "latest"
-    #   )
-    # )
+## End of Targets pipeline ##
 
+################################################################################
+## Redundant code which may still be useful for including ep/indiv files.
+# tar_qs(
+#   processed_data_list,
+#   list(
+#     source_acute_extract,
+#     source_ae_extract,
+#     source_cmh_extract,
+#     source_dn_extract,
+#     source_homelessness_extract,
+#     source_maternity_extract,
+#     source_mental_health_extract,
+#     source_nrs_deaths_extract,
+#     source_ooh_extract,
+#     source_outpatients_extract,
+#     source_prescribing_extract,
+#     source_sc_care_home,
+#     source_sc_home_care,
+#     source_sc_sds,
+#     source_sc_alarms_tele
+#   )
+# ),
+# tar_target(
+#   episode_file,
+#   create_episode_file(
+#     processed_data_list,
+#     year,
+#     homelessness_lookup = homelessness_lookup,
+#     dd_data = source_dd_extract %>% slfhelper::get_chi(),
+#     nsu_cohort = nsu_cohort %>% slfhelper::get_chi(),
+#     ltc_data = source_ltc_lookup %>% slfhelper::get_chi(),
+#     slf_pc_lookup = source_pc_lookup,
+#     slf_gpprac_lookup = source_gp_lookup,
+#     slf_deaths_lookup = slf_deaths_lookup %>% slfhelper::get_chi(),
+#     sc_client = sc_client_lookup %>% slfhelper::get_chi(),
+#     write_to_disk
+#   )
+# ),
+# tar_target(
+#   episode_file_tests,
+#   process_tests_episode_file(
+#     data = episode_file,
+#     year = year
+#   )
+# ) # ,
+# tar_target(
+#   cross_year_tests,
+#   process_tests_cross_year(year = year)
+# ), # ,
+# tar_target(
+#   individual_file,
+#   create_individual_file(
+#     episode_file = episode_file,
+#     year = year,
+#     homelessness_lookup = homelessness_lookup,
+#     write_to_disk = write_to_disk
+#   )
+# ),
+# tar_target(
+#   individual_file_tests,
+#   process_tests_individual_file(
+#     data = individual_file,
+#     year = year
+#   )
+# ) # ,
+# tar_target(
+#   episode_file_dataset,
+#   arrow::write_dataset(
+#     dataset = episode_file,
+#     path = fs::path(
+#       get_year_dir(year),
+#       stringr::str_glue("source-episode-file-{year}")
+#     ),
+#     format = "parquet",
+#     # Should correspond to the available slfhelper filters
+#     partitioning = c("recid", "hscp2018"),
+#     compression = "zstd",
+#     version = "latest"
+#   )
+# ),
+# tar_target(
+#   individual_file_dataset,
+#   arrow::write_dataset(
+#     dataset = individual_file,
+#     path = fs::path(
+#       get_year_dir(year),
+#       stringr::str_glue("source-individual-file-{year}")
+#     ),
+#     format = "parquet",
+#     # Should correspond to the available slfhelper filters
+#     partitioning = c("hscp2018"),
+#     compression = "zstd",
+#     version = "latest"
+#   )
+# )

From a4cf7b6523a7564ad4d34bc6eabf07541a220e0b Mon Sep 17 00:00:00 2001
From: Jennifer Thom <jennifer.thom@phs.scot>
Date: Wed, 3 Jul 2024 10:27:54 +0100
Subject: [PATCH 24/39] Update run targets scripts

---
 Run_SLF_Files_targets/run_targets_1718.R | 8 --------
 Run_SLF_Files_targets/run_targets_1819.R | 8 --------
 Run_SLF_Files_targets/run_targets_1920.R | 8 --------
 Run_SLF_Files_targets/run_targets_2021.R | 8 --------
 Run_SLF_Files_targets/run_targets_2122.R | 8 --------
 Run_SLF_Files_targets/run_targets_2223.R | 8 --------
 Run_SLF_Files_targets/run_targets_2324.R | 8 --------
 7 files changed, 56 deletions(-)

diff --git a/Run_SLF_Files_targets/run_targets_1718.R b/Run_SLF_Files_targets/run_targets_1718.R
index ac03edd3f..a0dc17cf0 100644
--- a/Run_SLF_Files_targets/run_targets_1718.R
+++ b/Run_SLF_Files_targets/run_targets_1718.R
@@ -8,11 +8,3 @@ tar_make_future(
   names = (targets::contains("1718"))
 )
 
-# use targets to create individual files due to RAM limit
-library(createslf)
-
-episode_file <- arrow::read_parquet(get_slf_episode_path(year))
-
-# Run individual file
-create_individual_file(episode_file, year = year) %>%
-  process_tests_individual_file(year = year)
diff --git a/Run_SLF_Files_targets/run_targets_1819.R b/Run_SLF_Files_targets/run_targets_1819.R
index b60728359..981b6ee33 100644
--- a/Run_SLF_Files_targets/run_targets_1819.R
+++ b/Run_SLF_Files_targets/run_targets_1819.R
@@ -8,11 +8,3 @@ tar_make_future(
   names = (targets::contains("1819"))
 )
 
-# use targets to create individual files due to RAM limit
-library(createslf)
-
-episode_file <- arrow::read_parquet(get_slf_episode_path(year))
-
-# Run individual file
-create_individual_file(episode_file, year = year) %>%
-  process_tests_individual_file(year = year)
diff --git a/Run_SLF_Files_targets/run_targets_1920.R b/Run_SLF_Files_targets/run_targets_1920.R
index 897ee0b7a..79ee50644 100644
--- a/Run_SLF_Files_targets/run_targets_1920.R
+++ b/Run_SLF_Files_targets/run_targets_1920.R
@@ -8,11 +8,3 @@ tar_make_future(
   names = (targets::contains("1920"))
 )
 
-# use targets to create individual files due to RAM limit
-library(createslf)
-
-episode_file <- arrow::read_parquet(get_slf_episode_path(year))
-
-# Run individual file
-create_individual_file(episode_file, year = year) %>%
-  process_tests_individual_file(year = year)
diff --git a/Run_SLF_Files_targets/run_targets_2021.R b/Run_SLF_Files_targets/run_targets_2021.R
index 53333c014..c0433117d 100644
--- a/Run_SLF_Files_targets/run_targets_2021.R
+++ b/Run_SLF_Files_targets/run_targets_2021.R
@@ -8,11 +8,3 @@ tar_make_future(
   names = (targets::contains("2021"))
 )
 
-# use targets to create individual files due to RAM limit
-library(createslf)
-
-episode_file <- arrow::read_parquet(get_slf_episode_path(year))
-
-# Run individual file
-create_individual_file(episode_file, year = year) %>%
-  process_tests_individual_file(year = year)
diff --git a/Run_SLF_Files_targets/run_targets_2122.R b/Run_SLF_Files_targets/run_targets_2122.R
index 457fe33e7..3424daf08 100644
--- a/Run_SLF_Files_targets/run_targets_2122.R
+++ b/Run_SLF_Files_targets/run_targets_2122.R
@@ -8,11 +8,3 @@ tar_make_future(
   names = (targets::contains("2122"))
 )
 
-# use targets to create individual files due to RAM limit
-library(createslf)
-
-episode_file <- arrow::read_parquet(get_slf_episode_path(year))
-
-# Run individual file
-create_individual_file(episode_file, year = year) %>%
-  process_tests_individual_file(year = year)
diff --git a/Run_SLF_Files_targets/run_targets_2223.R b/Run_SLF_Files_targets/run_targets_2223.R
index fc851f3f7..052daed3e 100644
--- a/Run_SLF_Files_targets/run_targets_2223.R
+++ b/Run_SLF_Files_targets/run_targets_2223.R
@@ -8,11 +8,3 @@ tar_make_future(
   names = (targets::contains("2223"))
 )
 
-# use targets to create individual files due to RAM limit
-library(createslf)
-
-episode_file <- arrow::read_parquet(get_slf_episode_path(year))
-
-# Run individual file
-create_individual_file(episode_file, year = year) %>%
-  process_tests_individual_file(year = year)
diff --git a/Run_SLF_Files_targets/run_targets_2324.R b/Run_SLF_Files_targets/run_targets_2324.R
index 3b4c9b240..c34ae7061 100644
--- a/Run_SLF_Files_targets/run_targets_2324.R
+++ b/Run_SLF_Files_targets/run_targets_2324.R
@@ -8,11 +8,3 @@ tar_make_future(
   names = (targets::contains("2324"))
 )
 
-# use targets to create individual files due to RAM limit
-library(createslf)
-
-episode_file <- arrow::read_parquet(get_slf_episode_path(year))
-
-# Run individual file
-create_individual_file(episode_file, year = year) %>%
-  process_tests_individual_file(year = year)

From f840c534709451e110e98f1f18f11fc90da31fe4 Mon Sep 17 00:00:00 2001
From: Jennifer Thom <jennifer.thom@phs.scot>
Date: Wed, 3 Jul 2024 10:44:03 +0100
Subject: [PATCH 25/39] Set up all-targets script

---
 Run_SLF_Files_targets/run_all_targets.R | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 Run_SLF_Files_targets/run_all_targets.R

diff --git a/Run_SLF_Files_targets/run_all_targets.R b/Run_SLF_Files_targets/run_all_targets.R
new file mode 100644
index 000000000..fb5b94fab
--- /dev/null
+++ b/Run_SLF_Files_targets/run_all_targets.R
@@ -0,0 +1,5 @@
+library(targets)
+
+# use tar_make_future() to run targets for all years
+# This will run everything needed for creating the episode file.
+tar_make_future()

From 6d20852f5859c7dc6d335920101452b121c0e9bd Mon Sep 17 00:00:00 2001
From: Jennit07 <Jennit07@users.noreply.github.com>
Date: Wed, 3 Jul 2024 09:45:29 +0000
Subject: [PATCH 26/39] Style code

---
 Run_SLF_Files_targets/run_targets_1718.R | 1 -
 Run_SLF_Files_targets/run_targets_1819.R | 1 -
 Run_SLF_Files_targets/run_targets_1920.R | 1 -
 Run_SLF_Files_targets/run_targets_2021.R | 1 -
 Run_SLF_Files_targets/run_targets_2122.R | 1 -
 Run_SLF_Files_targets/run_targets_2223.R | 1 -
 Run_SLF_Files_targets/run_targets_2324.R | 1 -
 7 files changed, 7 deletions(-)

diff --git a/Run_SLF_Files_targets/run_targets_1718.R b/Run_SLF_Files_targets/run_targets_1718.R
index a0dc17cf0..e85a89ff8 100644
--- a/Run_SLF_Files_targets/run_targets_1718.R
+++ b/Run_SLF_Files_targets/run_targets_1718.R
@@ -7,4 +7,3 @@ tar_make_future(
   # it does not recognise `contains(year)`
   names = (targets::contains("1718"))
 )
-
diff --git a/Run_SLF_Files_targets/run_targets_1819.R b/Run_SLF_Files_targets/run_targets_1819.R
index 981b6ee33..6957054c8 100644
--- a/Run_SLF_Files_targets/run_targets_1819.R
+++ b/Run_SLF_Files_targets/run_targets_1819.R
@@ -7,4 +7,3 @@ tar_make_future(
   # it does not recognise `contains(year)`
   names = (targets::contains("1819"))
 )
-
diff --git a/Run_SLF_Files_targets/run_targets_1920.R b/Run_SLF_Files_targets/run_targets_1920.R
index 79ee50644..ef6272fca 100644
--- a/Run_SLF_Files_targets/run_targets_1920.R
+++ b/Run_SLF_Files_targets/run_targets_1920.R
@@ -7,4 +7,3 @@ tar_make_future(
   # it does not recognise `contains(year)`
   names = (targets::contains("1920"))
 )
-
diff --git a/Run_SLF_Files_targets/run_targets_2021.R b/Run_SLF_Files_targets/run_targets_2021.R
index c0433117d..237078e05 100644
--- a/Run_SLF_Files_targets/run_targets_2021.R
+++ b/Run_SLF_Files_targets/run_targets_2021.R
@@ -7,4 +7,3 @@ tar_make_future(
   # it does not recognise `contains(year)`
   names = (targets::contains("2021"))
 )
-
diff --git a/Run_SLF_Files_targets/run_targets_2122.R b/Run_SLF_Files_targets/run_targets_2122.R
index 3424daf08..dc64b625f 100644
--- a/Run_SLF_Files_targets/run_targets_2122.R
+++ b/Run_SLF_Files_targets/run_targets_2122.R
@@ -7,4 +7,3 @@ tar_make_future(
   # it does not recognise `contains(year)`
   names = (targets::contains("2122"))
 )
-
diff --git a/Run_SLF_Files_targets/run_targets_2223.R b/Run_SLF_Files_targets/run_targets_2223.R
index 052daed3e..7d8677a80 100644
--- a/Run_SLF_Files_targets/run_targets_2223.R
+++ b/Run_SLF_Files_targets/run_targets_2223.R
@@ -7,4 +7,3 @@ tar_make_future(
   # it does not recognise `contains(year)`
   names = (targets::contains("2223"))
 )
-
diff --git a/Run_SLF_Files_targets/run_targets_2324.R b/Run_SLF_Files_targets/run_targets_2324.R
index c34ae7061..bcebe5fb8 100644
--- a/Run_SLF_Files_targets/run_targets_2324.R
+++ b/Run_SLF_Files_targets/run_targets_2324.R
@@ -7,4 +7,3 @@ tar_make_future(
   # it does not recognise `contains(year)`
   names = (targets::contains("2324"))
 )
-

From 5482a75599a76da435a2cd97a61c3361e85feb1f Mon Sep 17 00:00:00 2001
From: lizihao-anu <lizihao-anu@users.noreply.github.com>
Date: Thu, 4 Jul 2024 13:24:31 +0000
Subject: [PATCH 27/39] Style code

---
 R/process_extract_acute.R  | 40 +++++++++++++++++++++-----------------
 R/process_extract_gp_ooh.R |  1 -
 2 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/R/process_extract_acute.R b/R/process_extract_acute.R
index c68909c8a..82926bd92 100644
--- a/R/process_extract_acute.R
+++ b/R/process_extract_acute.R
@@ -16,7 +16,6 @@ process_extract_acute <- function(data,
                                   year,
                                   acute_cup_path = get_boxi_extract_path(year, "acute_cup"),
                                   write_to_disk = TRUE) {
-
   # Only run for a single year
   stopifnot(length(year) == 1L)
 
@@ -62,7 +61,7 @@ process_extract_acute <- function(data,
     ) %>%
     # Add oldtadm as a factor with labels
     dplyr::mutate(oldtadm = factor(.data$oldtadm,
-                                   levels = 0L:8L
+      levels = 0L:8L
     )) %>%
     dplyr::mutate(
       unique_row_num = dplyr::row_number()
@@ -80,25 +79,30 @@ process_extract_acute <- function(data,
       "CUP Marker" = readr::col_integer(),
       "CUP Pathway Name" = readr::col_character()
     )
-  ) %>% dplyr::select(
-    chi = "UPI Number [C]",
-    case_reference_number = "Case Reference Number [C]",
-    record_keydate1 = "Acute Admission Date",
-    record_keydate2 = "Acute Discharge Date",
-    tadm = "Acute Admission Type Code",
-    disch = "Acute Discharge Type Code",
-    cup_marker = "CUP Marker",
-    cup_pathway = "CUP Pathway Name"
-  ) %>% dplyr::distinct()
+  ) %>%
+    dplyr::select(
+      chi = "UPI Number [C]",
+      case_reference_number = "Case Reference Number [C]",
+      record_keydate1 = "Acute Admission Date",
+      record_keydate2 = "Acute Discharge Date",
+      tadm = "Acute Admission Type Code",
+      disch = "Acute Discharge Type Code",
+      cup_marker = "CUP Marker",
+      cup_pathway = "CUP Pathway Name"
+    ) %>%
+    dplyr::distinct()
 
   acute_clean <- acute_clean %>%
     dplyr::left_join(acute_cup,
-                     by = c("record_keydate1",
-                            "record_keydate2",
-                            "case_reference_number",
-                            "chi",
-                            "tadm",
-                            "disch"))
+      by = c(
+        "record_keydate1",
+        "record_keydate2",
+        "case_reference_number",
+        "chi",
+        "tadm",
+        "disch"
+      )
+    )
 
   acute_processed <- acute_clean %>%
     dplyr::select(
diff --git a/R/process_extract_gp_ooh.R b/R/process_extract_gp_ooh.R
index f663729d7..e8d07b9e8 100644
--- a/R/process_extract_gp_ooh.R
+++ b/R/process_extract_gp_ooh.R
@@ -16,7 +16,6 @@ process_extract_gp_ooh <- function(year,
                                    data_list,
                                    gp_ooh_cup_path = get_boxi_extract_path(year, "gp_ooh_cup"),
                                    write_to_disk = TRUE) {
-
   diagnosis_extract <- process_extract_ooh_diagnosis(data_list[["diagnosis"]], year)
   outcomes_extract <- process_extract_ooh_outcomes(data_list[["outcomes"]], year)
   consultations_extract <- process_extract_ooh_consultations(data_list[["consultations"]], year)

From f30344cd996a43d13905ef9bc138648d2f8681f8 Mon Sep 17 00:00:00 2001
From: Megan McNicol <43570769+SwiftySalmon@users.noreply.github.com>
Date: Mon, 8 Jul 2024 11:08:16 +0100
Subject: [PATCH 28/39] Update Run_SLF_Files_targets/run_all_targets.R

Co-authored-by: Zihao Li <zihao.li@phs.scot>
---
 Run_SLF_Files_targets/run_all_targets.R | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/Run_SLF_Files_targets/run_all_targets.R b/Run_SLF_Files_targets/run_all_targets.R
index fb5b94fab..73399cb75 100644
--- a/Run_SLF_Files_targets/run_all_targets.R
+++ b/Run_SLF_Files_targets/run_all_targets.R
@@ -3,3 +3,7 @@ library(targets)
 # use tar_make_future() to run targets for all years
 # This will run everything needed for creating the episode file.
 tar_make_future()
+
+# Combine deaths lookup here rather than in targets to make sure that
+# it is run after the death file for each year is produced.
+combined_deaths_lookup = process_combined_deaths_lookup()

From 06813ab42d4ed6e46eb57d98b282a4a8ef1656b0 Mon Sep 17 00:00:00 2001
From: SwiftySalmon <SwiftySalmon@users.noreply.github.com>
Date: Mon, 8 Jul 2024 10:09:39 +0000
Subject: [PATCH 29/39] Style code

---
 Run_SLF_Files_targets/run_all_targets.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Run_SLF_Files_targets/run_all_targets.R b/Run_SLF_Files_targets/run_all_targets.R
index 73399cb75..2e93f1cca 100644
--- a/Run_SLF_Files_targets/run_all_targets.R
+++ b/Run_SLF_Files_targets/run_all_targets.R
@@ -6,4 +6,4 @@ tar_make_future()
 
 # Combine deaths lookup here rather than in targets to make sure that
 # it is run after the death file for each year is produced.
-combined_deaths_lookup = process_combined_deaths_lookup()
+combined_deaths_lookup <- process_combined_deaths_lookup()

From 8b4706d7e62ad8e66c2e2a945593ce90efd786e5 Mon Sep 17 00:00:00 2001
From: Zihao Li <lizihao_anu@outlook.com>
Date: Mon, 8 Jul 2024 11:44:36 +0100
Subject: [PATCH 30/39] remove combined_deaths_lookup from targets

---
 _targets.R | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/_targets.R b/_targets.R
index 80c56701e..0c1b223ff 100644
--- a/_targets.R
+++ b/_targets.R
@@ -550,11 +550,11 @@ list(
         homelessness_data = source_homelessness_extract %>% slfhelper::get_chi()
       )
     )
-  ),
-  tar_target(
-    combined_deaths_lookup,
-    process_combined_deaths_lookup()
-  )
+  ) #,
+  # tar_target(
+  #   combined_deaths_lookup,
+  #   process_combined_deaths_lookup()
+  # )
 )
 ## End of Targets pipeline ##
 

From 09076f495a5235ed72564ea81a3f1da2b6c3dad5 Mon Sep 17 00:00:00 2001
From: lizihao-anu <lizihao-anu@users.noreply.github.com>
Date: Mon, 8 Jul 2024 10:45:59 +0000
Subject: [PATCH 31/39] Style code

---
 _targets.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_targets.R b/_targets.R
index 0c1b223ff..e2dbacbe8 100644
--- a/_targets.R
+++ b/_targets.R
@@ -550,7 +550,7 @@ list(
         homelessness_data = source_homelessness_extract %>% slfhelper::get_chi()
       )
     )
-  ) #,
+  ) # ,
   # tar_target(
   #   combined_deaths_lookup,
   #   process_combined_deaths_lookup()

From 400841eb509a7190e45c52399d9756d84747db24 Mon Sep 17 00:00:00 2001
From: Zihao Li <lizihao_anu@outlook.com>
Date: Mon, 8 Jul 2024 15:51:55 +0100
Subject: [PATCH 32/39] fix acute_cup and gp_ooh_cup paths

---
 R/get_boxi_extract_path.R | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/R/get_boxi_extract_path.R b/R/get_boxi_extract_path.R
index 8ea1c80f2..4ec1de3a8 100644
--- a/R/get_boxi_extract_path.R
+++ b/R/get_boxi_extract_path.R
@@ -23,6 +23,7 @@ get_boxi_extract_path <- function(
       "gp_ooh-c",
       "gp_ooh-d",
       "gp_ooh-o",
+      "gp_ooh_cup",
       "homelessness",
       "maternity",
       "mh",
@@ -45,13 +46,13 @@ get_boxi_extract_path <- function(
     "ae" ~ "anon-A&E-episode-level-extract",
     "ae_cup" ~ "anon-A&E-UCD-CUP-extract",
     "acute" ~ "anon-Acute-episode-level-extract",
-    "acute_cup" ~ "Actue-cup-extract",
+    "acute_cup" ~ "anon-Actue-cup-extract",
     "cmh" ~ "anon-Community-MH-contact-level-extract",
     "dn" ~ "anon-District-Nursing-contact-level-extract",
     "gp_ooh-c" ~ "anon-GP-OoH-consultations-extract",
     "gp_ooh-d" ~ "anon-GP-OoH-diagnosis-extract",
     "gp_ooh-o" ~ "anon-GP-OoH-outcomes-extract",
-    "gp_ooh_cup" ~ "GP-OoH-cup-extract",
+    "gp_ooh_cup" ~ "anon-GP-OoH-cup-extract",
     "homelessness" ~ "anon-Homelessness-extract",
     "maternity" ~ "anon-Maternity-episode-level-extract",
     "mh" ~ "anon-Mental-Health-episode-level-extract",

From 5879fe4d9fa971374c6c732bd5c62bf6cb07917b Mon Sep 17 00:00:00 2001
From: lizihao-anu <lizihao-anu@users.noreply.github.com>
Date: Mon, 8 Jul 2024 14:53:53 +0000
Subject: [PATCH 33/39] Update documentation

---
 man/get_boxi_extract_path.Rd | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/man/get_boxi_extract_path.Rd b/man/get_boxi_extract_path.Rd
index 69bb886c4..5a318834a 100644
--- a/man/get_boxi_extract_path.Rd
+++ b/man/get_boxi_extract_path.Rd
@@ -7,7 +7,8 @@
 get_boxi_extract_path(
   year,
   type = c("ae", "ae_cup", "acute", "acute_cup", "cmh", "deaths", "dn", "gp_ooh-c",
-    "gp_ooh-d", "gp_ooh-o", "homelessness", "maternity", "mh", "outpatients")
+    "gp_ooh-d", "gp_ooh-o", "gp_ooh_cup", "homelessness", "maternity", "mh",
+    "outpatients")
 )
 }
 \arguments{

From 2e4b1605a2b86cfbf445930810a8b784a04d156a Mon Sep 17 00:00:00 2001
From: Zihao Li <lizihao_anu@outlook.com>
Date: Mon, 8 Jul 2024 16:24:25 +0100
Subject: [PATCH 34/39] fix typo

---
 R/get_boxi_extract_path.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/R/get_boxi_extract_path.R b/R/get_boxi_extract_path.R
index 4ec1de3a8..04488dacb 100644
--- a/R/get_boxi_extract_path.R
+++ b/R/get_boxi_extract_path.R
@@ -46,7 +46,7 @@ get_boxi_extract_path <- function(
     "ae" ~ "anon-A&E-episode-level-extract",
     "ae_cup" ~ "anon-A&E-UCD-CUP-extract",
     "acute" ~ "anon-Acute-episode-level-extract",
-    "acute_cup" ~ "anon-Actue-cup-extract",
+    "acute_cup" ~ "anon-Acute-cup-extract",
     "cmh" ~ "anon-Community-MH-contact-level-extract",
     "dn" ~ "anon-District-Nursing-contact-level-extract",
     "gp_ooh-c" ~ "anon-GP-OoH-consultations-extract",

From b56ef2522f5d94affaf7f1910c73e8488f800d1e Mon Sep 17 00:00:00 2001
From: Zihao Li <lizihao_anu@outlook.com>
Date: Tue, 9 Jul 2024 14:45:41 +0100
Subject: [PATCH 35/39] adapt acute_cup for anon_chi

---
 R/process_extract_acute.R | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/R/process_extract_acute.R b/R/process_extract_acute.R
index 82926bd92..62969238f 100644
--- a/R/process_extract_acute.R
+++ b/R/process_extract_acute.R
@@ -67,30 +67,31 @@ process_extract_acute <- function(data,
       unique_row_num = dplyr::row_number()
     )
 
-  acute_cup <- read_file(
-    path = acute_cup_path,
+  acute_cup = read_file(
+    # path = get_boxi_extract_path(year, "acute_cup"),
+    path = cup_file_name,
     col_type = readr::cols(
+      "anon_chi" = readr::col_character(),
       "Acute Admission Date" = readr::col_date(format = "%Y/%m/%d %T"),
       "Acute Discharge Date" = readr::col_date(format = "%Y/%m/%d %T"),
       "Acute Admission Type Code" = readr::col_character(),
       "Acute Discharge Type Code" = readr::col_character(),
       "Case Reference Number [C]" = readr::col_character(),
-      "UPI Number [C]" = readr::col_character(),
       "CUP Marker" = readr::col_integer(),
       "CUP Pathway Name" = readr::col_character()
     )
+  ) %>% dplyr::select(
+    anon_chi = "anon_chi",
+    case_reference_number = "Case Reference Number [C]",
+    record_keydate1 = "Acute Admission Date",
+    record_keydate2 = "Acute Discharge Date",
+    tadm = "Acute Admission Type Code",
+    disch = "Acute Discharge Type Code",
+    cup_marker = "CUP Marker",
+    cup_pathway = "CUP Pathway Name"
   ) %>%
-    dplyr::select(
-      chi = "UPI Number [C]",
-      case_reference_number = "Case Reference Number [C]",
-      record_keydate1 = "Acute Admission Date",
-      record_keydate2 = "Acute Discharge Date",
-      tadm = "Acute Admission Type Code",
-      disch = "Acute Discharge Type Code",
-      cup_marker = "CUP Marker",
-      cup_pathway = "CUP Pathway Name"
-    ) %>%
-    dplyr::distinct()
+    dplyr::distinct() %>%
+    slfhelper::get_chi()
 
   acute_clean <- acute_clean %>%
     dplyr::left_join(acute_cup,

From 8c3873c2f0685a5a80acf276d45902a494974d57 Mon Sep 17 00:00:00 2001
From: lizihao-anu <lizihao-anu@users.noreply.github.com>
Date: Tue, 9 Jul 2024 13:47:52 +0000
Subject: [PATCH 36/39] Style code

---
 R/process_extract_acute.R | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/R/process_extract_acute.R b/R/process_extract_acute.R
index 62969238f..1c7e03322 100644
--- a/R/process_extract_acute.R
+++ b/R/process_extract_acute.R
@@ -67,7 +67,7 @@ process_extract_acute <- function(data,
       unique_row_num = dplyr::row_number()
     )
 
-  acute_cup = read_file(
+  acute_cup <- read_file(
     # path = get_boxi_extract_path(year, "acute_cup"),
     path = cup_file_name,
     col_type = readr::cols(
@@ -80,16 +80,17 @@ process_extract_acute <- function(data,
       "CUP Marker" = readr::col_integer(),
       "CUP Pathway Name" = readr::col_character()
     )
-  ) %>% dplyr::select(
-    anon_chi = "anon_chi",
-    case_reference_number = "Case Reference Number [C]",
-    record_keydate1 = "Acute Admission Date",
-    record_keydate2 = "Acute Discharge Date",
-    tadm = "Acute Admission Type Code",
-    disch = "Acute Discharge Type Code",
-    cup_marker = "CUP Marker",
-    cup_pathway = "CUP Pathway Name"
   ) %>%
+    dplyr::select(
+      anon_chi = "anon_chi",
+      case_reference_number = "Case Reference Number [C]",
+      record_keydate1 = "Acute Admission Date",
+      record_keydate2 = "Acute Discharge Date",
+      tadm = "Acute Admission Type Code",
+      disch = "Acute Discharge Type Code",
+      cup_marker = "CUP Marker",
+      cup_pathway = "CUP Pathway Name"
+    ) %>%
     dplyr::distinct() %>%
     slfhelper::get_chi()
 

From 283e00457b7ff1fc636ee0ac2ce2dd6dd6671bee Mon Sep 17 00:00:00 2001
From: Zihao Li <lizihao_anu@outlook.com>
Date: Wed, 10 Jul 2024 15:25:16 +0100
Subject: [PATCH 37/39] minor changes

---
 R/add_activity_after_death_flag.R       | 2 +-
 R/process_extract_acute.R               | 3 +--
 Run_SLF_Files_targets/run_all_targets.R | 2 +-
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/R/add_activity_after_death_flag.R b/R/add_activity_after_death_flag.R
index 5f973eb1e..c81e1fc79 100644
--- a/R/add_activity_after_death_flag.R
+++ b/R/add_activity_after_death_flag.R
@@ -125,7 +125,7 @@ add_activity_after_death_flag <- function(
 #'
 #'
 # Read data------------------------------------------------
-process_deaths_lookup <- function(update = latest_update(),
+process_combined_deaths_lookup <- function(update = latest_update(),
                                   write_to_disk = TRUE, ...) {
   dir_folder <- "/conf/hscdiip/SLF_Extracts/Deaths"
   file_names <- list.files(dir_folder,
diff --git a/R/process_extract_acute.R b/R/process_extract_acute.R
index 1c7e03322..c46f175c7 100644
--- a/R/process_extract_acute.R
+++ b/R/process_extract_acute.R
@@ -68,8 +68,7 @@ process_extract_acute <- function(data,
     )
 
   acute_cup <- read_file(
-    # path = get_boxi_extract_path(year, "acute_cup"),
-    path = cup_file_name,
+    path = acute_cup_path,
     col_type = readr::cols(
       "anon_chi" = readr::col_character(),
       "Acute Admission Date" = readr::col_date(format = "%Y/%m/%d %T"),
diff --git a/Run_SLF_Files_targets/run_all_targets.R b/Run_SLF_Files_targets/run_all_targets.R
index 2e93f1cca..9ea6e9e6f 100644
--- a/Run_SLF_Files_targets/run_all_targets.R
+++ b/Run_SLF_Files_targets/run_all_targets.R
@@ -6,4 +6,4 @@ tar_make_future()
 
 # Combine deaths lookup here rather than in targets to make sure that
 # it is run after the death file for each year is produced.
-combined_deaths_lookup <- process_combined_deaths_lookup()
+createslf::process_combined_deaths_lookup()

From 70b18506f1b95333fbfadbc0398d31e033e02727 Mon Sep 17 00:00:00 2001
From: lizihao-anu <lizihao-anu@users.noreply.github.com>
Date: Wed, 10 Jul 2024 14:27:10 +0000
Subject: [PATCH 38/39] Style code

---
 R/add_activity_after_death_flag.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/R/add_activity_after_death_flag.R b/R/add_activity_after_death_flag.R
index c81e1fc79..39c894681 100644
--- a/R/add_activity_after_death_flag.R
+++ b/R/add_activity_after_death_flag.R
@@ -126,7 +126,7 @@ add_activity_after_death_flag <- function(
 #'
 # Read data------------------------------------------------
 process_combined_deaths_lookup <- function(update = latest_update(),
-                                  write_to_disk = TRUE, ...) {
+                                           write_to_disk = TRUE, ...) {
   dir_folder <- "/conf/hscdiip/SLF_Extracts/Deaths"
   file_names <- list.files(dir_folder,
     pattern = "^anon-slf_deaths_lookup_.*parquet",

From a7bac7391b834f74c22787913b0648604cea7623 Mon Sep 17 00:00:00 2001
From: lizihao-anu <lizihao-anu@users.noreply.github.com>
Date: Wed, 10 Jul 2024 14:27:34 +0000
Subject: [PATCH 39/39] Update documentation

---
 NAMESPACE                                              |  2 +-
 ...ths_lookup.Rd => process_combined_deaths_lookup.Rd} | 10 +++++++---
 2 files changed, 8 insertions(+), 4 deletions(-)
 rename man/{process_deaths_lookup.Rd => process_combined_deaths_lookup.Rd} (81%)

diff --git a/NAMESPACE b/NAMESPACE
index c6bba9b38..b4314febd 100644
--- a/NAMESPACE
+++ b/NAMESPACE
@@ -88,11 +88,11 @@ export(midpoint_fy)
 export(next_fy)
 export(phs_db_connection)
 export(previous_update)
+export(process_combined_deaths_lookup)
 export(process_costs_ch_rmd)
 export(process_costs_dn_rmd)
 export(process_costs_gp_ooh_rmd)
 export(process_costs_hc_rmd)
-export(process_deaths_lookup)
 export(process_extract_acute)
 export(process_extract_ae)
 export(process_extract_alarms_telecare)
diff --git a/man/process_deaths_lookup.Rd b/man/process_combined_deaths_lookup.Rd
similarity index 81%
rename from man/process_deaths_lookup.Rd
rename to man/process_combined_deaths_lookup.Rd
index e897e49a2..7d0a75fc7 100644
--- a/man/process_deaths_lookup.Rd
+++ b/man/process_combined_deaths_lookup.Rd
@@ -1,10 +1,14 @@
 % Generated by roxygen2: do not edit by hand
 % Please edit documentation in R/add_activity_after_death_flag.R
-\name{process_deaths_lookup}
-\alias{process_deaths_lookup}
+\name{process_combined_deaths_lookup}
+\alias{process_combined_deaths_lookup}
 \title{Create and read SLF Deaths lookup from processed BOXI NRS deaths extracts}
 \usage{
-process_deaths_lookup(update = latest_update(), write_to_disk = TRUE, ...)
+process_combined_deaths_lookup(
+  update = latest_update(),
+  write_to_disk = TRUE,
+  ...
+)
 }
 \arguments{
 \item{update}{the update month (defaults to use \code{\link[=latest_update]{latest_update()}})}