Merge branch 'June-24-update' into add_activity_after_death

.github/actions/spelling/excludes.txt

@@ -57,4 +57,5 @@
 ^\Q.github/workflows/spelling.yml\E$
 ^\Q/tmp/check-spelling/Public-Health-Scotland/source-linkage-files/commits/f13483ca341940e8549dc23c930da2f23dd0ac43.message\E$
 ^\Q/tmp/check-spelling/Public-Health-Scotland/source-linkage-files/pull-request/613/summary.txt\E$
+^\Qinst/WORDLIST\E$
 ignore$

.github/actions/spelling/expect.txt

@@ -1,5 +1,5 @@
 acc
-accom
+Accom
 admloc
 admtype
 adpe
@@ -10,14 +10,15 @@ atlassian
 atrialfib
 attendcat
 aut
+bba
 bedday
 BFO
 birthtime
 bloodbfo
 bodyloc
 boxi
 callr
-canx
+Canx
 carehome
 careinspectorate
 categorises
@@ -45,6 +46,7 @@ costsfy
 covr
 cph
 createslf
+csf
 customise
 cvd
 dataframe
@@ -68,12 +70,15 @@ dischto
 disdest
 dminutes
 dna
+docx
 dontrun
 downup
 dplyr
 dsn
 dtplyr
 dvprod
+eee
+efa
 endomet
 envir
 fcase
@@ -117,6 +122,7 @@ idpc
 infyyear
 intzone
 ipdc
+Isdsf
 issuenumber
 itle
 iwalk
@@ -136,6 +142,7 @@ lgl
 lintr
 los
 ltc
+ltd
 lubridate
 magrittr
 markdownguide
@@ -207,6 +214,7 @@ rspm
 rstudio
 rstudioapi
 rtype
+sco
 scoial
 scotp
 SDcols
@@ -224,12 +232,14 @@ slfhelper
 smr
 smra
 smrtype
+sourcedev
 sparra
 spd
 spss
 stadm
 starwars
 stefanzweifel
+stopwords
 stringdist
 stringr
 submis
@@ -242,19 +252,22 @@ thom
 tibble
 tidyr
 tidyselect
+TJDX
 todo
 uid
 ungroup
 unicode
 updown
 upi
 vline
+wdbf
+WORDLIST
 workflows
 xintercept
 xlsx
 yearstay
 yml
 yyyyqx
-Zihao
+zihao
 zsav
 zstd

00_Sort_BI_Extracts.R

@@ -1,5 +1,5 @@
 # Define the source directory and financial year pattern
-compress_files <- FALSE
+compress_files <- TRUE
 source_dir <- "/conf/sourcedev/Source_Linkage_File_Updates/Extracts Temp"
 pattern <- "-20(\\d{4})\\.csv"
 
@@ -20,31 +20,49 @@ extract_financial_year <- function(filename) {
   }
 }
 
-# Create directories for each financial year and move files
-for (csv_file in csv_files) {
+# Create a function to read variable names
+is_chi_in_file <- function(filename) {
+  data <- read.csv(filename, nrow = 1)
+  return(grepl("UPI", names(data)) %>% any())
+}
+
+# function to move files
+move_temps_to_year_extract <- function(csv_file, compress_files = TRUE) {
   financial_year <- extract_financial_year(csv_file)
   # check if year directory exists
   if (!is.null(financial_year)) {
     financial_year_dir <- file.path("/conf/sourcedev/Source_Linkage_File_Updates", financial_year, "Extracts")
-    # if not, create the year directory
+    # if financial_year_dir does not exist, create the year directory
     if (!dir.exists(financial_year_dir)) {
       dir.create(financial_year_dir)
     }
 
+    new_file_path <- file.path(financial_year_dir, paste0("anon-", basename(csv_file)))
+
+    # set up new file path location to move each file to their destination.
+    chi_in_file <- is_chi_in_file(csv_file)
+    if (chi_in_file) {
+      read_file(csv_file) %>%
+        dplyr::rename_with(~ paste0("chi"), tidyselect::contains("UPI")) %>%
+        slfhelper::get_anon_chi() %>%
+        readr::write_csv(file = new_file_path)
+      cat("Replaced chi with anon chi:", csv_file, "to", new_file_path, "\n")
+    } else {
+      fs::file_copy(csv_file, new_file_path, overwrite = TRUE)
+      cat("Moved", csv_file, "to", new_file_path, "\n")
+    }
+
     # compress file
     if (compress_files) {
-      cat("Compressing:", basename(csv_file), "\n")
+      cat("Compressing:", basename(new_file_path), "\n")
       system2(
         command = "gzip",
-        args = shQuote(csv_file)
+        args = shQuote(new_file_path)
       )
-      csv_file <- paste0(csv_file, ".gz")
     }
-
-    # move file
-    new_file_path <- file.path(financial_year_dir, basename(csv_file))
-    fs::file_copy(csv_file, new_file_path, overwrite = TRUE)
+    # remove old files
     file.remove(csv_file)
-    cat("Moved:", csv_file, "to", new_file_path, "\n")
   }
 }
+
+lapply(csv_files, move_temps_to_year_extract, compress_files = compress_files)

NAMESPACE

@@ -125,6 +125,7 @@ export(process_tests_ae)
 export(process_tests_alarms_telecare)
 export(process_tests_care_home)
 export(process_tests_cmh)
+export(process_tests_cross_year)
 export(process_tests_delayed_discharges)
 export(process_tests_district_nursing)
 export(process_tests_episode_file)
@@ -153,6 +154,7 @@ export(produce_episode_file_tests)
 export(produce_sc_sandpit_tests)
 export(produce_source_extract_tests)
 export(produce_test_comparison)
+export(read_dev_slf_file)
 export(read_extract_acute)
 export(read_extract_ae)
 export(read_extract_cmh)