Updated dev with latest data update

dfe-analytical-services · Nov 24, 2023 · 86cd11d · 86cd11d
2 parents d9f5c9b + 8bc7c87
commit 86cd11d
Show file tree

Hide file tree

Showing 22 changed files with 71,887 additions and 36,548 deletions.
diff --git a/R/dashboard_panels.R b/R/dashboard_panels.R
@@ -85,7 +85,7 @@ homepage_panel <- function() {
                   br(),
                   h3("Coverage"),
                   h4(textOutput("daily_schools_count")),
-                  p("This number is approximately 83% of the number of schools participating in the School Census. As schools opt in to sharing of data, the number of schools reporting may change over time."),
+                  p("This number is approximately 86% of the number of schools participating in the School Census. As schools opt in to sharing of data, the number of schools reporting may change over time."),
                   p("Absence rates are provided broken down by state-funded primary, secondary and special schools. At national and regional level, absence figures are also provided across all schools. In recognition that response rates are not equal across school types and, therefore, not representative of the total school population, the total absence figure for all schools has been weighted based on the Spring 2023 school census. Weighted total figures are not included at local authority level due to the low number of schools involved."),
                   br(),
                   h3("National statistics"),

diff --git a/R/prerun_utils.R b/R/prerun_utils.R
@@ -6,7 +6,7 @@ run_data_update <- function() {
   # pa_autumn_file <- "data/export_autumn_pa_output_2023_08_07.csv"
   # pa_spring_file <- "data/export_spring_pa_output_2023_08_07.csv"
   # pa_summer_file <- "data/export_summer_pa_output_2023_08_07.csv"
-  attendance_data_raw <- fread("data/sql_export_2023_11_07_full.csv")
+  attendance_data_raw <- fread("data/sql_export_2023_11_20_full.csv")
 
   attendance_data <- process_attendance_data(
     attendance_data_raw,

diff --git a/data/EES_daily_data.csv b/data/EES_daily_data.csv
diff --git a/data/EES_weekly_data.csv b/data/EES_weekly_data.csv
diff --git a/data/EES_ytd_data.csv b/data/EES_ytd_data.csv
diff --git a/data/attendance_data_dashboard.csv b/data/attendance_data_dashboard.csv
diff --git a/data/sql_export_2023_11_20_full.csv b/data/sql_export_2023_11_20_full.csv
diff --git a/datafiles_log.csv b/datafiles_log.csv
@@ -86,4 +86,5 @@ data/sql_export_2023_10_09_full.csv,published
 data/enrolments_schools_denominator_041023.csv,published
 data/sql_export_2023_10_23_full.csv,published
 data/enrolments_schools_denominator_011123.csv,published
-data/sql_export_2023_11_07_full.csv,published
+data/sql_export_2023_11_07_full.csv,published
+data/sql_export_2023_11_20_full.csv,published
diff --git a/global.R b/global.R
@@ -90,7 +90,7 @@ google_analytics_key <- "DG7P4WLB0Y"
 # Read in data
 # attendance_data_raw <- fread("data/Weekly_dummy_data.csv")
 start_date <- as.Date("2023-09-11")
-end_date <- as.Date("2023-10-27")
+end_date <- as.Date("2023-11-10")
 # funeral_date <- as.Date("2022-09-19")
 # strike_date_1 <- as.Date("2023-02-01")
 # strike_date_2 <- as.Date("2023-03-15")
@@ -110,8 +110,8 @@ end_date <- as.Date("2023-10-27")
 # summer_start <- as.Date("2023-04-01")
 # summer_end <- as.Date("2023-07-21")
 
-most_recent_week_dates <- paste0("Latest week -", as.Date(end_date) - 11, " to ", as.Date(end_date) - 7)
-ytd_dates <- paste0("Year to date -", as.Date(start_date), "to", as.Date(end_date) - 7)
+most_recent_week_dates <- paste0("Latest week -", as.Date(end_date) - 4, " to ", as.Date(end_date))
+ytd_dates <- paste0("Year to date -", as.Date(start_date), "to", as.Date(end_date))
 
 
 school_freq_count <- fread("data/enrolments_schools_denominator_011123.csv")
@@ -241,8 +241,8 @@ mapshape <- st_read("data/CTYUA_MAY_2023_UK_BUC.shp") %>% st_transform(crs = 432
 mapdata0 <- attendance_data %>%
   mutate(time_identifier = as.numeric(str_remove_all(time_identifier, "Week "))) %>%
   filter(time_period == max(time_period)) %>%
-  # filter(time_identifier == max(time_identifier)) %>%
-  filter(time_identifier == max(time_identifier) - 1) %>%
+  filter(time_identifier == max(time_identifier)) %>%
+  # filter(time_identifier == max(time_identifier) - 1) %>%
   filter(geographic_level == "Local authority") %>%
   filter(breakdown == "Weekly")
 

diff --git a/server.R b/server.R
@@ -531,8 +531,8 @@ server <- function(input, output, session) {
       time_period == max(time_period),
       breakdown == "Weekly"
     ) %>%
-      # filter(time_identifier == max(time_identifier)) %>%
-      filter(time_identifier == max(time_identifier) - 1) %>%
+      filter(time_identifier == max(time_identifier)) %>%
+      #  filter(time_identifier == max(time_identifier) - 1) %>%
       mutate(
         overall_absence_perc = overall_absence_perc / 100,
         authorised_absence_perc = authorised_absence_perc / 100,
@@ -1119,8 +1119,8 @@ server <- function(input, output, session) {
     pull(attendance_date)
 
   output$daily_schools_count <- renderText({
-    paste0(scales::comma(schools_count), " schools provided information on the latest full day of data prior to half-term, i.e. ", schools_count_date)
-    # paste0(scales::comma(schools_count), " schools provided information on the latest full day of data, i.e. ", schools_count_date)
+    # paste0(scales::comma(schools_count), " schools provided information on the latest full day of data prior to half-term, i.e. ", schools_count_date)
+    paste0(scales::comma(schools_count), " schools provided information on the latest full day of data, i.e. ", schools_count_date)
     # paste0(scales::comma(schools_count_pre_ht), " schools provided information on the last full day of data prior to half-term, i.e. ", schools_count_date_pre_ht)
   })
 
@@ -1504,11 +1504,11 @@ server <- function(input, output, session) {
     last_update_date <- live_attendance_data_weekly() %>%
       pull(attendance_date)
 
-    # last_update_date <- as.Date(last_update_date) + 17
-    last_update_date <- as.Date(last_update_date) + 24
+    last_update_date <- as.Date(last_update_date) + 17
+    # last_update_date <- as.Date(last_update_date) + 24
 
-    # paste0("Data was last updated on ", last_update_date, ".")
-    paste0("Data was last updated on 2023-11-09")
+    paste0("Data was last updated on ", last_update_date, ".")
+    # paste0("Data was last updated on 2023-11-09")
   })
 
   output$la_clarity_dates <- renderText({
@@ -1518,8 +1518,8 @@ server <- function(input, output, session) {
     most_recent_fullweek_date <- live_attendance_data_weekly() %>%
       pull(week_commencing)
 
-    paste0("Data on this tab relates to the week commencing 2023-10-16")
-    # paste0("Data on this tab relates to the week commencing ", most_recent_fullweek_date, ".")
+    # paste0("Data on this tab relates to the week commencing 2023-10-16")
+    paste0("Data on this tab relates to the week commencing ", most_recent_fullweek_date, ".")
   })
 
   output$update_dates <- renderText({
@@ -1533,17 +1533,17 @@ server <- function(input, output, session) {
     last_update_date <- live_attendance_data_weekly() %>%
       pull(attendance_date)
 
-    # last_update_date <- as.Date(last_update_date) + 17
-    last_update_date <- as.Date(last_update_date) + 24
+    last_update_date <- as.Date(last_update_date) + 17
+    # last_update_date <- as.Date(last_update_date) + 24
 
     next_update_date <- live_attendance_data_weekly() %>%
       pull(attendance_date)
 
-    # next_update_date <- as.Date(next_update_date) + 31
-    next_update_date <- as.Date(next_update_date) + 38
+    next_update_date <- as.Date(next_update_date) + 31
+    # next_update_date <- as.Date(next_update_date) + 38
 
-    paste0("Data was last updated on 2023-11-09 and is next expected to be updated on 2023-11-23. The latest full week of data for this breakdown was the week commencing ", most_recent_fullweek_date, ".")
-    # paste0("Data was last updated on ", last_update_date, " and is next expected to be updated on ", next_update_date, ". The latest full week of data was the week commencing ", most_recent_fullweek_date, ".")
+    # paste0("Data was last updated on 2023-11-09 and is next expected to be updated on 2023-11-23. The latest full week of data for this breakdown was the week commencing ", most_recent_fullweek_date, ".")
+    paste0("Data was last updated on ", last_update_date, " and is next expected to be updated on ", next_update_date, ". The latest full week of data was the week commencing ", most_recent_fullweek_date, ".")
     # paste0("Data was last updated on ", last_update_date, ". The latest full week of data was the week commencing ", most_recent_fullweek_date, ".")
   })
 
@@ -1558,17 +1558,17 @@ server <- function(input, output, session) {
     last_update_date <- live_attendance_data_weekly() %>%
       pull(attendance_date)
 
-    # last_update_date <- as.Date(last_update_date) + 17
-    last_update_date <- as.Date(last_update_date) + 24
+    last_update_date <- as.Date(last_update_date) + 17
+    # last_update_date <- as.Date(last_update_date) + 24
 
     next_update_date <- live_attendance_data_weekly() %>%
       pull(attendance_date)
 
-    # next_update_date <- as.Date(next_update_date) + 31
-    next_update_date <- as.Date(next_update_date) + 38
+    next_update_date <- as.Date(next_update_date) + 31
+    # next_update_date <- as.Date(next_update_date) + 38
 
-    paste0("Data was last updated on 2023-11-09 and is next expected to be updated on 2023-11-23. The latest full week of data for this breakdown was the week commencing ", most_recent_fullweek_date, ".")
-    # paste0("Data was last updated on ", last_update_date, " and is next expected to be updated on ", next_update_date, ". The latest full week of data was the week commencing ", most_recent_fullweek_date, ".")
+    #  paste0("Data was last updated on 2023-11-09 and is next expected to be updated on 2023-11-23. The latest full week of data for this breakdown was the week commencing ", most_recent_fullweek_date, ".")
+    paste0("Data was last updated on ", last_update_date, " and is next expected to be updated on ", next_update_date, ". The latest full week of data was the week commencing ", most_recent_fullweek_date, ".")
     # paste0("Data was last updated on ", last_update_date, ". The latest full week of data was the week commencing ", most_recent_fullweek_date, ".")
   })
 
@@ -1582,16 +1582,16 @@ server <- function(input, output, session) {
 
     last_update_date <- live_attendance_data_weekly() %>%
       pull(attendance_date) %>%
-      # as.Date() + 17
-      as.Date() + 24
+      as.Date() + 17
+    # as.Date() + 24
 
     next_update_date <- live_attendance_data_weekly() %>%
       pull(attendance_date) %>%
-      # as.Date() + 31
-      as.Date() + 38
+      as.Date() + 31
+    # as.Date() + 38
 
-    paste0("Data was last updated on 2023-11-09 and is next expected to be updated on 2023-11-23. The latest full week of data was the week commencing 2023-10-16.")
-    # paste0("Data was last updated on ", last_update_date, " and is next expected to be updated on ", next_update_date, ". The latest full week of data was the week commencing ", most_recent_fullweek_date, ".")
+    # paste0("Data was last updated on 2023-11-09 and is next expected to be updated on 2023-11-23. The latest full week of data was the week commencing 2023-10-16.")
+    paste0("Data was last updated on ", last_update_date, " and is next expected to be updated on ", next_update_date, ". The latest full week of data was the week commencing ", most_recent_fullweek_date, ".")
     # paste0("Data was last updated on ", last_update_date, ". The latest full week of data was the week commencing ", most_recent_fullweek_date, ".")
   })
 

diff --git a/tests/shinytest/UI_tests-expected/001.json b/tests/shinytest/UI_tests-expected/001.json
@@ -13,8 +13,8 @@
     "ts_choice": "latestweeks"
   },
   "output": {
-    "daily_schools_count": "16,880 schools provided information on the latest full day of data prior to half-term, i.e. 2023-10-20",
-    "headline_update_date": "Data was last updated on 2023-11-09",
-    "homepage_update_dates": "Data was last updated on 2023-11-09 and is next expected to be updated on 2023-11-23. The latest full week of data was the week commencing 2023-10-16."
+    "daily_schools_count": "18,390 schools provided information on the latest full day of data, i.e. 2023-11-10",
+    "headline_update_date": "Data was last updated on 2023-11-23.",
+    "homepage_update_dates": "Data was last updated on 2023-11-23 and is next expected to be updated on 2023-12-07. The latest full week of data was the week commencing 2023-11-06."
   }
 }
diff --git a/tests/shinytest/UI_tests-expected/002.json b/tests/shinytest/UI_tests-expected/002.json
@@ -30,11 +30,11 @@
             "automargin": true,
             "title": "Date",
             "tickvals": [
-              "2023-10-16",
-              "2023-10-17",
-              "2023-10-18",
-              "2023-10-19",
-              "2023-10-20"
+              "2023-11-06",
+              "2023-11-07",
+              "2023-11-08",
+              "2023-11-09",
+              "2023-11-10"
             ],
             "zeroline": true,
             "zerolinewidth": 2,
@@ -106,18 +106,18 @@
             "mode": "markers+lines",
             "type": "scatter",
             "x": [
-              "2023-10-16",
-              "2023-10-17",
-              "2023-10-18",
-              "2023-10-19",
-              "2023-10-20"
+              "2023-11-06",
+              "2023-11-07",
+              "2023-11-08",
+              "2023-11-09",
+              "2023-11-10"
             ],
             "y": [
-              5.2564623033347999,
-              4.8347632735574297,
-              4.9881143549088298,
-              5.5787811939996699,
-              7.7984194133993299
+              5.3825239595253302,
+              4.6010978416983104,
+              4.5592878450999299,
+              4.5162825268901896,
+              5.4116188253968502
             ],
             "line": {
               "color": "#12436D"
@@ -150,18 +150,18 @@
             "mode": "markers+lines",
             "type": "scatter",
             "x": [
-              "2023-10-16",
-              "2023-10-17",
-              "2023-10-18",
-              "2023-10-19",
-              "2023-10-20"
+              "2023-11-06",
+              "2023-11-07",
+              "2023-11-08",
+              "2023-11-09",
+              "2023-11-10"
             ],
             "y": [
-              3.6401202009688101,
-              3.3624021495062402,
-              3.35253107097164,
-              3.6583923173618,
-              4.9058606539901604
+              3.7361614826628702,
+              3.3490376859058899,
+              3.3269865810875898,
+              3.34155566917133,
+              3.97604469197285
             ],
             "line": {
               "color": "#28A197"
@@ -194,18 +194,18 @@
             "mode": "markers+lines",
             "type": "scatter",
             "x": [
-              "2023-10-16",
-              "2023-10-17",
-              "2023-10-18",
-              "2023-10-19",
-              "2023-10-20"
+              "2023-11-06",
+              "2023-11-07",
+              "2023-11-08",
+              "2023-11-09",
+              "2023-11-10"
             ],
             "y": [
-              1.6163421023659901,
-              1.4723611240512,
-              1.63558328393719,
-              1.9203888766378701,
-              2.89255875940917
+              1.6463624768624601,
+              1.25206015579242,
+              1.2323012640123401,
+              1.1747268577188601,
+              1.435574133424
             ],
             "line": {
               "color": "#F46A25"
@@ -348,9 +348,9 @@
         }
       ]
     },
-    "headline_bullet_title_nat": "Headline figures for the latest week - 2023-10-16 to 2023-10-20: primary state-funded school attendance at national level",
-    "weekly_absence_rate_nat": "• 5.6% of sessions were recorded as absence",
-    "weekly_attendance_rate_nat": "• 94.4% of sessions were recorded as attending",
-    "weekly_illness_rate_nat": "• 3% of sessions were recorded as illness"
+    "headline_bullet_title_nat": "Headline figures for the latest week - 2023-11-06 to 2023-11-10: primary state-funded school attendance at national level",
+    "weekly_absence_rate_nat": "• 4.9% of sessions were recorded as absence",
+    "weekly_attendance_rate_nat": "• 95.1% of sessions were recorded as attending",
+    "weekly_illness_rate_nat": "• 2.9% of sessions were recorded as illness"
   }
 }