incorporated convolution kernel, negative binomial dispersion paramet…

…ers, and fix for incidence calculation #194

misc/experiments/convolution/convolution.R

@@ -34,7 +34,7 @@ sir = Compartmental(system.file("starter_models", "sir", package = "macpan2"))$s
   , .mats_to_return = c("state", "incidence", "reports")
 )
 sir$insert$expressions(
-    incidence ~ foi * S
+  incidence ~ foi * S
   , reports ~ convolution(incidence, kernel)
   , .at = Inf
   , .phase = "during"
@@ -45,3 +45,29 @@ sir$insert$expressions(
   %>% ggplot()
   + geom_line(aes(time, value, colour = matrix))
 )
+
+## added by JF - converting to new macpan2 interface
+sir = (mp_tmb_library(system.file("starter_models", "sir", package = "macpan2"))
+        |> mp_tmb_insert(phase = "during"
+             , at = Inf
+             , expressions = list(incidence ~ infection * S, reports ~ convolution(incidence, kernel))
+             , default = list(
+                 N = 10000 # need to set N instead of S, because S is derived
+               , I = 1
+               , R = 0
+               , incidence = empty_matrix
+               , gamma = 0.2
+               , beta = 0.4
+               , kernel = make_delay_kernel(0.1, 11, 0.25)
+               , reports = empty_matrix
+               )
+         )
+         |> mp_simulator(time_steps = 100, outputs = c("incidence","reports"))
+)
+
+(sir
+  %>% mp_trajectory()
+  %>% ggplot()
+  + geom_line(aes(time, value, colour = matrix))
+  + facet_wrap(vars(matrix),scales = 'free')
+)

misc/experiments/macpan-base-fitting/data.R

@@ -66,3 +66,6 @@ mobility_dat = (
   %>% summarize(mobility_ind = mean(c_across(where(is.numeric)),na.rm = TRUE))
   %>% ungroup()
 )
+
+saveRDS(clean_tsdata, "./misc/experiments/macpan-base-fitting/clean_tsdata.RDS")
+saveRDS(mobility_dat, "./misc/experiments/macpan-base-fitting/mobility_dat.RDS")

misc/experiments/macpan-base-fitting/fit.R

@@ -1,5 +1,7 @@
 library(macpan2)
-source('./misc/experiments/macpan-base-fitting/data.R') #~ 2GB of data
+library(dplyr)
+library(ggplot2)
+
 
 ## -------------------------
 ## local function
@@ -26,6 +28,11 @@ backtrans <- function(x) {
 ## format data
 ## -------------------------
 
+# read in data
+#source('./misc/experiments/macpan-base-fitting/data.R') #~ 2GB of data
+clean_tsdata = readRDS("./misc/experiments/macpan-base-fitting/clean_tsdata.RDS")
+mobility_dat = readRDS("./misc/experiments/macpan-base-fitting/mobility_dat.RDS")
+
 # get data in the right format for calibration
 formatted_tsdata = (clean_tsdata
                     |> rename(matrix=var)
@@ -86,9 +93,7 @@ X = cbind(
   , S_j(1:time_steps, beta_changepoints[3], 3) * formatted_mobility_dat$log_mobility_ind
 ) %>% as.matrix()
 
-#X_sparse = macpan2:::sparse_matrix_notation(X, TRUE)
-X_sparse = sparse_matrix_notation(X, TRUE)
-
+X_sparse = macpan2:::sparse_matrix_notation(X, TRUE)
 
 model_matrix_values = X_sparse$values
 row_ind = X_sparse$row_index
@@ -97,87 +102,102 @@ col_ind = X_sparse$col_index
 log_model_coefficients = rep(log(1e-2),ncol(X))
 
 
-# kernel, for convolution
-# should be Gamma distribution with moments 
-# chosen to match empirical estimates of case-reporting delays (where is this info)
-k=c(0.5, 0.25, 0.25)
-
-## To Add
-# observation error for reports and deaths
-
 # --------------- model spec ---------------
 #macpan_base = (mp_tmb_library("starter_models","macpan_base",package="macpan2")
 source("./inst/starter_models/macpan_base/tmb.R")
 macpan_base = (spec 
   # add variable transformations:
   |> mp_tmb_insert(phase = "before"
-                   , at = 1L,
-                   expressions = list(
-                       zeta ~ exp(log_zeta)
-                     , beta0 ~ exp(log_beta0)
-                    ),
-                   default = list(
-                      log_zeta = log(1e-2)
-                    , log_beta0 = log(1e-2)
-                    ) 
-                   )
+    , at = 1L,
+    expressions = list(
+       zeta ~ exp(log_zeta)
+     , beta0 ~ exp(log_beta0)
+     , nonhosp_mort ~ 1/(1+exp((-logit_nonhosp_mort)))
+    ),
+    default = list(
+      log_zeta = log(1e-2)
+    , log_beta0 = log(1e-2)
+    , logit_nonhosp_mort = log((1e-2)/(1-(1e-2)))
+    ) 
+  )
 
   # add accumulator variables:
   # H2 - individuals in acute care after discharge (already computed)
   # D - cumulative deaths (already computed)
   # X - cumulative hospital admissions
   # death - new deaths each time step
   |> mp_tmb_insert(phase = "during"
-                   , at = Inf
-                   , expressions = list(
-                       mp_per_capita_flow("H", "X", H.X ~ Is.ICUs + Is.ICUd)
-                     , death ~ ICUd.D * ICUd + Is.D * Is
-                     )
-                   , default = list(X = 0, theta_death = 1)
-                   )
+    , at = Inf
+    , expressions = list(
+        mp_per_capita_flow("H", "X", H.X ~ Is.ICUs + Is.ICUd)
+      , death ~ ICUd.D * ICUd + Is.D * Is
+      )
+    , default = list(X = 0)
+  )
 
   # add phenomenological heterogeneity:
   |> mp_tmb_update(phase = "during"
-                   , at =1L
-                   , expressions = list(mp_per_capita_flow("S", "E", S.E ~ (S^(zeta - 1)) * (beta / (N^zeta)) * (Ia * Ca + Ip * Cp + Im * Cm * (1 - iso_m) + Is * Cs *(1 - iso_s))))
-                   , default = list(zeta = 0,beta = 1)
-                   )
+    , at =1L
+    , expressions = list(mp_per_capita_flow("S", "E", S.E ~ (S^(zeta-1)) * (beta / (N^zeta)) * (Ia * Ca + Ip * Cp + Im * Cm * (1 - iso_m) + Is * Cs *(1 - iso_s))))
+    , default = list(beta = 1)
+  )
 
   # add condensation variables:
   # I = sum of all infectious compartments (time step incidence)
   |> mp_tmb_insert(phase = "during"
-                   , at = Inf
-                   , expressions = list(I ~ Ia + Ip + Im + Is)
-                   , default = list(I = 0)
-                   )
-
+    , at = Inf
+    , expressions = list(I ~ Ia + Ip + Im + Is)
+    , default = list(I = 0)
+  )
+  # add incidence:
+  |> mp_tmb_insert(phase = "during"
+    , at = Inf
+    #, expressions = list(incidence ~ S.E * (S^2))
+    , expressions = list(incidence ~ S.E * S)
+    , default = list(incidence = 0)
+  )
+
+  # compute gamma-density delay kernel for convolution:
+   |> mp_tmb_insert(phase = "before"
+    , at = Inf
+    , expressions = list(
+        gamma_shape ~ 1 / (c_delay_cv^2)
+      , gamma_scale ~ c_delay_mean * c_delay_cv^2
+      , gamma ~ pgamma(1:(qmax+1), gamma_shape, gamma_scale)
+      , delta ~ gamma[1:(qmax)] - gamma[0:(qmax - 1)]
+      , kappa ~ c_prop * delta / sum(delta)
+    )
+    , default = list(qmax = 16)
+  )
+
   # add convolution to compute case reports:
   |> mp_tmb_insert(phase = "during"
-                  , at = Inf
-                  , expressions = list(report ~ c_prop * convolution(I, k))
-                  , default = list(k = k, theta_report = 100)
-                  )
-  |> mp_tmb_insert(phase = "after"
-                  , at = 1L
-                  # remove beginning time steps depending on kernel length
-                  , expressions = list(report ~ rbind_time(report, i))
-                  , integers = list(i = length(k):time_steps)
-                  )
-
+    , at = Inf
+    , expressions = list(report ~ convolution(incidence, kappa))
+    , default = list(report = 0)
+  )
+   ##|> mp_tmb_insert(phase = "after"
+    # , at = 1L
+    # remove beginning time steps depending on kernel length
+    # , expressions = list(report ~ rbind_time(report, i))
+    # , integers = list(i = 16:(time_steps-1))
+  #   , expressions = list(report ~ rbind_time(report, 1:189), report[i] ~ 0)
+  #   , integers = list(i = c(0:14))
+  # )
+
   # add time-varying transmission with mobility data:
   |> mp_tmb_insert(phase = "before"
-                  , at = Inf
-                  , expressions = list(relative_beta_values ~ exp(group_sums(model_matrix_values * log_model_coefficients[col_ind], row_ind, model_matrix_values))
-                                       )
-                  , default = list(log_model_coefficients = log_model_coefficients, model_matrix_values = model_matrix_values, relative_beta_values=model_matrix_values)
-                  , integers = list(row_ind = row_ind, col_ind = col_ind)
-                  )
+    , at = Inf
+    , expressions = list(relative_beta_values ~ exp(group_sums(model_matrix_values * log_model_coefficients[col_ind], row_ind, model_matrix_values)))
+    , default = list(log_model_coefficients = log_model_coefficients, model_matrix_values = model_matrix_values, relative_beta_values=model_matrix_values)
+    , integers = list(row_ind = row_ind, col_ind = col_ind)
+  )
   |> mp_tmb_insert(phase = "during"
-                  , at = 1L
-                  , expressions = list(
-                     beta ~ beta0 * relative_beta_values[time_step(1)]
-                    )
-                  )
+    , at = 1L
+    , expressions = list(
+       beta ~ beta0 * relative_beta_values[time_step(1)]
+      )
+  )
 )
 # ------------------------------------------
 
@@ -186,29 +206,56 @@ macpan_base = (spec
   %>% mp_simulator(time_steps = time_steps, outputs = c("Ia","Ip","Im","report"))
   %>% mp_trajectory()
   %>% ggplot(aes(time,value))+
-    geom_point()+
+    geom_line(aes(colour=matrix))+
+    facet_wrap(vars(matrix),scales = 'free')
+)
+(macpan_base
+  %>% mp_simulator(time_steps = time_steps, outputs = c("S","E","death","report"))
+  %>% mp_trajectory()
+  %>% ggplot(aes(time,value))+
+    geom_line(aes(colour=matrix))+
+    facet_wrap(vars(matrix),scales = 'free')
+)
+(macpan_base
+  %>% mp_simulator(time_steps = time_steps, outputs = c("incidence","report"))
+  %>% mp_trajectory()
+  %>% ggplot(aes(time,value))+
+    geom_line(aes(colour=matrix))+
     facet_wrap(vars(matrix),scales = 'free')
 )
 
+(macpan_base
+  %>% mp_simulator(time_steps = 1L, outputs = c("kappa"))
+  %>% mp_trajectory()
+  %>% select(value)
+  %>% pull()
+  %>% plot(., type='l')
+)
+
+
 ## -------------------------
 ## calibration (base model)
 ## -------------------------
+options(macpan2_default_loss = "neg_bin") 
 
 # fitting to deaths and reports
 mb_calib = mp_tmb_calibrator(
     spec = macpan_base ##|> mp_euler_multinomial()
   , data = formatted_tsdata
   , traj = c("report","death") 
-  ##more in table 1
+  ## Table 1 in manuscript
   , par = c(
       "E"
+    , "Ia"
+    , "Ip"
+    , "Im"
+    , "Is"
     , "log_zeta"
     , "log_beta0"
     , "log_model_coefficients"
-    , "nonhosp_mort" #I think this is eta - 'Probability of mortality without hospitalization'
-    #, "theta_report"
-    #, "theta_death"
+    , "logit_nonhosp_mort" #I think this is eta - 'Probability of mortality without hospitalization'
     ) 
+  , outputs = c("S","E","I","death","report")
 )
 
 mp_optimize(mb_calib)
@@ -223,15 +270,20 @@ mp_tmb_coef(mb_calib, conf.int = TRUE) |> backtrans()
 (ggplot(formatted_tsdata, aes(time,value))
    + geom_point()
    + geom_line(aes(time, value)
-               , data = fitted_data
+               , data = fitted_data |> filter(matrix %in% c("death","report"))
                , colour = "red"
    )
    + geom_ribbon(aes(time, ymin = conf.low, ymax = conf.high)
-                 , data = fitted_data
+                 , data = fitted_data |> filter(matrix %in% c("death","report"))
                  , alpha = 0.2
                  , colour = "red"
    )
    + facet_wrap(vars(matrix),scales = 'free')
    + theme_bw()
-   + ylab("prevalence")
+)
+
+(ggplot(fitted_data |> filter(matrix %in% c("S","E","I")), aes(time,value))
+  + geom_point()
+  + facet_wrap(vars(matrix),scales = 'free')
+  + theme_bw()
 )

tests/testthat/test-convolution.R

@@ -57,3 +57,31 @@ s = mp_tmb_model_spec(
 mp_simulator(s, time_steps = 10L, outputs = "Z") |> mp_final() |> cbind(manual_convolution[3:10])
 mp_simulator(s, time_steps = 10L, outputs = "Y") |> mp_trajectory() |> cbind(manual_convolution)
 
+
+# macpan_base delay kernel and convolution example
+# https://canmod.net/misc/flex_specs#computing-convolutions
+qmax = 4 # length of kernel
+n = 6  # number of time steps
+c_delay_cv = 0.25
+c_delay_mean = 11
+c_prop = 0.1
+gamma_shape = 1 / (c_delay_cv^2)
+gamma_scale = c_delay_mean * c_delay_cv^2
+gamma = pgamma(1:(qmax+1), gamma_shape, gamma_scale)
+delta = gamma[2:(qmax+1)] - gamma[1:(qmax)]
+kappa = c_prop * delta / sum(delta)
+
+sim_data = simple_sims(
+    iteration_exprs = expr
+  , time_steps = n
+  , mats = list(X = X, Y = Y, k = kappa)
+) |> arrange(matrix)
+
+x = sim_data %>% filter(matrix=="X") %>% select(value) %>% pull()
+
+macpan2_convolution = sim_data %>% filter(matrix=="Y" & time %in% c(4,5)) %>% pull(value)
+# z calculation from https://canmod.net/misc/flex_specs#computing-convolutions
+manual_convolution = c(kappa[4]*x[1]+kappa[3]*x[2]+kappa[2]*x[2]+kappa[1]*x[4]
+                       , kappa[4]*x[2]+kappa[3]*x[3]+kappa[2]*x[4]+kappa[1]*x[5])
+
+cbind(manual_convolution, macpan2_convolution)