From 7b199a5febde86a3fdb7b869f8f5be4221284ccd Mon Sep 17 00:00:00 2001 From: Roger Bivand Date: Mon, 14 Oct 2024 13:01:14 +0200 Subject: [PATCH 01/14] attempt duplicate sample handling in licd_multi --- DESCRIPTION | 4 +-- R/licd_boots.R | 88 ++++++++++++++++++++++++++++++++--------------- man/licd_multi.Rd | 4 +-- 3 files changed, 64 insertions(+), 32 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 9c455562..808dfc2c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: spdep -Version: 1.3-6 -Date: 2024-08-31 +Version: 1.3-7 +Date: 2024-10-13 Title: Spatial Dependence: Weighting Schemes, Statistics Encoding: UTF-8 Authors@R: c(person("Roger", "Bivand", role = c("cre", "aut"), diff --git a/R/licd_boots.R b/R/licd_boots.R index 24074c1c..c920cd45 100644 --- a/R/licd_boots.R +++ b/R/licd_boots.R @@ -2,7 +2,7 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), adjust.n=TRUE, nsim = 0L, iseed = NULL, no_repeat_in_row=FALSE, control=list()) { con <- list(comp_binary=TRUE, binomial_punif_alternative="greater", - jcm_same_punif_alternative="less", jcm_diff_punif_alternative="greater") + jcm_same_punif_alternative="less", jcm_diff_punif_alternative="greater", rank_ties.method="min") nmsC <- names(con) con[(namc <- names(control))] <- control if (length(noNms <- namc[!namc %in% nmsC])) @@ -43,6 +43,7 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), assign("no_repeat_in_row", no_repeat_in_row, envir=env) assign("almost.1", almost.1, envir=env) assign("comp_binary", con$comp_binary, envir=env) + assign("ties.method", con$rank_ties.method, envir=env) varlist = ls(envir = env) permLICD_int <- function(i, env) { @@ -63,6 +64,7 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), k <- get("k", envir=env) lk <- get("lk", envir=env) comp_binary <- get("comp_binary", envir=env) + ties.method <- get("ties.method", envir=env) local_jcm_BW <- function(xi, sn, wc) { xi <- factor(as.numeric(!xi)+1, levels=c(1L, 2L), @@ -170,43 +172,61 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), } sx_i <- cbind(rep(xi, times=nsim), sx_i) + u_sx_i <- unique(sx_i) + reps <- integer(nrow(u_sx_i)) + len_reps <- length(reps) + for (i in seq(along=reps)) + reps[i] <- sum(apply(sx_i, 1, function(x) all(x == u_sx_i[i,]))) - c1_comp_sim_i <- apply(sx_i, 1, + u_c1_comp_sim_i <- apply(u_sx_i, 1, function(y) ifelse(comp_binary, sum(y == xi), sum(w_i_i * (y == xi)) + 1)) + c1_comp_sim_i <- rep(u_c1_comp_sim_i, times=reps) c1_comp_sim_i_rank <- rank(c(c1_comp_sim_i, - c1_comp_obs_i))[(nsim + 1L)] - c4_comp_bin_BW_sim_i <- sapply(c1_comp_sim_i, function(y) { + c1_comp_obs_i), ties.method=ties.method)[(nsim + 1L)] + + u_c4_comp_bin_BW_sim_i <- sapply(u_c1_comp_sim_i, function(y) { pbinom(y, c3_crdip1, c2_prop_level_i, lower.tail=TRUE) }) + c4_comp_bin_BW_sim_i <- rep(u_c4_comp_bin_BW_sim_i, times=reps) c4_comp_bin_BW_sim_i_rank <- rank(c(c4_comp_bin_BW_sim_i, - almost.1 * c4_comp_bin_BW_i))[(nsim + 1L)] - c5_comp_bin_BW_sim_i <- sapply(c1_comp_sim_i, function(y) { + c4_comp_bin_BW_i), ties.method=ties.method)[(nsim + 1L)] + + u_c5_comp_bin_BW_sim_i <- sapply(u_c1_comp_sim_i, function(y) { pbinom(y, c3_crdip1, c2_prop_level_i, lower.tail=FALSE) }) + c5_comp_bin_BW_sim_i <- rep(u_c5_comp_bin_BW_sim_i, times=reps) c5_comp_bin_BW_sim_i_rank <- rank(c(c5_comp_bin_BW_sim_i, - almost.1 * c5_comp_bin_BW_i))[(nsim + 1L)] - c5a_comp_bin_BW_sim_i <- sapply(c1_comp_sim_i, function(y) { + c5_comp_bin_BW_i), ties.method=ties.method)[(nsim + 1L)] + + u_c5a_comp_bin_BW_sim_i <- sapply(u_c1_comp_sim_i, function(y) { pbinom(y-1, c3_crdip1, c2_prop_level_i, lower.tail=FALSE) }) + c5a_comp_bin_BW_sim_i <- rep(u_c5a_comp_bin_BW_sim_i, times=reps) c5a_comp_bin_BW_sim_i_rank <- rank(c(c5a_comp_bin_BW_sim_i, - almost.1 * c5a_comp_bin_BW_i))[(nsim + 1L)] - c6_comp_chi_BW_sim_i <- sapply(c1_comp_sim_i, function(y) { + c5a_comp_bin_BW_i), ties.method=ties.method)[(nsim + 1L)] + + u_c6_comp_chi_BW_sim_i <- sapply(u_c1_comp_sim_i, function(y) { local_chi(c(y, c3_crdip1-y), E_BW) }) + c6_comp_chi_BW_sim_i <- rep(u_c6_comp_chi_BW_sim_i, times=reps) c6_comp_chi_BW_sim_i_rank <- rank(c(c6_comp_chi_BW_sim_i, - almost.1 * c6_comp_chi_BW_i))[(nsim + 1L)] - c7_comp_chi_K_sim_i <- apply(sx_i, 1, function(y) { + c6_comp_chi_BW_i), ties.method=ties.method)[(nsim + 1L)] + + u_c7_comp_chi_K_sim_i <- apply(u_sx_i, 1, function(y) { O_k <- table(factor(y, levels=1:k)) local_chi(O_k, E_k) }) + c7_comp_chi_K_sim_i <- rep(u_c7_comp_chi_K_sim_i, times=reps) c7_comp_chi_K_sim_i_rank <- rank(c(c7_comp_chi_K_sim_i, - almost.1 * c7_comp_chi_K_i))[(nsim + 1L)] - c8_comp_ans_BW_sim_i <- sapply(c1_comp_sim_i, function(s) { + c7_comp_chi_K_i), ties.method=ties.method)[(nsim + 1L)] + + u_c8_comp_ans_BW_sim_i <- sapply(u_c1_comp_sim_i, function(s) { local_anscombe(s, c3_crdip1) }) + c8_comp_ans_BW_sim_i <- rep(u_c8_comp_ans_BW_sim_i, times=reps) c8_comp_ans_BW_sim_i_rank <- rank(c(c8_comp_ans_BW_sim_i, - almost.1 * c8_comp_ans_BW_i))[(nsim + 1L)] + c8_comp_ans_BW_i), ties.method=ties.method)[(nsim + 1L)] # create matrix of replicates for configuration x_nb_iBW <- x[nb_i] == xi @@ -225,31 +245,42 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), else sx_i_i <- cbind(sx_i_i[, 1:(i_here-1)], xi1, sx_i_i[, i_here:crdi]) - local_jcm_sim <- apply(sx_i_i, 1, function(y) { + u_sx_i_i <- unique(sx_i_i) + reps_i <- integer(nrow(u_sx_i_i)) + len_reps_i <- length(reps_i) + for (i in seq(along=reps_i)) + reps_i[i] <- sum(apply(sx_i_i, 1, function(x) all(x == u_sx_i_i[i,]))) + + u_local_jcm_sim <- apply(u_sx_i_i, 1, function(y) { local_jcm_BW(y, sn, wc) }, simplify=FALSE) - local_jcm_chi_BW_sim <- sapply(local_jcm_sim, "[[", 1) + u_local_jcm_chi_BW_sim <- sapply(u_local_jcm_sim, "[[", 1) + local_jcm_chi_BW_sim <- rep(u_local_jcm_chi_BW_sim, times=reps_i) local_jcm_chi_BW_sim_rank <- rank(c(local_jcm_chi_BW_sim, - almost.1 * local_jcm_obs_chi_BW))[(nsim + 1L)] + almost.1 * local_jcm_obs_chi_BW), ties.method=ties.method)[(nsim + 1L)] - zs <- t(sapply(local_jcm_sim, function(y) { + u_zs <- t(sapply(u_local_jcm_sim, function(y) { (y[[2]] - y[[3]]) / sqrt(y[[4]]) })) - local_jcm_z_BB_sim_rank <- rank(c(zs[,1], - almost.1 * local_jcm_obs_z_BB))[(nsim + 1L)] - local_jcm_z_WW_sim_rank <- rank(c(zs[,2], - almost.1 * local_jcm_obs_z_WW))[(nsim + 1L)] - local_jcm_z_BW_sim_rank <- rank(c(zs[,3], - almost.1 * local_jcm_obs_z_BW))[(nsim + 1L)] + zs1 <- rep(u_zs[,1], times=reps_i) + zs2 <- rep(u_zs[,2], times=reps_i) + zs3 <- rep(u_zs[,3], times=reps_i) + local_jcm_z_BB_sim_rank <- rank(c(zs1, + local_jcm_obs_z_BB), ties.method=ties.method)[(nsim + 1L)] + local_jcm_z_WW_sim_rank <- rank(c(zs2, + local_jcm_obs_z_WW), ties.method=ties.method)[(nsim + 1L)] + local_jcm_z_BW_sim_rank <- rank(c(zs3, + local_jcm_obs_z_BW), ties.method=ties.method)[(nsim + 1L)] } else { c1_comp_sim_i_rank <- c4_comp_bin_BW_sim_i_rank <- c5_comp_bin_BW_sim_i_rank <- c5a_comp_bin_BW_sim_i_rank <- c6_comp_chi_BW_sim_i_rank <- c7_comp_chi_K_sim_i_rank <- c8_comp_ans_BW_sim_i_rank <- local_jcm_chi_BW_sim_rank <- local_jcm_z_BB_sim_rank <- local_jcm_z_BW_sim_rank <- - local_jcm_z_WW_sim_rank <- NA_real_ + local_jcm_z_WW_sim_rank <- len_reps <- + len_reps_i <- NA_real_ } res_i <- c(xi, c1_comp_obs_i, unname(c2_prop_level_i), c3_crdip1, @@ -263,7 +294,8 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), c6_comp_chi_BW_sim_i_rank, c7_comp_chi_K_sim_i_rank, c8_comp_ans_BW_sim_i_rank, local_jcm_chi_BW_sim_rank, local_jcm_z_BB_sim_rank, local_jcm_z_BW_sim_rank, - local_jcm_z_WW_sim_rank, local_jcm_all_BB) + local_jcm_z_WW_sim_rank, local_jcm_all_BB, as.numeric(len_reps), + as.numeric(len_reps_i)) res_i } out <- run_perm(fun=permLICD_int, idx=1:n, env=env, iseed=iseed, @@ -332,7 +364,7 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), "rank_sim_bin_unlike_BW", "rank_sim_bin_unlike_BW_alt", "rank_sim_chi_BW", "rank_sim_chi_K", "rank_sim_anscombe_BW", "jcm_chi_sim_rank", "jcm_z_BB_sim_rank", "jcm_z_BW_sim_rank", - "jcm_z_WW_sim_rank", "local_jcm_all_BB") + "jcm_z_WW_sim_rank", "local_jcm_all_BB", "len_reps", "len_reps_i") res <- list(local_comp=local_comp, local_config=local_config, local_comp_sim=local_comp_sim, local_config_sim=local_config_sim) attr(res, "out") <- out attr(res, "ncpus") <- attr(out, "ncpus") diff --git a/man/licd_multi.Rd b/man/licd_multi.Rd index 730a92f6..ad843550 100644 --- a/man/licd_multi.Rd +++ b/man/licd_multi.Rd @@ -18,8 +18,8 @@ licd_multi(fx, listw, zero.policy = attr(listw, "zero.policy"), adjust.n = TRUE, \item{nsim}{default 0, number of conditonal permutation simulations} \item{iseed}{default NULL, used to set the seed; the output will only be reproducible if the count of CPU cores across which computation is distributed is the same} \item{no_repeat_in_row}{default \code{FALSE}, if \code{TRUE}, sample conditionally in each row without replacements to avoid duplicate values, \url{https://github.com/r-spatial/spdep/issues/124}} - \item{control}{comp_binary=TRUE, binomial_punif_alternative="greater", - jcm_same_punif_alternative="less", jcm_diff_punif_alternative="greater"} + \item{control}{comp_binary=\code{TRUE}, binomial_punif_alternative=\code{"greater"}, + jcm_same_punif_alternative=\code{"less"}, jcm_diff_punif_alternative=\code{"greater"}, rank_ties.method=\code{"min"}} } \details{The original code may be found at \doi{10.5281/zenodo.4283766}} From ddd196df7443e3fab3507919d89c759d4ea88356 Mon Sep 17 00:00:00 2001 From: Roger Date: Mon, 14 Oct 2024 13:17:59 +0200 Subject: [PATCH 02/14] tidy --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 808dfc2c..847f9da0 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Package: spdep -Version: 1.3-7 +Version: 1.3-7.1 Date: 2024-10-13 Title: Spatial Dependence: Weighting Schemes, Statistics Encoding: UTF-8 From 33453733329903a014c01d24bb243d1a3bcddf22 Mon Sep 17 00:00:00 2001 From: Roger Date: Mon, 14 Oct 2024 20:47:35 +0200 Subject: [PATCH 03/14] correct reps in licd_multi --- R/licd_boots.R | 172 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 120 insertions(+), 52 deletions(-) diff --git a/R/licd_boots.R b/R/licd_boots.R index c920cd45..6d27dc73 100644 --- a/R/licd_boots.R +++ b/R/licd_boots.R @@ -2,7 +2,7 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), adjust.n=TRUE, nsim = 0L, iseed = NULL, no_repeat_in_row=FALSE, control=list()) { con <- list(comp_binary=TRUE, binomial_punif_alternative="greater", - jcm_same_punif_alternative="less", jcm_diff_punif_alternative="greater", rank_ties.method="min") + jcm_same_punif_alternative="less", jcm_diff_punif_alternative="greater", rank_ties.method="min", unique_ceiling=1/3) nmsC <- names(con) con[(namc <- names(control))] <- control if (length(noNms <- namc[!namc %in% nmsC])) @@ -44,6 +44,7 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), assign("almost.1", almost.1, envir=env) assign("comp_binary", con$comp_binary, envir=env) assign("ties.method", con$rank_ties.method, envir=env) + assign("unique_ceiling", nsim*con$unique_ceiling, envir=env) varlist = ls(envir = env) permLICD_int <- function(i, env) { @@ -65,7 +66,7 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), lk <- get("lk", envir=env) comp_binary <- get("comp_binary", envir=env) ties.method <- get("ties.method", envir=env) - + unique_ceiling <- get("unique_ceiling", envir=env) local_jcm_BW <- function(xi, sn, wc) { xi <- factor(as.numeric(!xi)+1, levels=c(1L, 2L), labels=c("1", "2")) @@ -173,58 +174,105 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), sx_i <- cbind(rep(xi, times=nsim), sx_i) u_sx_i <- unique(sx_i) - reps <- integer(nrow(u_sx_i)) - len_reps <- length(reps) - for (i in seq(along=reps)) + len_reps <- nrow(u_sx_i) + do_reps <- FALSE + if (len_reps < unique_ceiling) do_reps <- TRUE + if (do_reps) { + reps <- integer(len_reps) + for (i in seq(along=reps)) reps[i] <- sum(apply(sx_i, 1, function(x) all(x == u_sx_i[i,]))) - - u_c1_comp_sim_i <- apply(u_sx_i, 1, - function(y) ifelse(comp_binary, sum(y == xi), - sum(w_i_i * (y == xi)) + 1)) - c1_comp_sim_i <- rep(u_c1_comp_sim_i, times=reps) + } + + if (do_reps) { + u_c1_comp_sim_i <- apply(u_sx_i, 1, + function(y) ifelse(comp_binary, sum(y == xi), + sum(w_i_i * (y == xi)) + 1)) + c1_comp_sim_i <- rep(u_c1_comp_sim_i, times=reps) + } else { + c1_comp_sim_i <- apply(sx_i, 1, + function(y) ifelse(comp_binary, sum(y == xi), + sum(w_i_i * (y == xi)) + 1)) + } c1_comp_sim_i_rank <- rank(c(c1_comp_sim_i, c1_comp_obs_i), ties.method=ties.method)[(nsim + 1L)] - u_c4_comp_bin_BW_sim_i <- sapply(u_c1_comp_sim_i, function(y) { - pbinom(y, c3_crdip1, c2_prop_level_i, lower.tail=TRUE) - }) - c4_comp_bin_BW_sim_i <- rep(u_c4_comp_bin_BW_sim_i, times=reps) + if (do_reps) { + u_c4_comp_bin_BW_sim_i <- sapply(u_c1_comp_sim_i, function(y) { + pbinom(y, c3_crdip1, c2_prop_level_i, lower.tail=TRUE) + }) + c4_comp_bin_BW_sim_i <- rep(u_c4_comp_bin_BW_sim_i, times=reps) + } else { + c4_comp_bin_BW_sim_i <- sapply(c1_comp_sim_i, function(y) { + pbinom(y, c3_crdip1, c2_prop_level_i, lower.tail=TRUE) + }) + } c4_comp_bin_BW_sim_i_rank <- rank(c(c4_comp_bin_BW_sim_i, c4_comp_bin_BW_i), ties.method=ties.method)[(nsim + 1L)] - u_c5_comp_bin_BW_sim_i <- sapply(u_c1_comp_sim_i, function(y) { - pbinom(y, c3_crdip1, c2_prop_level_i, lower.tail=FALSE) - }) - c5_comp_bin_BW_sim_i <- rep(u_c5_comp_bin_BW_sim_i, times=reps) + if (do_reps) { + u_c5_comp_bin_BW_sim_i <- sapply(u_c1_comp_sim_i, function(y) { + pbinom(y, c3_crdip1, c2_prop_level_i, lower.tail=FALSE) + }) + c5_comp_bin_BW_sim_i <- rep(u_c5_comp_bin_BW_sim_i, times=reps) + } else { + c5_comp_bin_BW_sim_i <- sapply(c1_comp_sim_i, function(y) { + pbinom(y, c3_crdip1, c2_prop_level_i, lower.tail=FALSE) + }) + } c5_comp_bin_BW_sim_i_rank <- rank(c(c5_comp_bin_BW_sim_i, c5_comp_bin_BW_i), ties.method=ties.method)[(nsim + 1L)] - u_c5a_comp_bin_BW_sim_i <- sapply(u_c1_comp_sim_i, function(y) { - pbinom(y-1, c3_crdip1, c2_prop_level_i, lower.tail=FALSE) - }) - c5a_comp_bin_BW_sim_i <- rep(u_c5a_comp_bin_BW_sim_i, times=reps) + if (do_reps) { + u_c5a_comp_bin_BW_sim_i <- sapply(u_c1_comp_sim_i, function(y) { + pbinom(y-1, c3_crdip1, c2_prop_level_i, lower.tail=FALSE) + }) + c5a_comp_bin_BW_sim_i <- rep(u_c5a_comp_bin_BW_sim_i, times=reps) + } else { + c5a_comp_bin_BW_sim_i <- sapply(c1_comp_sim_i, function(y) { + pbinom(y-1, c3_crdip1, c2_prop_level_i, lower.tail=FALSE) + }) + } c5a_comp_bin_BW_sim_i_rank <- rank(c(c5a_comp_bin_BW_sim_i, c5a_comp_bin_BW_i), ties.method=ties.method)[(nsim + 1L)] - u_c6_comp_chi_BW_sim_i <- sapply(u_c1_comp_sim_i, function(y) { - local_chi(c(y, c3_crdip1-y), E_BW) - }) - c6_comp_chi_BW_sim_i <- rep(u_c6_comp_chi_BW_sim_i, times=reps) + if (do_reps) { + u_c6_comp_chi_BW_sim_i <- sapply(u_c1_comp_sim_i, function(y) { + local_chi(c(y, c3_crdip1-y), E_BW) + }) + c6_comp_chi_BW_sim_i <- rep(u_c6_comp_chi_BW_sim_i, times=reps) + } else { + c6_comp_chi_BW_sim_i <- sapply(c1_comp_sim_i, function(y) { + local_chi(c(y, c3_crdip1-y), E_BW) + }) + } c6_comp_chi_BW_sim_i_rank <- rank(c(c6_comp_chi_BW_sim_i, c6_comp_chi_BW_i), ties.method=ties.method)[(nsim + 1L)] - u_c7_comp_chi_K_sim_i <- apply(u_sx_i, 1, function(y) { - O_k <- table(factor(y, levels=1:k)) - local_chi(O_k, E_k) - }) - c7_comp_chi_K_sim_i <- rep(u_c7_comp_chi_K_sim_i, times=reps) + if (do_reps) { + u_c7_comp_chi_K_sim_i <- apply(u_sx_i, 1, function(y) { + O_k <- table(factor(y, levels=1:k)) + local_chi(O_k, E_k) + }) + c7_comp_chi_K_sim_i <- rep(u_c7_comp_chi_K_sim_i, times=reps) + } else { + c7_comp_chi_K_sim_i <- apply(sx_i, 1, function(y) { + O_k <- table(factor(y, levels=1:k)) + local_chi(O_k, E_k) + }) + } c7_comp_chi_K_sim_i_rank <- rank(c(c7_comp_chi_K_sim_i, c7_comp_chi_K_i), ties.method=ties.method)[(nsim + 1L)] - u_c8_comp_ans_BW_sim_i <- sapply(u_c1_comp_sim_i, function(s) { - local_anscombe(s, c3_crdip1) - }) - c8_comp_ans_BW_sim_i <- rep(u_c8_comp_ans_BW_sim_i, times=reps) + if (do_reps) { + u_c8_comp_ans_BW_sim_i <- sapply(u_c1_comp_sim_i, function(s) { + local_anscombe(s, c3_crdip1) + }) + c8_comp_ans_BW_sim_i <- rep(u_c8_comp_ans_BW_sim_i, times=reps) + } else { + c8_comp_ans_BW_sim_i <- sapply(c1_comp_sim_i, function(s) { + local_anscombe(s, c3_crdip1) + }) + } c8_comp_ans_BW_sim_i_rank <- rank(c(c8_comp_ans_BW_sim_i, c8_comp_ans_BW_i), ties.method=ties.method)[(nsim + 1L)] @@ -246,27 +294,47 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), i_here:crdi]) u_sx_i_i <- unique(sx_i_i) - reps_i <- integer(nrow(u_sx_i_i)) - len_reps_i <- length(reps_i) - for (i in seq(along=reps_i)) - reps_i[i] <- sum(apply(sx_i_i, 1, function(x) all(x == u_sx_i_i[i,]))) + len_reps_i <- nrow(u_sx_i_i) + do_reps_i <- FALSE + if (len_reps_i < unique_ceiling) do_reps_i <- TRUE + if (do_reps_i) { + reps_i <- integer(len_reps_i) + len_reps_i <- length(reps_i) + for (i in seq(along=reps_i)) reps_i[i] <- sum(apply(sx_i_i, 1, + function(x) all(x == u_sx_i_i[i,]))) + } - u_local_jcm_sim <- apply(u_sx_i_i, 1, function(y) { - local_jcm_BW(y, sn, wc) - }, simplify=FALSE) - u_local_jcm_chi_BW_sim <- sapply(u_local_jcm_sim, "[[", 1) - local_jcm_chi_BW_sim <- rep(u_local_jcm_chi_BW_sim, times=reps_i) + if (do_reps_i) { + u_local_jcm_sim <- apply(u_sx_i_i, 1, function(y) { + local_jcm_BW(y, sn, wc) + }, simplify=FALSE) + u_local_jcm_chi_BW_sim <- sapply(u_local_jcm_sim, "[[", 1) + local_jcm_chi_BW_sim <- rep(u_local_jcm_chi_BW_sim, times=reps_i) + } else { + local_jcm_sim <- apply(sx_i_i, 1, function(y) { + local_jcm_BW(y, sn, wc) + }, simplify=FALSE) + local_jcm_chi_BW_sim <- sapply(local_jcm_sim, "[[", 1) + } local_jcm_chi_BW_sim_rank <- rank(c(local_jcm_chi_BW_sim, - almost.1 * local_jcm_obs_chi_BW), ties.method=ties.method)[(nsim + 1L)] + local_jcm_obs_chi_BW), ties.method=ties.method)[(nsim + 1L)] - u_zs <- t(sapply(u_local_jcm_sim, function(y) { - (y[[2]] - y[[3]]) / sqrt(y[[4]]) - })) - - zs1 <- rep(u_zs[,1], times=reps_i) - zs2 <- rep(u_zs[,2], times=reps_i) - zs3 <- rep(u_zs[,3], times=reps_i) + if (do_reps_i) { + u_zs <- t(sapply(u_local_jcm_sim, function(y) { + (y[[2]] - y[[3]]) / sqrt(y[[4]]) + })) + zs1 <- rep(u_zs[,1], times=reps_i) + zs2 <- rep(u_zs[,2], times=reps_i) + zs3 <- rep(u_zs[,3], times=reps_i) + } else { + zs <- t(sapply(local_jcm_sim, function(y) { + (y[[2]] - y[[3]]) / sqrt(y[[4]]) + })) + zs1 <- zs[,1] + zs2 <- zs[,2] + zs3 <- zs[,3] + } local_jcm_z_BB_sim_rank <- rank(c(zs1, local_jcm_obs_z_BB), ties.method=ties.method)[(nsim + 1L)] local_jcm_z_WW_sim_rank <- rank(c(zs2, From f7438545e3f3f279b606ad156568b47038464982 Mon Sep 17 00:00:00 2001 From: Roger Date: Tue, 29 Oct 2024 15:02:22 +0100 Subject: [PATCH 04/14] local rank to expose ties --- R/licd_boots.R | 212 ++++++++++++++++++++++++++++------------------ man/licd_multi.Rd | 3 +- 2 files changed, 131 insertions(+), 84 deletions(-) diff --git a/R/licd_boots.R b/R/licd_boots.R index 6d27dc73..fd4d3150 100644 --- a/R/licd_boots.R +++ b/R/licd_boots.R @@ -1,8 +1,11 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), adjust.n=TRUE, nsim = 0L, iseed = NULL, no_repeat_in_row=FALSE, control=list()) { + timings <- list() + .ptime_start <- proc.time() con <- list(comp_binary=TRUE, binomial_punif_alternative="greater", - jcm_same_punif_alternative="less", jcm_diff_punif_alternative="greater", rank_ties.method="min", unique_ceiling=1/3) + jcm_same_punif_alternative="less", jcm_diff_punif_alternative="greater", rank_ties.method="min", unique_ceiling=1/3, check_reps=FALSE, + pysal_rank=FALSE, pysal_sim_obs="GE") nmsC <- names(con) con[(namc <- names(control))] <- control if (length(noNms <- namc[!namc %in% nmsC])) @@ -28,6 +31,67 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), wt <- listw$weights almost.1 <- 1 - 64 * .Machine$double.eps + local_jcm_BW <- function(xi, sn, wc) { + xi <- factor(as.numeric(!xi)+1, levels=c(1L, 2L), + labels=c("1", "2")) + tab <- table(xi) + kBW <- length(tab) + y <- factor(paste(xi[sn[,1]], ":", xi[sn[,2]], sep=""), + levels=as.vector(outer(1:kBW, 1:kBW, + FUN=function(X,Y) paste(X,Y,sep=":")))) + res <- matrix(tapply(sn[,3], y, sum), ncol=kBW)/2 + res[is.na(res)] <- 0 + if (all(dim(res) == c(1L, 1L))) res <- rbind(cbind(res, 0), c(0, 0)) + BWlk <- c("B", "W") + ldiag <- res[2,1] + res[1,2] + ntab <- as.numeric(as.vector(tab)) + Ejc <- (wc$S0*(ntab*(ntab-1))) / (2*wc$nwcn1) + Vjc <- (wc$S1*(ntab*(ntab-1))) / (wc$nwcn1) + Vjc <- Vjc + (((wc$S2 - 2*wc$S1)*ntab*(ntab-1)*(ntab-2)) / + (wc$nwcn2)) + Vjc <- Vjc + (((wc$S02 + wc$S1 - wc$S2)*ntab*(ntab-1)*(ntab-2)* + (ntab-3)) / (wc$nwcn2*wc$n3)) + Vjc <- (0.25 * Vjc) - Ejc^2 + nrns <- ntab[2]*ntab[1] + pntab <- (ntab*(ntab-1)) + nrns1 <- pntab[2]*pntab[1] + Exp <- (wc$S0*(nrns)) / (wc$nwcn1) + Var <- (2*wc$S1*nrns)/(wc$nwcn1) + Var <- Var + (((wc$S2 - 2*wc$S1) * nrns * (ntab[2]+ntab[1]-2))/ + (wc$nwcn2)) + Var <- Var + ((4*(wc$S02 + wc$S1 - wc$S2)*nrns1) / (wc$nwcn2*wc$n3)) + Var <- (0.25 * Var) - Exp^2 + JtotExp <- sum(Exp) + O_jc_BW <- c(diag(res), ldiag) + E_jc_BW <- c(Ejc, Exp) + V_jc_BW <- c(Vjc, Var) + jc_conf_chi_BW <- local_chi(O_jc_BW, E_jc_BW) + list(jc_conf_chi_BW, O_jc_BW, E_jc_BW, V_jc_BW) + } + local_chi <- function(O, E) { + sum((O - E)^2 / E, na.rm=TRUE) + } + local_anscombe <- function(s, n) { + asin(sqrt((s+(3/8))/(n+(3/4)))) + } + + if (con$pysal_rank) { + if (con$pysal_sim_obs == "GE") { +# esda/crand.py line 328 + local_rank <- function(sims, obs, ties.method) { + sum(sims >= obs)+1 + } + } else if (con$pysal_sim_obs == "GT") { + local_rank <- function(sims, obs, ties.method) { + sum(sims > obs)+1 + } + } else stop("pysal_sim_obs given", con$pysal_sim_obs, "must be GE or GT") + } else { + local_rank <- function(sims, obs, ties.method) { + rank(c(sims, obs), ties.method=ties.method) + } + } + env <- new.env() assign("crd", card(nb), envir = env) # cardinality assign("nb", nb, envir = env) # nbs @@ -45,6 +109,11 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), assign("comp_binary", con$comp_binary, envir=env) assign("ties.method", con$rank_ties.method, envir=env) assign("unique_ceiling", nsim*con$unique_ceiling, envir=env) + assign("check_reps", con$check_reps, envir=env) + assign("local_rank", local_rank, envir=env) + assign("local_jcm_BW", local_jcm_BW, envir=env) + assign("local_chi", local_chi, envir=env) + assign("local_anscombe", local_anscombe, envir=env) varlist = ls(envir = env) permLICD_int <- function(i, env) { @@ -67,51 +136,13 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), comp_binary <- get("comp_binary", envir=env) ties.method <- get("ties.method", envir=env) unique_ceiling <- get("unique_ceiling", envir=env) - local_jcm_BW <- function(xi, sn, wc) { - xi <- factor(as.numeric(!xi)+1, levels=c(1L, 2L), - labels=c("1", "2")) - tab <- table(xi) - kBW <- length(tab) - y <- factor(paste(xi[sn[,1]], ":", xi[sn[,2]], sep=""), - levels=as.vector(outer(1:kBW, 1:kBW, - FUN=function(X,Y) paste(X,Y,sep=":")))) - res <- matrix(tapply(sn[,3], y, sum), ncol=kBW)/2 - res[is.na(res)] <- 0 - if (all(dim(res) == c(1L, 1L))) res <- rbind(cbind(res, 0), c(0, 0)) - BWlk <- c("B", "W") - ldiag <- res[2,1] + res[1,2] - ntab <- as.numeric(as.vector(tab)) - Ejc <- (wc$S0*(ntab*(ntab-1))) / (2*wc$nwcn1) - Vjc <- (wc$S1*(ntab*(ntab-1))) / (wc$nwcn1) - Vjc <- Vjc + (((wc$S2 - 2*wc$S1)*ntab*(ntab-1)*(ntab-2)) / - (wc$nwcn2)) - Vjc <- Vjc + (((wc$S02 + wc$S1 - wc$S2)*ntab*(ntab-1)*(ntab-2)* - (ntab-3)) / (wc$nwcn2*wc$n3)) - Vjc <- (0.25 * Vjc) - Ejc^2 - nrns <- ntab[2]*ntab[1] - pntab <- (ntab*(ntab-1)) - nrns1 <- pntab[2]*pntab[1] - Exp <- (wc$S0*(nrns)) / (wc$nwcn1) - Var <- (2*wc$S1*nrns)/(wc$nwcn1) - Var <- Var + (((wc$S2 - 2*wc$S1) * nrns * (ntab[2]+ntab[1]-2))/ - (wc$nwcn2)) - Var <- Var + ((4*(wc$S02 + wc$S1 - wc$S2)*nrns1) / (wc$nwcn2*wc$n3)) - Var <- (0.25 * Var) - Exp^2 - JtotExp <- sum(Exp) - O_jc_BW <- c(diag(res), ldiag) - E_jc_BW <- c(Ejc, Exp) - V_jc_BW <- c(Vjc, Var) - jc_conf_chi_BW <- local_chi(O_jc_BW, E_jc_BW) - list(jc_conf_chi_BW, O_jc_BW, E_jc_BW, V_jc_BW) - } - local_chi <- function(O, E) { - sum((O - E)^2 / E, na.rm=TRUE) - } - local_anscombe <- function(s, n) { - asin(sqrt((s+(3/8))/(n+(3/4)))) - } + check_reps <- get("check_reps", envir=env) + local_rank <- get("local_rank", envir=env) + local_jcm_BW <- get("local_jcm_BW", envir=env) + local_chi <- get("local_chi", envir=env) + local_anscombe <- get("local_anscombe", envir=env) - if (crdi == 0L) return(rep(NA_real_, 28)) + if (crdi == 0L) return(rep(NA_real_, 31)) x_nb_i <- c(xi, x[nb_i]) x_nb_i_xi <- x_nb_i == xi @@ -173,16 +204,22 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), } sx_i <- cbind(rep(xi, times=nsim), sx_i) - u_sx_i <- unique(sx_i) - len_reps <- nrow(u_sx_i) do_reps <- FALSE - if (len_reps < unique_ceiling) do_reps <- TRUE - if (do_reps) { - reps <- integer(len_reps) - for (i in seq(along=reps)) - reps[i] <- sum(apply(sx_i, 1, function(x) all(x == u_sx_i[i,]))) + if (check_reps) { + u_sx_i <- unique(sx_i) + len_reps <- nrow(u_sx_i) + if (len_reps < unique_ceiling) do_reps <- TRUE + if (do_reps) { + reps <- integer(len_reps) + for (i in seq(along=reps)) + reps[i] <- sum(apply(sx_i, 1, function(x) + all(x == u_sx_i[i,]))) + } + } else { + len_reps <- NA_real_ } + if (do_reps) { u_c1_comp_sim_i <- apply(u_sx_i, 1, function(y) ifelse(comp_binary, sum(y == xi), @@ -193,8 +230,8 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), function(y) ifelse(comp_binary, sum(y == xi), sum(w_i_i * (y == xi)) + 1)) } - c1_comp_sim_i_rank <- rank(c(c1_comp_sim_i, - c1_comp_obs_i), ties.method=ties.method)[(nsim + 1L)] + c1_comp_sim_i_rank <- local_rank(c1_comp_sim_i, + c1_comp_obs_i, ties.method=ties.method)[(nsim + 1L)] if (do_reps) { u_c4_comp_bin_BW_sim_i <- sapply(u_c1_comp_sim_i, function(y) { @@ -206,8 +243,8 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), pbinom(y, c3_crdip1, c2_prop_level_i, lower.tail=TRUE) }) } - c4_comp_bin_BW_sim_i_rank <- rank(c(c4_comp_bin_BW_sim_i, - c4_comp_bin_BW_i), ties.method=ties.method)[(nsim + 1L)] + c4_comp_bin_BW_sim_i_rank <- local_rank(c4_comp_bin_BW_sim_i, + c4_comp_bin_BW_i, ties.method=ties.method)[(nsim + 1L)] if (do_reps) { u_c5_comp_bin_BW_sim_i <- sapply(u_c1_comp_sim_i, function(y) { @@ -219,8 +256,8 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), pbinom(y, c3_crdip1, c2_prop_level_i, lower.tail=FALSE) }) } - c5_comp_bin_BW_sim_i_rank <- rank(c(c5_comp_bin_BW_sim_i, - c5_comp_bin_BW_i), ties.method=ties.method)[(nsim + 1L)] + c5_comp_bin_BW_sim_i_rank <- local_rank(c5_comp_bin_BW_sim_i, + c5_comp_bin_BW_i, ties.method=ties.method)[(nsim + 1L)] if (do_reps) { u_c5a_comp_bin_BW_sim_i <- sapply(u_c1_comp_sim_i, function(y) { @@ -232,8 +269,8 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), pbinom(y-1, c3_crdip1, c2_prop_level_i, lower.tail=FALSE) }) } - c5a_comp_bin_BW_sim_i_rank <- rank(c(c5a_comp_bin_BW_sim_i, - c5a_comp_bin_BW_i), ties.method=ties.method)[(nsim + 1L)] + c5a_comp_bin_BW_sim_i_rank <- local_rank(c5a_comp_bin_BW_sim_i, + c5a_comp_bin_BW_i, ties.method=ties.method)[(nsim + 1L)] if (do_reps) { u_c6_comp_chi_BW_sim_i <- sapply(u_c1_comp_sim_i, function(y) { @@ -245,8 +282,8 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), local_chi(c(y, c3_crdip1-y), E_BW) }) } - c6_comp_chi_BW_sim_i_rank <- rank(c(c6_comp_chi_BW_sim_i, - c6_comp_chi_BW_i), ties.method=ties.method)[(nsim + 1L)] + c6_comp_chi_BW_sim_i_rank <- local_rank(c6_comp_chi_BW_sim_i, + c6_comp_chi_BW_i, ties.method=ties.method)[(nsim + 1L)] if (do_reps) { u_c7_comp_chi_K_sim_i <- apply(u_sx_i, 1, function(y) { @@ -260,8 +297,8 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), local_chi(O_k, E_k) }) } - c7_comp_chi_K_sim_i_rank <- rank(c(c7_comp_chi_K_sim_i, - c7_comp_chi_K_i), ties.method=ties.method)[(nsim + 1L)] + c7_comp_chi_K_sim_i_rank <- local_rank(c7_comp_chi_K_sim_i, + c7_comp_chi_K_i, ties.method=ties.method)[(nsim + 1L)] if (do_reps) { u_c8_comp_ans_BW_sim_i <- sapply(u_c1_comp_sim_i, function(s) { @@ -273,8 +310,8 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), local_anscombe(s, c3_crdip1) }) } - c8_comp_ans_BW_sim_i_rank <- rank(c(c8_comp_ans_BW_sim_i, - c8_comp_ans_BW_i), ties.method=ties.method)[(nsim + 1L)] + c8_comp_ans_BW_sim_i_rank <- local_rank(c8_comp_ans_BW_sim_i, + c8_comp_ans_BW_i, ties.method=ties.method)[(nsim + 1L)] # create matrix of replicates for configuration x_nb_iBW <- x[nb_i] == xi @@ -293,15 +330,19 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), else sx_i_i <- cbind(sx_i_i[, 1:(i_here-1)], xi1, sx_i_i[, i_here:crdi]) - u_sx_i_i <- unique(sx_i_i) - len_reps_i <- nrow(u_sx_i_i) do_reps_i <- FALSE - if (len_reps_i < unique_ceiling) do_reps_i <- TRUE - if (do_reps_i) { - reps_i <- integer(len_reps_i) - len_reps_i <- length(reps_i) - for (i in seq(along=reps_i)) reps_i[i] <- sum(apply(sx_i_i, 1, - function(x) all(x == u_sx_i_i[i,]))) + if (check_reps) { + u_sx_i_i <- unique(sx_i_i) + len_reps_i <- nrow(u_sx_i_i) + if (len_reps_i < unique_ceiling) do_reps_i <- TRUE + if (do_reps_i) { + reps_i <- integer(len_reps_i) + len_reps_i <- length(reps_i) + for (i in seq(along=reps_i)) reps_i[i] <- sum(apply( + sx_i_i, 1, function(x) all(x == u_sx_i_i[i,]))) + } + } else { + len_reps_i <- NA_real_ } @@ -317,8 +358,8 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), }, simplify=FALSE) local_jcm_chi_BW_sim <- sapply(local_jcm_sim, "[[", 1) } - local_jcm_chi_BW_sim_rank <- rank(c(local_jcm_chi_BW_sim, - local_jcm_obs_chi_BW), ties.method=ties.method)[(nsim + 1L)] + local_jcm_chi_BW_sim_rank <- local_rank(local_jcm_chi_BW_sim, + local_jcm_obs_chi_BW, ties.method=ties.method)[(nsim + 1L)] if (do_reps_i) { u_zs <- t(sapply(u_local_jcm_sim, function(y) { @@ -335,12 +376,12 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), zs2 <- zs[,2] zs3 <- zs[,3] } - local_jcm_z_BB_sim_rank <- rank(c(zs1, - local_jcm_obs_z_BB), ties.method=ties.method)[(nsim + 1L)] - local_jcm_z_WW_sim_rank <- rank(c(zs2, - local_jcm_obs_z_WW), ties.method=ties.method)[(nsim + 1L)] - local_jcm_z_BW_sim_rank <- rank(c(zs3, - local_jcm_obs_z_BW), ties.method=ties.method)[(nsim + 1L)] + local_jcm_z_BB_sim_rank <- local_rank(zs1, + local_jcm_obs_z_BB, ties.method=ties.method)[(nsim + 1L)] + local_jcm_z_WW_sim_rank <- local_rank(zs2, + local_jcm_obs_z_WW, ties.method=ties.method)[(nsim + 1L)] + local_jcm_z_BW_sim_rank <- local_rank(zs3, + local_jcm_obs_z_BW, ties.method=ties.method)[(nsim + 1L)] } else { c1_comp_sim_i_rank <- c4_comp_bin_BW_sim_i_rank <- c5_comp_bin_BW_sim_i_rank <- c5a_comp_bin_BW_sim_i_rank <- @@ -366,8 +407,12 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), as.numeric(len_reps_i)) res_i } + timings[["set_up"]] <- proc.time() - .ptime_start + .ptime_start <- proc.time() out <- run_perm(fun=permLICD_int, idx=1:n, env=env, iseed=iseed, varlist=varlist) + timings[["processing"]] <- proc.time() - .ptime_start + .ptime_start <- proc.time() local_comp <- data.frame(ID=1:n, category_i=out[,1], count_like_i=out[,2], prop_i=out[,3], count_nbs_i=out[,4], pbinom_like_BW=out[,5], @@ -433,7 +478,10 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), "rank_sim_chi_BW", "rank_sim_chi_K", "rank_sim_anscombe_BW", "jcm_chi_sim_rank", "jcm_z_BB_sim_rank", "jcm_z_BW_sim_rank", "jcm_z_WW_sim_rank", "local_jcm_all_BB", "len_reps", "len_reps_i") + + timings[["postprocessing"]] <- proc.time() - .ptime_start res <- list(local_comp=local_comp, local_config=local_config, local_comp_sim=local_comp_sim, local_config_sim=local_config_sim) + attr(res, "timings") <- timings attr(res, "out") <- out attr(res, "ncpus") <- attr(out, "ncpus") attr(res, "nsim") <- nsim diff --git a/man/licd_multi.Rd b/man/licd_multi.Rd index ad843550..da8f906d 100644 --- a/man/licd_multi.Rd +++ b/man/licd_multi.Rd @@ -18,8 +18,7 @@ licd_multi(fx, listw, zero.policy = attr(listw, "zero.policy"), adjust.n = TRUE, \item{nsim}{default 0, number of conditonal permutation simulations} \item{iseed}{default NULL, used to set the seed; the output will only be reproducible if the count of CPU cores across which computation is distributed is the same} \item{no_repeat_in_row}{default \code{FALSE}, if \code{TRUE}, sample conditionally in each row without replacements to avoid duplicate values, \url{https://github.com/r-spatial/spdep/issues/124}} - \item{control}{comp_binary=\code{TRUE}, binomial_punif_alternative=\code{"greater"}, - jcm_same_punif_alternative=\code{"less"}, jcm_diff_punif_alternative=\code{"greater"}, rank_ties.method=\code{"min"}} + \item{control}{comp_binary=\code{TRUE}, binomial_punif_alternative=\code{"greater"}, jcm_same_punif_alternative=\code{"less"}, jcm_diff_punif_alternative=\code{"greater"}, rank_ties.method=\code{"min"}, check_reps=\code{FALSE}, pysal_rank=\code{FALSE}} } \details{The original code may be found at \doi{10.5281/zenodo.4283766}} From 5f3cbc19ed6289e0bef1300366bcaa1e4271d775 Mon Sep 17 00:00:00 2001 From: Roger Date: Sat, 9 Nov 2024 18:52:15 +0100 Subject: [PATCH 05/14] reverse JCM notB alt --- R/licd_boots.R | 248 +++++++++++++++++++++++++++------------------- man/hotspotmap.Rd | 2 +- man/licd_multi.Rd | 2 +- 3 files changed, 148 insertions(+), 104 deletions(-) diff --git a/R/licd_boots.R b/R/licd_boots.R index fd4d3150..18fbbdfc 100644 --- a/R/licd_boots.R +++ b/R/licd_boots.R @@ -1,11 +1,44 @@ +local_jcm_BW <- function(res, tab, wc) { + ldiag <- res[2,1] + res[1,2] + ntab <- as.numeric(as.vector(tab)) + Ejc <- (wc$S0*(ntab*(ntab-1))) / (2*wc$nwcn1) + Vjc <- (wc$S1*(ntab*(ntab-1))) / (wc$nwcn1) + Vjc <- Vjc + (((wc$S2 - 2*wc$S1)*ntab*(ntab-1)*(ntab-2)) / + (wc$nwcn2)) + Vjc <- Vjc + (((wc$S02 + wc$S1 - wc$S2)*ntab*(ntab-1)*(ntab-2)* + (ntab-3)) / (wc$nwcn2*wc$n3)) + Vjc <- (0.25 * Vjc) - Ejc^2 + nrns <- ntab[2]*ntab[1] + pntab <- (ntab*(ntab-1)) + nrns1 <- pntab[2]*pntab[1] + Exp <- (wc$S0*(nrns)) / (wc$nwcn1) + Var <- (2*wc$S1*nrns)/(wc$nwcn1) + Var <- Var + (((wc$S2 - 2*wc$S1) * nrns * (ntab[2]+ntab[1]-2))/ + (wc$nwcn2)) + Var <- Var + ((4*(wc$S02 + wc$S1 - wc$S2)*nrns1) / (wc$nwcn2*wc$n3)) + Var <- (0.25 * Var) - Exp^2 + JtotExp <- sum(Exp) + O_jc_BW <- c(diag(res), ldiag) + E_jc_BW <- c(Ejc, Exp) + V_jc_BW <- c(Vjc, Var) + jc_conf_chi_BW <- local_chi(O_jc_BW, E_jc_BW) + list(jc_conf_chi_BW, O_jc_BW, E_jc_BW, V_jc_BW) +} +local_chi <- function(O, E) { + sum((O - E)^2 / E, na.rm=TRUE) +} +local_anscombe <- function(s, n) { + asin(sqrt((s+(3/8))/(n+(3/4)))) +} + licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), adjust.n=TRUE, nsim = 0L, iseed = NULL, no_repeat_in_row=FALSE, control=list()) { timings <- list() .ptime_start <- proc.time() con <- list(comp_binary=TRUE, binomial_punif_alternative="greater", - jcm_same_punif_alternative="less", jcm_diff_punif_alternative="greater", rank_ties.method="min", unique_ceiling=1/3, check_reps=FALSE, - pysal_rank=FALSE, pysal_sim_obs="GE") + jcm_same_punif_alternative="less", jcm_diff_punif_alternative="less", rank_ties.method="min", unique_ceiling=1/3, check_reps=FALSE, + pysal_rank=FALSE, pysal_sim_obs="GE", xtras=FALSE) nmsC <- names(con) con[(namc <- names(control))] <- control if (length(noNms <- namc[!namc %in% nmsC])) @@ -31,49 +64,6 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), wt <- listw$weights almost.1 <- 1 - 64 * .Machine$double.eps - local_jcm_BW <- function(xi, sn, wc) { - xi <- factor(as.numeric(!xi)+1, levels=c(1L, 2L), - labels=c("1", "2")) - tab <- table(xi) - kBW <- length(tab) - y <- factor(paste(xi[sn[,1]], ":", xi[sn[,2]], sep=""), - levels=as.vector(outer(1:kBW, 1:kBW, - FUN=function(X,Y) paste(X,Y,sep=":")))) - res <- matrix(tapply(sn[,3], y, sum), ncol=kBW)/2 - res[is.na(res)] <- 0 - if (all(dim(res) == c(1L, 1L))) res <- rbind(cbind(res, 0), c(0, 0)) - BWlk <- c("B", "W") - ldiag <- res[2,1] + res[1,2] - ntab <- as.numeric(as.vector(tab)) - Ejc <- (wc$S0*(ntab*(ntab-1))) / (2*wc$nwcn1) - Vjc <- (wc$S1*(ntab*(ntab-1))) / (wc$nwcn1) - Vjc <- Vjc + (((wc$S2 - 2*wc$S1)*ntab*(ntab-1)*(ntab-2)) / - (wc$nwcn2)) - Vjc <- Vjc + (((wc$S02 + wc$S1 - wc$S2)*ntab*(ntab-1)*(ntab-2)* - (ntab-3)) / (wc$nwcn2*wc$n3)) - Vjc <- (0.25 * Vjc) - Ejc^2 - nrns <- ntab[2]*ntab[1] - pntab <- (ntab*(ntab-1)) - nrns1 <- pntab[2]*pntab[1] - Exp <- (wc$S0*(nrns)) / (wc$nwcn1) - Var <- (2*wc$S1*nrns)/(wc$nwcn1) - Var <- Var + (((wc$S2 - 2*wc$S1) * nrns * (ntab[2]+ntab[1]-2))/ - (wc$nwcn2)) - Var <- Var + ((4*(wc$S02 + wc$S1 - wc$S2)*nrns1) / (wc$nwcn2*wc$n3)) - Var <- (0.25 * Var) - Exp^2 - JtotExp <- sum(Exp) - O_jc_BW <- c(diag(res), ldiag) - E_jc_BW <- c(Ejc, Exp) - V_jc_BW <- c(Vjc, Var) - jc_conf_chi_BW <- local_chi(O_jc_BW, E_jc_BW) - list(jc_conf_chi_BW, O_jc_BW, E_jc_BW, V_jc_BW) - } - local_chi <- function(O, E) { - sum((O - E)^2 / E, na.rm=TRUE) - } - local_anscombe <- function(s, n) { - asin(sqrt((s+(3/8))/(n+(3/4)))) - } if (con$pysal_rank) { if (con$pysal_sim_obs == "GE") { @@ -110,14 +100,17 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), assign("ties.method", con$rank_ties.method, envir=env) assign("unique_ceiling", nsim*con$unique_ceiling, envir=env) assign("check_reps", con$check_reps, envir=env) + assign("xtras", con$xtras, envir=env) assign("local_rank", local_rank, envir=env) - assign("local_jcm_BW", local_jcm_BW, envir=env) - assign("local_chi", local_chi, envir=env) - assign("local_anscombe", local_anscombe, envir=env) +# assign("local_jcm_BW", local_jcm_BW, envir=env) +# assign("local_chi", local_chi, envir=env) +# assign("local_anscombe", local_anscombe, envir=env) varlist = ls(envir = env) permLICD_int <- function(i, env) { + timingsi <- list() + .ptime_starti <- proc.time() crdi <- get("crd", envir = env)[i] # get the ith cardinality x <- get("xi", envir = env) # get xs fx <- get("fxi", envir = env) # get xs @@ -137,10 +130,11 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), ties.method <- get("ties.method", envir=env) unique_ceiling <- get("unique_ceiling", envir=env) check_reps <- get("check_reps", envir=env) + xtras <- get("xtras", envir=env) local_rank <- get("local_rank", envir=env) - local_jcm_BW <- get("local_jcm_BW", envir=env) - local_chi <- get("local_chi", envir=env) - local_anscombe <- get("local_anscombe", envir=env) +# local_jcm_BW <- get("local_jcm_BW", envir=env) +# local_chi <- get("local_chi", envir=env) +# local_anscombe <- get("local_anscombe", envir=env) if (crdi == 0L) return(rep(NA_real_, 31)) @@ -158,17 +152,23 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), c2_prop_level_i, lower.tail=FALSE) c5a_comp_bin_BW_i <- pbinom(c1_comp_obs_i-1, c3_crdip1, c2_prop_level_i, lower.tail=FALSE) - O_BW <- c(c1_comp_obs_i, (c3_crdip1-c1_comp_obs_i)) - E_BW <- c3_crdip1 * c(c2_prop_level_i, 1-c2_prop_level_i) - c6_comp_chi_BW_i <- local_chi(O_BW, E_BW) - O_k <- table(factor(x_nb_i, levels=1:k)) - E_k <- c3_crdip1 * tifx - c7_comp_chi_K_i <- local_chi(O_k, E_k) - c8_comp_ans_BW_i <- local_anscombe(c1_comp_obs_i, c3_crdip1) + if (xtras) { + O_BW <- c(c1_comp_obs_i, (c3_crdip1-c1_comp_obs_i)) + E_BW <- c3_crdip1 * c(c2_prop_level_i, 1-c2_prop_level_i) + c6_comp_chi_BW_i <- local_chi(O_BW, E_BW) + O_k <- table(factor(x_nb_i, levels=1:k)) + E_k <- c3_crdip1 * tifx + c7_comp_chi_K_i <- local_chi(O_k, E_k) + c8_comp_ans_BW_i <- local_anscombe(c1_comp_obs_i, c3_crdip1) + } else { + c6_comp_chi_BW_i <- c7_comp_chi_K_i <- c8_comp_ans_BW_i <- NA_real_ + } + timingsi[["compi"]] <- proc.time() - .ptime_starti + .ptime_starti <- proc.time() sub_i <- sort(c(i, nb_i)) sub_i <- 1:n %in% sub_i - sub_xi <- x[sub_i] == xi + sub_xi0 <- x[sub_i] == xi i_here <- which(i %in% which(sub_i)) listw_subi <- spdep::subset.listw(listw, sub_i) sn <- spdep::listw2sn(listw_subi) @@ -177,8 +177,18 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), wc$S02 <- wc$S0*wc$S0 wc$nwcn1 <- wc$n*wc$n1 wc$nwcn2 <- wc$nwcn1*wc$n2 + sub_xi <- factor(as.numeric(!sub_xi0)+1, levels=c(1L, 2L), + labels=c("1", "2")) + tab <- table(sub_xi) + kBW <- length(tab) + y <- factor(paste(sub_xi[sn[,1]], ":", sub_xi[sn[,2]], sep=""), + levels=as.vector(outer(1:kBW, 1:kBW, + FUN=function(X,Y) paste(X,Y,sep=":")))) + res <- matrix(tapply(sn[,3], y, sum), ncol=kBW)/2 + res[is.na(res)] <- 0 + if (all(dim(res) == c(1L, 1L))) res <- rbind(cbind(res, 0), c(0, 0)) - local_jcm_obs <- local_jcm_BW(sub_xi, sn, wc) + local_jcm_obs <- local_jcm_BW(res, tab, wc) local_jcm_obs_chi_BW <- local_jcm_obs[[1]] local_jcm_obs_count_BB <- local_jcm_obs[[2]][1] local_jcm_obs_count_WW <- local_jcm_obs[[2]][2] @@ -188,7 +198,8 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), local_jcm_obs_z_BB <- zs[1] local_jcm_obs_z_WW <- zs[2] local_jcm_obs_z_BW <- zs[3] - local_jcm_all_BB <- all(sub_xi) + local_jcm_all_BB <- all(sub_xi0) + timingsi[["configi"]] <- proc.time() - .ptime_starti if (permutation) { @@ -272,46 +283,51 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), c5a_comp_bin_BW_sim_i_rank <- local_rank(c5a_comp_bin_BW_sim_i, c5a_comp_bin_BW_i, ties.method=ties.method)[(nsim + 1L)] - if (do_reps) { - u_c6_comp_chi_BW_sim_i <- sapply(u_c1_comp_sim_i, function(y) { - local_chi(c(y, c3_crdip1-y), E_BW) - }) - c6_comp_chi_BW_sim_i <- rep(u_c6_comp_chi_BW_sim_i, times=reps) - } else { - c6_comp_chi_BW_sim_i <- sapply(c1_comp_sim_i, function(y) { - local_chi(c(y, c3_crdip1-y), E_BW) - }) - } - c6_comp_chi_BW_sim_i_rank <- local_rank(c6_comp_chi_BW_sim_i, - c6_comp_chi_BW_i, ties.method=ties.method)[(nsim + 1L)] + if (xtras) { + if (do_reps) { + u_c6_comp_chi_BW_sim_i <- sapply(u_c1_comp_sim_i, function(y) { + local_chi(c(y, c3_crdip1-y), E_BW) + }) + c6_comp_chi_BW_sim_i <- rep(u_c6_comp_chi_BW_sim_i, times=reps) + } else { + c6_comp_chi_BW_sim_i <- sapply(c1_comp_sim_i, function(y) { + local_chi(c(y, c3_crdip1-y), E_BW) + }) + } + c6_comp_chi_BW_sim_i_rank <- local_rank(c6_comp_chi_BW_sim_i, + c6_comp_chi_BW_i, ties.method=ties.method)[(nsim + 1L)] - if (do_reps) { - u_c7_comp_chi_K_sim_i <- apply(u_sx_i, 1, function(y) { - O_k <- table(factor(y, levels=1:k)) - local_chi(O_k, E_k) - }) - c7_comp_chi_K_sim_i <- rep(u_c7_comp_chi_K_sim_i, times=reps) - } else { - c7_comp_chi_K_sim_i <- apply(sx_i, 1, function(y) { - O_k <- table(factor(y, levels=1:k)) - local_chi(O_k, E_k) - }) - } - c7_comp_chi_K_sim_i_rank <- local_rank(c7_comp_chi_K_sim_i, - c7_comp_chi_K_i, ties.method=ties.method)[(nsim + 1L)] + if (do_reps) { + u_c7_comp_chi_K_sim_i <- apply(u_sx_i, 1, function(y) { + O_k <- table(factor(y, levels=1:k)) + local_chi(O_k, E_k) + }) + c7_comp_chi_K_sim_i <- rep(u_c7_comp_chi_K_sim_i, times=reps) + } else { + c7_comp_chi_K_sim_i <- apply(sx_i, 1, function(y) { + O_k <- table(factor(y, levels=1:k)) + local_chi(O_k, E_k) + }) + } + c7_comp_chi_K_sim_i_rank <- local_rank(c7_comp_chi_K_sim_i, + c7_comp_chi_K_i, ties.method=ties.method)[(nsim + 1L)] - if (do_reps) { - u_c8_comp_ans_BW_sim_i <- sapply(u_c1_comp_sim_i, function(s) { - local_anscombe(s, c3_crdip1) - }) - c8_comp_ans_BW_sim_i <- rep(u_c8_comp_ans_BW_sim_i, times=reps) + if (do_reps) { + u_c8_comp_ans_BW_sim_i <- sapply(u_c1_comp_sim_i, function(s) { + local_anscombe(s, c3_crdip1) + }) + c8_comp_ans_BW_sim_i <- rep(u_c8_comp_ans_BW_sim_i, times=reps) + } else { + c8_comp_ans_BW_sim_i <- sapply(c1_comp_sim_i, function(s) { + local_anscombe(s, c3_crdip1) + }) + } + c8_comp_ans_BW_sim_i_rank <- local_rank(c8_comp_ans_BW_sim_i, + c8_comp_ans_BW_i, ties.method=ties.method)[(nsim + 1L)] } else { - c8_comp_ans_BW_sim_i <- sapply(c1_comp_sim_i, function(s) { - local_anscombe(s, c3_crdip1) - }) + c6_comp_chi_BW_sim_i_rank <- c7_comp_chi_K_sim_i_rank <- + c8_comp_ans_BW_sim_i_rank <- NA_real_ } - c8_comp_ans_BW_sim_i_rank <- local_rank(c8_comp_ans_BW_sim_i, - c8_comp_ans_BW_i, ties.method=ties.method)[(nsim + 1L)] # create matrix of replicates for configuration x_nb_iBW <- x[nb_i] == xi @@ -348,13 +364,35 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), if (do_reps_i) { u_local_jcm_sim <- apply(u_sx_i_i, 1, function(y) { - local_jcm_BW(y, sn, wc) + sub_xi <- factor(as.numeric(!y)+1, levels=c(1L, 2L), + labels=c("1", "2")) + tab <- table(sub_xi) + kBW <- length(tab) + yy <- factor(paste(sub_xi[sn[,1]], ":", sub_xi[sn[,2]], + sep=""), levels=as.vector(outer(1:kBW, 1:kBW, + FUN=function(X,Y) paste(X,Y,sep=":")))) + res <- matrix(tapply(sn[,3], yy, sum), ncol=kBW)/2 + res[is.na(res)] <- 0 + if (all(dim(res) == c(1L, 1L))) res <- rbind(cbind(res, 0), + c(0, 0)) + local_jcm_BW(res, tab, wc) }, simplify=FALSE) u_local_jcm_chi_BW_sim <- sapply(u_local_jcm_sim, "[[", 1) local_jcm_chi_BW_sim <- rep(u_local_jcm_chi_BW_sim, times=reps_i) } else { local_jcm_sim <- apply(sx_i_i, 1, function(y) { - local_jcm_BW(y, sn, wc) + sub_xi <- factor(as.numeric(!y)+1, levels=c(1L, 2L), + labels=c("1", "2")) + tab <- table(sub_xi) + kBW <- length(tab) + yy <- factor(paste(sub_xi[sn[,1]], ":", sub_xi[sn[,2]], + sep=""), levels=as.vector(outer(1:kBW, 1:kBW, + FUN=function(X,Y) paste(X,Y,sep=":")))) + res <- matrix(tapply(sn[,3], yy, sum), ncol=kBW)/2 + res[is.na(res)] <- 0 + if (all(dim(res) == c(1L, 1L))) res <- rbind(cbind(res, 0), + c(0, 0)) + local_jcm_BW(res, tab, wc) }, simplify=FALSE) local_jcm_chi_BW_sim <- sapply(local_jcm_sim, "[[", 1) } @@ -391,6 +429,9 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), local_jcm_z_WW_sim_rank <- len_reps <- len_reps_i <- NA_real_ } + tcompi <- as.numeric(timingsi[["compi"]][1]) + tconfigi <- as.numeric(timingsi[["configi"]][1]) + res_i <- c(xi, c1_comp_obs_i, unname(c2_prop_level_i), c3_crdip1, c4_comp_bin_BW_i, c5_comp_bin_BW_i, c5a_comp_bin_BW_i, @@ -404,13 +445,16 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), c8_comp_ans_BW_sim_i_rank, local_jcm_chi_BW_sim_rank, local_jcm_z_BB_sim_rank, local_jcm_z_BW_sim_rank, local_jcm_z_WW_sim_rank, local_jcm_all_BB, as.numeric(len_reps), - as.numeric(len_reps_i)) + as.numeric(len_reps_i), tcompi, tconfigi) res_i } timings[["set_up"]] <- proc.time() - .ptime_start .ptime_start <- proc.time() + inputSGO <- get.SubgraphOption() + invisible(set.SubgraphOption(FALSE)) out <- run_perm(fun=permLICD_int, idx=1:n, env=env, iseed=iseed, varlist=varlist) + invisible(set.SubgraphOption(inputSGO)) timings[["processing"]] <- proc.time() - .ptime_start .ptime_start <- proc.time() @@ -472,12 +516,12 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), "chi_K_i", "anscombe_BW", "jcm_chi_obs", "jcm_count_BB_obs", "jcm_count_BW_obs", "jcm_count_WW_obs", "local_jcm_obs_z_BB", "local_jcm_obs_z_WW", "local_jcm_obs_z_BW", - "rank_sim_count_like_i", "rank_sim_bin_like_BW", "rank_sim_bin_unlike_BW", "rank_sim_bin_unlike_BW_alt", "rank_sim_chi_BW", "rank_sim_chi_K", "rank_sim_anscombe_BW", "jcm_chi_sim_rank", "jcm_z_BB_sim_rank", "jcm_z_BW_sim_rank", - "jcm_z_WW_sim_rank", "local_jcm_all_BB", "len_reps", "len_reps_i") + "jcm_z_WW_sim_rank", "local_jcm_all_BB", "len_reps", "len_reps_i", + "tcompi", "tconfigi") timings[["postprocessing"]] <- proc.time() - .ptime_start res <- list(local_comp=local_comp, local_config=local_config, local_comp_sim=local_comp_sim, local_config_sim=local_config_sim) diff --git a/man/hotspotmap.Rd b/man/hotspotmap.Rd index 327da19b..6335b54d 100644 --- a/man/hotspotmap.Rd +++ b/man/hotspotmap.Rd @@ -34,7 +34,7 @@ hotspot(obj, ...) \item{cutoff}{Default 0.005, the probability value cutoff larger than which the observation is not found \dQuote{interesting}} -\item{p.adjust}{Default \code{"fdr"}, the \code{p.adjust()} methood used, one of \code{c("holm", "hochberg", "hommel", "bonferroni", "BH", "BY", "fdr", "none")}} +\item{p.adjust}{Default \code{"fdr"}, the \code{p.adjust()} method used, one of \code{c("holm", "hochberg", "hommel", "bonferroni", "BH", "BY", "fdr", "none")}} \item{droplevels}{Default \code{TRUE}, should empty levels of the input cluster factor be dropped} diff --git a/man/licd_multi.Rd b/man/licd_multi.Rd index da8f906d..fc64215b 100644 --- a/man/licd_multi.Rd +++ b/man/licd_multi.Rd @@ -18,7 +18,7 @@ licd_multi(fx, listw, zero.policy = attr(listw, "zero.policy"), adjust.n = TRUE, \item{nsim}{default 0, number of conditonal permutation simulations} \item{iseed}{default NULL, used to set the seed; the output will only be reproducible if the count of CPU cores across which computation is distributed is the same} \item{no_repeat_in_row}{default \code{FALSE}, if \code{TRUE}, sample conditionally in each row without replacements to avoid duplicate values, \url{https://github.com/r-spatial/spdep/issues/124}} - \item{control}{comp_binary=\code{TRUE}, binomial_punif_alternative=\code{"greater"}, jcm_same_punif_alternative=\code{"less"}, jcm_diff_punif_alternative=\code{"greater"}, rank_ties.method=\code{"min"}, check_reps=\code{FALSE}, pysal_rank=\code{FALSE}} + \item{control}{comp_binary=\code{TRUE}, binomial_punif_alternative=\code{"greater"}, jcm_same_punif_alternative=\code{"less"}, jcm_diff_punif_alternative=\code{"less"}, rank_ties.method=\code{"min"}, check_reps=\code{FALSE}, pysal_rank=\code{FALSE}, xtras=\code{FALSE}} } \details{The original code may be found at \doi{10.5281/zenodo.4283766}} From 1c007fbbf3a260a7c72acd1eb7e0aaf3efa1f0a9 Mon Sep 17 00:00:00 2001 From: Roger Bivand Date: Mon, 11 Nov 2024 09:09:37 +0100 Subject: [PATCH 06/14] tidy --- R/licd_boots.R | 32 ++++++++++++++++---------------- man/licd_multi.Rd | 4 ++-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/R/licd_boots.R b/R/licd_boots.R index 18fbbdfc..1959fde5 100644 --- a/R/licd_boots.R +++ b/R/licd_boots.R @@ -37,7 +37,7 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), timings <- list() .ptime_start <- proc.time() con <- list(comp_binary=TRUE, binomial_punif_alternative="greater", - jcm_same_punif_alternative="less", jcm_diff_punif_alternative="less", rank_ties.method="min", unique_ceiling=1/3, check_reps=FALSE, + jcm_same_punif_alternative="less", jcm_diff_punif_alternative="greater", rank_ties.method="min", unique_ceiling=1/3, check_reps=FALSE, pysal_rank=FALSE, pysal_sim_obs="GE", xtras=FALSE) nmsC <- names(con) con[(namc <- names(control))] <- control @@ -493,22 +493,22 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), alternative=con$jcm_same_punif_alternative) pr_jcmnsim1 <- probs_lut("jcm_diff", nsim, alternative=con$jcm_diff_punif_alternative) - pval_jcm_obs_BB_sim <- pr_jcmnsim[out[,26]] - pval_jcm_obs_BW_sim <- pr_jcmnsim[out[,27]] - pval_jcm_obs_WW_sim <- pr_jcmnsim1[out[,28]] + pval_jcm_obs_BB <- pr_jcmnsim[out[,26]] + pval_jcm_obs_BW <- pr_jcmnsim[out[,27]] + pval_jcm_obs_WW <- pr_jcmnsim1[out[,28]] if (any(sameB)) { - pval_jcm_obs_BB_sim[sameB] <- 0 - pval_jcm_obs_WW_sim[sameB] <- 1 - pval_jcm_obs_BW_sim[sameB] <- 1 + pval_jcm_obs_BB[sameB] <- 0 + pval_jcm_obs_WW[sameB] <- 1 + pval_jcm_obs_BW[sameB] <- 1 } - pval_jcm_obs_BB_sim[is.nan(pval_jcm_obs_BB)] <- 1 - pval_jcm_obs_WW_sim[is.nan(pval_jcm_obs_WW)] <- 1 - pval_jcm_obs_BW_sim[is.nan(pval_jcm_obs_BW)] <- 1 + pval_jcm_obs_BB[is.nan(pval_jcm_obs_BB)] <- 1 + pval_jcm_obs_WW[is.nan(pval_jcm_obs_WW)] <- 1 + pval_jcm_obs_BW[is.nan(pval_jcm_obs_BW)] <- 1 local_config_sim <- data.frame(ID=1:n, jcm_chi_sim_rank=out[,25], - pval_jcm_obs_BB_sim=pval_jcm_obs_BB_sim, - pval_jcm_obs_BW_sim=pval_jcm_obs_BW_sim, - pval_jcm_obs_WW_sim=pval_jcm_obs_WW_sim) + pval_jcm_obs_BB=pval_jcm_obs_BB, + pval_jcm_obs_BW=pval_jcm_obs_BW, + pval_jcm_obs_WW=pval_jcm_obs_WW) } colnames(out) <- c("category_i", "count_like_i", "prop_i", "count_nbs_i", @@ -591,9 +591,9 @@ hotspot.licd <- function(obj, type="both", cutoff=0.05, p.adjust="none", local_config <- factor(local_config) local_config_sim <- NULL if (attr(obj, "nsim") > 0) { - BB <- obj$local_config_sim$pval_jcm_obs_BB_sim - WW <- obj$local_config_sim$pval_jcm_obs_WW_sim - BW <- obj$local_config_sim$pval_jcm_obs_BW_sim + BB <- obj$local_config_sim$pval_jcm_obs_BB + WW <- obj$local_config_sim$pval_jcm_obs_WW + BW <- obj$local_config_sim$pval_jcm_obs_BW BB <- p.adjust(BB, p.adjust) WW <- p.adjust(WW, p.adjust) BW <- p.adjust(BW, p.adjust) diff --git a/man/licd_multi.Rd b/man/licd_multi.Rd index fc64215b..6f1e9980 100644 --- a/man/licd_multi.Rd +++ b/man/licd_multi.Rd @@ -18,7 +18,7 @@ licd_multi(fx, listw, zero.policy = attr(listw, "zero.policy"), adjust.n = TRUE, \item{nsim}{default 0, number of conditonal permutation simulations} \item{iseed}{default NULL, used to set the seed; the output will only be reproducible if the count of CPU cores across which computation is distributed is the same} \item{no_repeat_in_row}{default \code{FALSE}, if \code{TRUE}, sample conditionally in each row without replacements to avoid duplicate values, \url{https://github.com/r-spatial/spdep/issues/124}} - \item{control}{comp_binary=\code{TRUE}, binomial_punif_alternative=\code{"greater"}, jcm_same_punif_alternative=\code{"less"}, jcm_diff_punif_alternative=\code{"less"}, rank_ties.method=\code{"min"}, check_reps=\code{FALSE}, pysal_rank=\code{FALSE}, xtras=\code{FALSE}} + \item{control}{comp_binary=\code{TRUE}, binomial_punif_alternative=\code{"greater"}, jcm_same_punif_alternative=\code{"less"}, jcm_diff_punif_alternative=\code{"greater"}, rank_ties.method=\code{"min"}, check_reps=\code{FALSE}, pysal_rank=\code{FALSE}, xtras=\code{FALSE}} } \details{The original code may be found at \doi{10.5281/zenodo.4283766}} @@ -28,7 +28,7 @@ licd_multi(fx, listw, zero.policy = attr(listw, "zero.policy"), adjust.n = TRUE, pbinom_unlike_BW, pbinom_unlike_BW_alt, chi_BW_i, chi_K_i, anscombe_BW} \item{local_config}{data.frame object with LICD local configuration columns: ID, jcm_chi_obs, jcm_count_BB_obs, jcm_count_BW_obs, jcm_count_WW_obs, pval_jcm_obs_BB, pval_jcm_obs_WW, pval_jcm_obs_BW} \item{local_comp_sim}{data.frame object with permutation-based LICD local composition columns: ID, pbinom_like_BW, pbinom_unlike_BW, pbinom_unlike_BW_alt, rank_sim_chi_BW, rank_sim_chi_K, rank_sim_anscombe} - \item{local_config_sim}{data.frame object with permutation-based LICD local configuration columns: ID, jcm_chi_sim_rank, pval_jcm_obs_BB_sim, pval_jcm_obs_BW_sim, pval_jcm_obs_WW_sim} + \item{local_config_sim}{data.frame object with permutation-based LICD local configuration columns: ID, jcm_chi_sim_rank, pval_jcm_obs_BB, pval_jcm_obs_BW, pval_jcm_obs_WW} } \references{ Cliff, A. D., Ord, J. K. 1981 Spatial processes, Pion, p. 20; From 01a8b0df3ee877f8ccaca7a16ffe71449c5d9854 Mon Sep 17 00:00:00 2001 From: Roger Date: Tue, 12 Nov 2024 09:05:03 +0100 Subject: [PATCH 07/14] tidy --- R/licd_boots.R | 6 +++--- man/licd_multi.Rd | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/R/licd_boots.R b/R/licd_boots.R index 1959fde5..4a986421 100644 --- a/R/licd_boots.R +++ b/R/licd_boots.R @@ -37,7 +37,7 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), timings <- list() .ptime_start <- proc.time() con <- list(comp_binary=TRUE, binomial_punif_alternative="greater", - jcm_same_punif_alternative="less", jcm_diff_punif_alternative="greater", rank_ties.method="min", unique_ceiling=1/3, check_reps=FALSE, + jcm_same_punif_alternative="less", jcm_diff_punif_alternative="less", rank_ties.method="min", unique_ceiling=1/3, check_reps=FALSE, pysal_rank=FALSE, pysal_sim_obs="GE", xtras=FALSE) nmsC <- names(con) con[(namc <- names(control))] <- control @@ -494,8 +494,8 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), pr_jcmnsim1 <- probs_lut("jcm_diff", nsim, alternative=con$jcm_diff_punif_alternative) pval_jcm_obs_BB <- pr_jcmnsim[out[,26]] - pval_jcm_obs_BW <- pr_jcmnsim[out[,27]] - pval_jcm_obs_WW <- pr_jcmnsim1[out[,28]] + pval_jcm_obs_BW <- pr_jcmnsim1[out[,27]] + pval_jcm_obs_WW <- pr_jcmnsim[out[,28]] if (any(sameB)) { pval_jcm_obs_BB[sameB] <- 0 pval_jcm_obs_WW[sameB] <- 1 diff --git a/man/licd_multi.Rd b/man/licd_multi.Rd index 6f1e9980..1a452151 100644 --- a/man/licd_multi.Rd +++ b/man/licd_multi.Rd @@ -18,7 +18,7 @@ licd_multi(fx, listw, zero.policy = attr(listw, "zero.policy"), adjust.n = TRUE, \item{nsim}{default 0, number of conditonal permutation simulations} \item{iseed}{default NULL, used to set the seed; the output will only be reproducible if the count of CPU cores across which computation is distributed is the same} \item{no_repeat_in_row}{default \code{FALSE}, if \code{TRUE}, sample conditionally in each row without replacements to avoid duplicate values, \url{https://github.com/r-spatial/spdep/issues/124}} - \item{control}{comp_binary=\code{TRUE}, binomial_punif_alternative=\code{"greater"}, jcm_same_punif_alternative=\code{"less"}, jcm_diff_punif_alternative=\code{"greater"}, rank_ties.method=\code{"min"}, check_reps=\code{FALSE}, pysal_rank=\code{FALSE}, xtras=\code{FALSE}} + \item{control}{comp_binary=\code{TRUE}, binomial_punif_alternative=\code{"greater"}, jcm_same_punif_alternative=\code{"less"}, jcm_diff_punif_alternative=\code{"less"}, rank_ties.method=\code{"min"}, check_reps=\code{FALSE}, pysal_rank=\code{FALSE}, xtras=\code{FALSE}} } \details{The original code may be found at \doi{10.5281/zenodo.4283766}} From 2f9b4067046a78f311849d61fa86c9defcd4cb2c Mon Sep 17 00:00:00 2001 From: Roger Bivand Date: Tue, 12 Nov 2024 19:09:11 +0100 Subject: [PATCH 08/14] tidy --- R/licd_boots.R | 4 ++-- man/licd_multi.Rd | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/R/licd_boots.R b/R/licd_boots.R index 4a986421..fb6890b9 100644 --- a/R/licd_boots.R +++ b/R/licd_boots.R @@ -37,7 +37,7 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), timings <- list() .ptime_start <- proc.time() con <- list(comp_binary=TRUE, binomial_punif_alternative="greater", - jcm_same_punif_alternative="less", jcm_diff_punif_alternative="less", rank_ties.method="min", unique_ceiling=1/3, check_reps=FALSE, + jcm_same_punif_alternative="less", jcm_diff_punif_alternative="greater", rank_ties.method="min", unique_ceiling=1/3, check_reps=FALSE, pysal_rank=FALSE, pysal_sim_obs="GE", xtras=FALSE) nmsC <- names(con) con[(namc <- names(control))] <- control @@ -491,7 +491,7 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), rank_sim_anscombe_BW=out[,24]) pr_jcmnsim <- probs_lut("jcm_same", nsim, alternative=con$jcm_same_punif_alternative) - pr_jcmnsim1 <- probs_lut("jcm_diff", nsim, + pr_jcmnsim1 <- probs_lut("jcm_same", nsim, alternative=con$jcm_diff_punif_alternative) pval_jcm_obs_BB <- pr_jcmnsim[out[,26]] pval_jcm_obs_BW <- pr_jcmnsim1[out[,27]] diff --git a/man/licd_multi.Rd b/man/licd_multi.Rd index 1a452151..6f1e9980 100644 --- a/man/licd_multi.Rd +++ b/man/licd_multi.Rd @@ -18,7 +18,7 @@ licd_multi(fx, listw, zero.policy = attr(listw, "zero.policy"), adjust.n = TRUE, \item{nsim}{default 0, number of conditonal permutation simulations} \item{iseed}{default NULL, used to set the seed; the output will only be reproducible if the count of CPU cores across which computation is distributed is the same} \item{no_repeat_in_row}{default \code{FALSE}, if \code{TRUE}, sample conditionally in each row without replacements to avoid duplicate values, \url{https://github.com/r-spatial/spdep/issues/124}} - \item{control}{comp_binary=\code{TRUE}, binomial_punif_alternative=\code{"greater"}, jcm_same_punif_alternative=\code{"less"}, jcm_diff_punif_alternative=\code{"less"}, rank_ties.method=\code{"min"}, check_reps=\code{FALSE}, pysal_rank=\code{FALSE}, xtras=\code{FALSE}} + \item{control}{comp_binary=\code{TRUE}, binomial_punif_alternative=\code{"greater"}, jcm_same_punif_alternative=\code{"less"}, jcm_diff_punif_alternative=\code{"greater"}, rank_ties.method=\code{"min"}, check_reps=\code{FALSE}, pysal_rank=\code{FALSE}, xtras=\code{FALSE}} } \details{The original code may be found at \doi{10.5281/zenodo.4283766}} From 5b375ca4ad0aebee6ea7356e6723e91f42fbab3a Mon Sep 17 00:00:00 2001 From: Roger Date: Mon, 18 Nov 2024 13:59:05 +0100 Subject: [PATCH 09/14] local jcm NA handling --- R/licd_boots.R | 93 ++++++++++++++++++++++++++++------------------- man/licd_multi.Rd | 2 +- 2 files changed, 56 insertions(+), 39 deletions(-) diff --git a/R/licd_boots.R b/R/licd_boots.R index fb6890b9..b9252acc 100644 --- a/R/licd_boots.R +++ b/R/licd_boots.R @@ -38,7 +38,8 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), .ptime_start <- proc.time() con <- list(comp_binary=TRUE, binomial_punif_alternative="greater", jcm_same_punif_alternative="less", jcm_diff_punif_alternative="greater", rank_ties.method="min", unique_ceiling=1/3, check_reps=FALSE, - pysal_rank=FALSE, pysal_sim_obs="GE", xtras=FALSE) + pysal_rank=FALSE, pysal_sim_obs="GE", xtras=FALSE, + set_NA_jcm_probs=TRUE, set_allB_jcm_probs=TRUE) nmsC <- names(con) con[(namc <- names(control))] <- control if (length(noNms <- namc[!namc %in% nmsC])) @@ -68,17 +69,23 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), if (con$pysal_rank) { if (con$pysal_sim_obs == "GE") { # esda/crand.py line 328 - local_rank <- function(sims, obs, ties.method) { - sum(sims >= obs)+1 + local_rank <- function(sims, obs, ties.method, nsim) { + if (is.finite(obs)) ret <- sum(sims >= obs)+1 + else ret <- NA_real_ + ret } } else if (con$pysal_sim_obs == "GT") { - local_rank <- function(sims, obs, ties.method) { - sum(sims > obs)+1 - } + local_rank <- function(sims, obs, ties.method, nsim) { + if (is.finite(obs)) ret <- sum(sims > obs)+1 + else ret <- NA_real_ + ret } } else stop("pysal_sim_obs given", con$pysal_sim_obs, "must be GE or GT") } else { - local_rank <- function(sims, obs, ties.method) { - rank(c(sims, obs), ties.method=ties.method) + local_rank <- function(sims, obs, ties.method, nsim) { + if (is.finite(obs)) ret <- as.numeric(rank(c(sims, obs), + ties.method=ties.method)[(nsim + 1L)]) + else ret <- NA_real_ + ret } } @@ -242,7 +249,7 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), sum(w_i_i * (y == xi)) + 1)) } c1_comp_sim_i_rank <- local_rank(c1_comp_sim_i, - c1_comp_obs_i, ties.method=ties.method)[(nsim + 1L)] + c1_comp_obs_i, ties.method=ties.method, nsim=nsim) if (do_reps) { u_c4_comp_bin_BW_sim_i <- sapply(u_c1_comp_sim_i, function(y) { @@ -255,7 +262,7 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), }) } c4_comp_bin_BW_sim_i_rank <- local_rank(c4_comp_bin_BW_sim_i, - c4_comp_bin_BW_i, ties.method=ties.method)[(nsim + 1L)] + c4_comp_bin_BW_i, ties.method=ties.method, nsim=nsim) if (do_reps) { u_c5_comp_bin_BW_sim_i <- sapply(u_c1_comp_sim_i, function(y) { @@ -268,7 +275,7 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), }) } c5_comp_bin_BW_sim_i_rank <- local_rank(c5_comp_bin_BW_sim_i, - c5_comp_bin_BW_i, ties.method=ties.method)[(nsim + 1L)] + c5_comp_bin_BW_i, ties.method=ties.method, nsim=nsim) if (do_reps) { u_c5a_comp_bin_BW_sim_i <- sapply(u_c1_comp_sim_i, function(y) { @@ -281,7 +288,7 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), }) } c5a_comp_bin_BW_sim_i_rank <- local_rank(c5a_comp_bin_BW_sim_i, - c5a_comp_bin_BW_i, ties.method=ties.method)[(nsim + 1L)] + c5a_comp_bin_BW_i, ties.method=ties.method, nsim=nsim) if (xtras) { if (do_reps) { @@ -295,7 +302,7 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), }) } c6_comp_chi_BW_sim_i_rank <- local_rank(c6_comp_chi_BW_sim_i, - c6_comp_chi_BW_i, ties.method=ties.method)[(nsim + 1L)] + c6_comp_chi_BW_i, ties.method=ties.method, nsim=nsim) if (do_reps) { u_c7_comp_chi_K_sim_i <- apply(u_sx_i, 1, function(y) { @@ -310,7 +317,7 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), }) } c7_comp_chi_K_sim_i_rank <- local_rank(c7_comp_chi_K_sim_i, - c7_comp_chi_K_i, ties.method=ties.method)[(nsim + 1L)] + c7_comp_chi_K_i, ties.method=ties.method, nsim=nsim) if (do_reps) { u_c8_comp_ans_BW_sim_i <- sapply(u_c1_comp_sim_i, function(s) { @@ -323,7 +330,7 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), }) } c8_comp_ans_BW_sim_i_rank <- local_rank(c8_comp_ans_BW_sim_i, - c8_comp_ans_BW_i, ties.method=ties.method)[(nsim + 1L)] + c8_comp_ans_BW_i, ties.method=ties.method, nsim=nsim) } else { c6_comp_chi_BW_sim_i_rank <- c7_comp_chi_K_sim_i_rank <- c8_comp_ans_BW_sim_i_rank <- NA_real_ @@ -397,7 +404,7 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), local_jcm_chi_BW_sim <- sapply(local_jcm_sim, "[[", 1) } local_jcm_chi_BW_sim_rank <- local_rank(local_jcm_chi_BW_sim, - local_jcm_obs_chi_BW, ties.method=ties.method)[(nsim + 1L)] + local_jcm_obs_chi_BW, ties.method=ties.method, nsim=nsim) if (do_reps_i) { u_zs <- t(sapply(u_local_jcm_sim, function(y) { @@ -415,11 +422,11 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), zs3 <- zs[,3] } local_jcm_z_BB_sim_rank <- local_rank(zs1, - local_jcm_obs_z_BB, ties.method=ties.method)[(nsim + 1L)] + local_jcm_obs_z_BB, ties.method=ties.method, nsim=nsim) local_jcm_z_WW_sim_rank <- local_rank(zs2, - local_jcm_obs_z_WW, ties.method=ties.method)[(nsim + 1L)] + local_jcm_obs_z_WW, ties.method=ties.method, nsim=nsim) local_jcm_z_BW_sim_rank <- local_rank(zs3, - local_jcm_obs_z_BW, ties.method=ties.method)[(nsim + 1L)] + local_jcm_obs_z_BW, ties.method=ties.method, nsim=nsim) } else { c1_comp_sim_i_rank <- c4_comp_bin_BW_sim_i_rank <- c5_comp_bin_BW_sim_i_rank <- c5a_comp_bin_BW_sim_i_rank <- @@ -466,15 +473,19 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), pval_jcm_obs_BB <- pnorm(out[,15], lower.tail=FALSE) pval_jcm_obs_WW <- pnorm(out[,16], lower.tail=FALSE) pval_jcm_obs_BW <- pnorm(out[,17], lower.tail=TRUE) - sameB <- as.logical(out[,29]) - if (any(sameB)) { - pval_jcm_obs_BB[sameB] <- 0 - pval_jcm_obs_WW[sameB] <- 1 - pval_jcm_obs_BW[sameB] <- 1 + if (con$set_allB_jcm_probs) { + sameB <- as.logical(out[,29]) + if (any(sameB)) { + pval_jcm_obs_BB[sameB] <- 0 + pval_jcm_obs_WW[sameB] <- 1 + pval_jcm_obs_BW[sameB] <- 1 + } + } + if (con$set_NA_jcm_probs) { + pval_jcm_obs_BB[is.na(pval_jcm_obs_BB)] <- 1 + pval_jcm_obs_WW[is.na(pval_jcm_obs_WW)] <- 1 + pval_jcm_obs_BW[is.na(pval_jcm_obs_BW)] <- 1 } - pval_jcm_obs_BB[is.nan(pval_jcm_obs_BB)] <- 1 - pval_jcm_obs_WW[is.nan(pval_jcm_obs_WW)] <- 1 - pval_jcm_obs_BW[is.nan(pval_jcm_obs_BW)] <- 1 local_config <- data.frame(ID=1:n, jcm_chi_obs=out[,11], jcm_count_BB_obs=out[,12], jcm_count_BW_obs=out[,13], jcm_count_WW_obs=out[,14], pval_jcm_obs_BB=pval_jcm_obs_BB, @@ -493,18 +504,24 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), alternative=con$jcm_same_punif_alternative) pr_jcmnsim1 <- probs_lut("jcm_same", nsim, alternative=con$jcm_diff_punif_alternative) - pval_jcm_obs_BB <- pr_jcmnsim[out[,26]] - pval_jcm_obs_BW <- pr_jcmnsim1[out[,27]] - pval_jcm_obs_WW <- pr_jcmnsim[out[,28]] - if (any(sameB)) { - pval_jcm_obs_BB[sameB] <- 0 - pval_jcm_obs_WW[sameB] <- 1 - pval_jcm_obs_BW[sameB] <- 1 + rnk <- out[,26] + pval_jcm_obs_BB <- ifelse(!is.finite(rnk), pr_jcmnsim[rnk], NA_real_) + rnk <- out[,27] + pval_jcm_obs_BW <- ifelse(!is.finite(rnk), pr_jcmnsim1[rnk], NA_real_) + rnk <- out[,28] + pval_jcm_obs_WW <- ifelse(!is.finite(rnk), pr_jcmnsim[rnk], NA_real_) + if (con$set_allB_jcm_probs) { + if (any(sameB)) { + pval_jcm_obs_BB[sameB] <- 0 + pval_jcm_obs_WW[sameB] <- 1 + pval_jcm_obs_BW[sameB] <- 1 + } + } + if (con$set_NA_jcm_probs) { + pval_jcm_obs_BB[is.nan(pval_jcm_obs_BB)] <- 1 + pval_jcm_obs_WW[is.nan(pval_jcm_obs_WW)] <- 1 + pval_jcm_obs_BW[is.nan(pval_jcm_obs_BW)] <- 1 } - pval_jcm_obs_BB[is.nan(pval_jcm_obs_BB)] <- 1 - pval_jcm_obs_WW[is.nan(pval_jcm_obs_WW)] <- 1 - pval_jcm_obs_BW[is.nan(pval_jcm_obs_BW)] <- 1 - local_config_sim <- data.frame(ID=1:n, jcm_chi_sim_rank=out[,25], pval_jcm_obs_BB=pval_jcm_obs_BB, pval_jcm_obs_BW=pval_jcm_obs_BW, diff --git a/man/licd_multi.Rd b/man/licd_multi.Rd index 6f1e9980..53aa4187 100644 --- a/man/licd_multi.Rd +++ b/man/licd_multi.Rd @@ -18,7 +18,7 @@ licd_multi(fx, listw, zero.policy = attr(listw, "zero.policy"), adjust.n = TRUE, \item{nsim}{default 0, number of conditonal permutation simulations} \item{iseed}{default NULL, used to set the seed; the output will only be reproducible if the count of CPU cores across which computation is distributed is the same} \item{no_repeat_in_row}{default \code{FALSE}, if \code{TRUE}, sample conditionally in each row without replacements to avoid duplicate values, \url{https://github.com/r-spatial/spdep/issues/124}} - \item{control}{comp_binary=\code{TRUE}, binomial_punif_alternative=\code{"greater"}, jcm_same_punif_alternative=\code{"less"}, jcm_diff_punif_alternative=\code{"greater"}, rank_ties.method=\code{"min"}, check_reps=\code{FALSE}, pysal_rank=\code{FALSE}, xtras=\code{FALSE}} + \item{control}{comp_binary=\code{TRUE}, binomial_punif_alternative=\code{"greater"}, jcm_same_punif_alternative=\code{"less"}, jcm_diff_punif_alternative=\code{"greater"}, rank_ties.method=\code{"min"}, check_reps=\code{FALSE}, pysal_rank=\code{FALSE}, xtras=\code{FALSE}, set_NA_jcm_probs=\code{TRUE}, set_allB_jcm_probs=\code{TRUE}} } \details{The original code may be found at \doi{10.5281/zenodo.4283766}} From 624ee4c37fdb8d61f0a25c9cf55cafcec237e9cc Mon Sep 17 00:00:00 2001 From: Roger Bivand Date: Tue, 19 Nov 2024 13:08:23 +0100 Subject: [PATCH 10/14] tidy --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index ac0fe9be..efaeda0a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,6 @@ # Version 1.3-6 (development) -* adding vignette desribing recent changes in `poly2nb` from #162, subgraph and no-neighbour (island) handling +* adding vignette describing recent changes in `poly2nb` from #162, subgraph and no-neighbour (island) handling * adding prototype of LICD ESDA function `licd_multi` and `hotspot` method From 8eb99e6588b699fcf4bd49e1110fe6908faf487e Mon Sep 17 00:00:00 2001 From: Roger Bivand Date: Thu, 21 Nov 2024 18:10:08 +0100 Subject: [PATCH 11/14] change simulation ranking defaults --- R/licd_boots.R | 121 ++++++++++++++++++++++++++++------------------ man/licd_multi.Rd | 2 +- 2 files changed, 76 insertions(+), 47 deletions(-) diff --git a/R/licd_boots.R b/R/licd_boots.R index b9252acc..d52b3222 100644 --- a/R/licd_boots.R +++ b/R/licd_boots.R @@ -38,13 +38,28 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), .ptime_start <- proc.time() con <- list(comp_binary=TRUE, binomial_punif_alternative="greater", jcm_same_punif_alternative="less", jcm_diff_punif_alternative="greater", rank_ties.method="min", unique_ceiling=1/3, check_reps=FALSE, - pysal_rank=FALSE, pysal_sim_obs="GE", xtras=FALSE, - set_NA_jcm_probs=TRUE, set_allB_jcm_probs=TRUE) + pysal_rank=FALSE, pysal_sim_obs="GT", na.last="keep", xtras=FALSE) nmsC <- names(con) con[(namc <- names(control))] <- control if (length(noNms <- namc[!namc %in% nmsC])) warning("unknown names in control: ", paste(noNms, collapse = ", ")) - if(!inherits(listw, "listw")) stop(paste(deparse(substitute(listw)), + if (length(con$rank_ties.method) != 1L) stop("control rank_ties.method invalid") + if (!is.character(con$rank_ties.method)) + stop("control rank_ties.method invalid") + if (is.na(match(con$rank_ties.method, + c("average", "first", "last", "random", "max", "min")))) { + stop("control rank_ties.method string invalid: ", con$rank_ties.method) + } + if (length(con$na.last) != 1L) stop("control na.last invalid") + if (!is.na(con$na.last)) { + if (is.character(con$na.last)) { + if (is.na(match(con$na.last, "keep"))) + stop("control na.last string invalid: ", con$na.last) + } else if (!is.logical(con$na.last)) { + stop("control na.last invalid") + } + } + if(!inherits(listw, "listw")) stop(paste(deparse(substitute(listw)), "is not a listw object")) if(!is.factor(fx)) stop(paste(deparse(substitute(fx)), "is not a factor")) @@ -68,25 +83,44 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), if (con$pysal_rank) { if (con$pysal_sim_obs == "GE") { -# esda/crand.py line 328 +# esda/crand.py line 328 modified to handle sampling exceptions +# now matches ties.method="min", na.last="keep" local_rank <- function(sims, obs, ties.method, nsim) { - if (is.finite(obs)) ret <- sum(sims >= obs)+1 - else ret <- NA_real_ + if (!is.finite(obs)) ret <- NA_real_ + else ret <- sum(sims <= obs, na.rm=TRUE)+1 ret } } else if (con$pysal_sim_obs == "GT") { local_rank <- function(sims, obs, ties.method, nsim) { - if (is.finite(obs)) ret <- sum(sims > obs)+1 - else ret <- NA_real_ - ret } + if (!is.finite(obs)) ret <- NA_real_ + else ret <- sum(sims < obs, na.rm=TRUE)+1 + ret + } } else stop("pysal_sim_obs given", con$pysal_sim_obs, "must be GE or GT") } else { - local_rank <- function(sims, obs, ties.method, nsim) { - if (is.finite(obs)) ret <- as.numeric(rank(c(sims, obs), - ties.method=ties.method)[(nsim + 1L)]) - else ret <- NA_real_ - ret - } + if (!is.na(con$na.last) && is.logical(con$na.last) && con$na.last) { + local_rank <- function(sims, obs, ties.method, nsim) { + as.numeric(rank(c(sims, obs), + na.last=TRUE, ties.method=ties.method)[(nsim + 1L)]) + } + } else if (!is.na(con$na.last) && is.logical(con$na.last) && + !con$na.last) { + local_rank <- function(sims, obs, ties.method, nsim) { + as.numeric(rank(c(sims, obs), + na.last=FALSE, ties.method=ties.method)[(nsim + 1L)]) + } + } else if (is.na(con$na.last)) { + local_rank <- function(sims, obs, ties.method, nsim) { + as.numeric(rank(c(sims, obs), + na.last=NA, ties.method=ties.method)[(nsim + 1L)]) + } + } else if (!is.na(con$na.last) && is.character(con$na.last) && + con$na.last == "keep") { + local_rank <- function(sims, obs, ties.method, nsim) { + as.numeric(rank(c(sims, obs), + na.last="keep", ties.method=ties.method)[(nsim + 1L)]) + } + } else stop('na.last given', con$na.last, 'must be valid') } env <- new.env() @@ -200,8 +234,10 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), local_jcm_obs_count_BB <- local_jcm_obs[[2]][1] local_jcm_obs_count_WW <- local_jcm_obs[[2]][2] local_jcm_obs_count_BW <- local_jcm_obs[[2]][3] - zs <- (local_jcm_obs[[2]] - local_jcm_obs[[3]]) / - sqrt(local_jcm_obs[[4]]) + vs <- local_jcm_obs[[4]] + vs_non_pos <- any(vs <= 0) + Oi_Ei_zero <- all(local_jcm_obs[[2]]-local_jcm_obs[[3]] == 0) + zs <- (local_jcm_obs[[2]] - local_jcm_obs[[3]]) / sqrt(vs) local_jcm_obs_z_BB <- zs[1] local_jcm_obs_z_WW <- zs[2] local_jcm_obs_z_BW <- zs[3] @@ -434,7 +470,7 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), c8_comp_ans_BW_sim_i_rank <- local_jcm_chi_BW_sim_rank <- local_jcm_z_BB_sim_rank <- local_jcm_z_BW_sim_rank <- local_jcm_z_WW_sim_rank <- len_reps <- - len_reps_i <- NA_real_ + len_reps_i <- vs_non_pos <- Oi_Ei_zero <- NA_real_ } tcompi <- as.numeric(timingsi[["compi"]][1]) tconfigi <- as.numeric(timingsi[["configi"]][1]) @@ -452,7 +488,8 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), c8_comp_ans_BW_sim_i_rank, local_jcm_chi_BW_sim_rank, local_jcm_z_BB_sim_rank, local_jcm_z_BW_sim_rank, local_jcm_z_WW_sim_rank, local_jcm_all_BB, as.numeric(len_reps), - as.numeric(len_reps_i), tcompi, tconfigi) + as.numeric(len_reps_i), tcompi, tconfigi, as.numeric(vs_non_pos), + as.numeric(Oi_Ei_zero)) res_i } timings[["set_up"]] <- proc.time() - .ptime_start @@ -473,19 +510,15 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), pval_jcm_obs_BB <- pnorm(out[,15], lower.tail=FALSE) pval_jcm_obs_WW <- pnorm(out[,16], lower.tail=FALSE) pval_jcm_obs_BW <- pnorm(out[,17], lower.tail=TRUE) - if (con$set_allB_jcm_probs) { - sameB <- as.logical(out[,29]) - if (any(sameB)) { - pval_jcm_obs_BB[sameB] <- 0 - pval_jcm_obs_WW[sameB] <- 1 - pval_jcm_obs_BW[sameB] <- 1 - } - } - if (con$set_NA_jcm_probs) { - pval_jcm_obs_BB[is.na(pval_jcm_obs_BB)] <- 1 - pval_jcm_obs_WW[is.na(pval_jcm_obs_WW)] <- 1 - pval_jcm_obs_BW[is.na(pval_jcm_obs_BW)] <- 1 + sameB <- as.logical(out[,29]) + if (any(sameB)) { + pval_jcm_obs_BB[sameB] <- 0 + pval_jcm_obs_WW[sameB] <- 1 + pval_jcm_obs_BW[sameB] <- 1 } + pval_jcm_obs_BB[!is.finite(pval_jcm_obs_BB)] <- 1 + pval_jcm_obs_WW[!is.finite(pval_jcm_obs_WW)] <- 1 + pval_jcm_obs_BW[!is.finite(pval_jcm_obs_BW)] <- 1 local_config <- data.frame(ID=1:n, jcm_chi_obs=out[,11], jcm_count_BB_obs=out[,12], jcm_count_BW_obs=out[,13], jcm_count_WW_obs=out[,14], pval_jcm_obs_BB=pval_jcm_obs_BB, @@ -505,23 +538,19 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), pr_jcmnsim1 <- probs_lut("jcm_same", nsim, alternative=con$jcm_diff_punif_alternative) rnk <- out[,26] - pval_jcm_obs_BB <- ifelse(!is.finite(rnk), pr_jcmnsim[rnk], NA_real_) + pval_jcm_obs_BB <- ifelse(is.finite(rnk), pr_jcmnsim[rnk], NA_real_) rnk <- out[,27] - pval_jcm_obs_BW <- ifelse(!is.finite(rnk), pr_jcmnsim1[rnk], NA_real_) + pval_jcm_obs_BW <- ifelse(is.finite(rnk), pr_jcmnsim1[rnk], NA_real_) rnk <- out[,28] - pval_jcm_obs_WW <- ifelse(!is.finite(rnk), pr_jcmnsim[rnk], NA_real_) - if (con$set_allB_jcm_probs) { - if (any(sameB)) { - pval_jcm_obs_BB[sameB] <- 0 - pval_jcm_obs_WW[sameB] <- 1 - pval_jcm_obs_BW[sameB] <- 1 - } - } - if (con$set_NA_jcm_probs) { - pval_jcm_obs_BB[is.nan(pval_jcm_obs_BB)] <- 1 - pval_jcm_obs_WW[is.nan(pval_jcm_obs_WW)] <- 1 - pval_jcm_obs_BW[is.nan(pval_jcm_obs_BW)] <- 1 + pval_jcm_obs_WW <- ifelse(is.finite(rnk), pr_jcmnsim[rnk], NA_real_) + if (any(sameB)) { + pval_jcm_obs_BB[sameB] <- 0 + pval_jcm_obs_WW[sameB] <- 1 + pval_jcm_obs_BW[sameB] <- 1 } + pval_jcm_obs_BB[is.nan(pval_jcm_obs_BB)] <- 1 + pval_jcm_obs_WW[is.nan(pval_jcm_obs_WW)] <- 1 + pval_jcm_obs_BW[is.nan(pval_jcm_obs_BW)] <- 1 local_config_sim <- data.frame(ID=1:n, jcm_chi_sim_rank=out[,25], pval_jcm_obs_BB=pval_jcm_obs_BB, pval_jcm_obs_BW=pval_jcm_obs_BW, @@ -538,7 +567,7 @@ licd_multi <- function(fx, listw, zero.policy=attr(listw, "zero.policy"), "rank_sim_chi_BW", "rank_sim_chi_K", "rank_sim_anscombe_BW", "jcm_chi_sim_rank", "jcm_z_BB_sim_rank", "jcm_z_BW_sim_rank", "jcm_z_WW_sim_rank", "local_jcm_all_BB", "len_reps", "len_reps_i", - "tcompi", "tconfigi") + "tcompi", "tconfigi", "vs_non_pos", "Oi_Ei_zero") timings[["postprocessing"]] <- proc.time() - .ptime_start res <- list(local_comp=local_comp, local_config=local_config, local_comp_sim=local_comp_sim, local_config_sim=local_config_sim) diff --git a/man/licd_multi.Rd b/man/licd_multi.Rd index 53aa4187..a09b39b4 100644 --- a/man/licd_multi.Rd +++ b/man/licd_multi.Rd @@ -18,7 +18,7 @@ licd_multi(fx, listw, zero.policy = attr(listw, "zero.policy"), adjust.n = TRUE, \item{nsim}{default 0, number of conditonal permutation simulations} \item{iseed}{default NULL, used to set the seed; the output will only be reproducible if the count of CPU cores across which computation is distributed is the same} \item{no_repeat_in_row}{default \code{FALSE}, if \code{TRUE}, sample conditionally in each row without replacements to avoid duplicate values, \url{https://github.com/r-spatial/spdep/issues/124}} - \item{control}{comp_binary=\code{TRUE}, binomial_punif_alternative=\code{"greater"}, jcm_same_punif_alternative=\code{"less"}, jcm_diff_punif_alternative=\code{"greater"}, rank_ties.method=\code{"min"}, check_reps=\code{FALSE}, pysal_rank=\code{FALSE}, xtras=\code{FALSE}, set_NA_jcm_probs=\code{TRUE}, set_allB_jcm_probs=\code{TRUE}} + \item{control}{comp_binary=\code{TRUE} default TRUE, ignoring other weights styles than binary for composition measures, binomial_punif_alternative=\code{"greater"}, jcm_same_punif_alternative=\code{"less"}, jcm_diff_punif_alternative=\code{"greater"}, rank_ties.method=\code{"min"} default "min", for others see ?rank, check_reps=\code{FALSE}, unique_ceiling=1/3 used if check_reps TRUE, check_reps=FALSE if TRUE, check and report how many unique draws are made among the nsim draws, and if the number of unique draws is less than unique_ceiling, compute measures only for unique draws and copy out to replicated draws, pysal_rank=\code{FALSE} use rank with rank_ties.method and na.last; if TRUE, use pysal-style ranking to find the rank of sum(sims <= obs, na.rm=TRUE)+1 for pysal_sim_obs "GE", sum(sims < obs, na.rm=TRUE)+1 for "GT", pysal_sim_obs="GT" may also be "GE", na.last=\code{"keep"} leading to rank NA being returned if the observed joincount variance is non-positive; if TRUE joincount NAs are ranked highest when using rank, xtras=\code{FALSE} if TRUE return calculated compostion values of BW chi-squared, k-colour chi-squared, and BW Anscombe} } \details{The original code may be found at \doi{10.5281/zenodo.4283766}} From 78713ef3b2693309fbf45f91aa76cce1e01dfa70 Mon Sep 17 00:00:00 2001 From: Roger Bivand Date: Thu, 21 Nov 2024 18:17:42 +0100 Subject: [PATCH 12/14] change simulation ranking defaults --- man/licd_multi.Rd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/licd_multi.Rd b/man/licd_multi.Rd index a09b39b4..4a3dbdae 100644 --- a/man/licd_multi.Rd +++ b/man/licd_multi.Rd @@ -18,7 +18,7 @@ licd_multi(fx, listw, zero.policy = attr(listw, "zero.policy"), adjust.n = TRUE, \item{nsim}{default 0, number of conditonal permutation simulations} \item{iseed}{default NULL, used to set the seed; the output will only be reproducible if the count of CPU cores across which computation is distributed is the same} \item{no_repeat_in_row}{default \code{FALSE}, if \code{TRUE}, sample conditionally in each row without replacements to avoid duplicate values, \url{https://github.com/r-spatial/spdep/issues/124}} - \item{control}{comp_binary=\code{TRUE} default TRUE, ignoring other weights styles than binary for composition measures, binomial_punif_alternative=\code{"greater"}, jcm_same_punif_alternative=\code{"less"}, jcm_diff_punif_alternative=\code{"greater"}, rank_ties.method=\code{"min"} default "min", for others see ?rank, check_reps=\code{FALSE}, unique_ceiling=1/3 used if check_reps TRUE, check_reps=FALSE if TRUE, check and report how many unique draws are made among the nsim draws, and if the number of unique draws is less than unique_ceiling, compute measures only for unique draws and copy out to replicated draws, pysal_rank=\code{FALSE} use rank with rank_ties.method and na.last; if TRUE, use pysal-style ranking to find the rank of sum(sims <= obs, na.rm=TRUE)+1 for pysal_sim_obs "GE", sum(sims < obs, na.rm=TRUE)+1 for "GT", pysal_sim_obs="GT" may also be "GE", na.last=\code{"keep"} leading to rank NA being returned if the observed joincount variance is non-positive; if TRUE joincount NAs are ranked highest when using rank, xtras=\code{FALSE} if TRUE return calculated compostion values of BW chi-squared, k-colour chi-squared, and BW Anscombe} + \item{control}{comp_binary=\code{TRUE} default TRUE, ignoring other weights styles than binary for composition measures, binomial_punif_alternative=\code{"greater"}, jcm_same_punif_alternative=\code{"less"}, jcm_diff_punif_alternative=\code{"greater"}, rank_ties.method=\code{"min"} default "min", for others see ?rank, check_reps=\code{FALSE}, unique_ceiling=1/3 used if check_reps TRUE, check_reps=FALSE if TRUE, check and report how many unique draws are made among the nsim draws, and if the number of unique draws is less than unique_ceiling, compute measures only for unique draws and copy out to replicated draws, pysal_rank=\code{FALSE} use rank with rank_ties.method and na.last; if TRUE, use pysal-style ranking to find the rank of sum(sims <= obs, na.rm=TRUE)+1 for pysal_sim_obs "GE", sum(sims < obs, na.rm=TRUE)+1 for the count of observed greater than "GT" the simulated values, pysal_sim_obs="GT" may also be "GE", na.last=\code{"keep"} leading to rank NA being returned if the observed joincount variance is non-positive; if TRUE joincount NAs are ranked highest when using rank, xtras=\code{FALSE} if TRUE return calculated compostion values of BW chi-squared, k-colour chi-squared, and BW Anscombe} } \details{The original code may be found at \doi{10.5281/zenodo.4283766}} From 406742b08d6cc52e25141002823f2c3df49692b5 Mon Sep 17 00:00:00 2001 From: Roger Bivand Date: Thu, 21 Nov 2024 18:24:04 +0100 Subject: [PATCH 13/14] change simulation ranking defaults --- man/licd_multi.Rd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/licd_multi.Rd b/man/licd_multi.Rd index 4a3dbdae..6213d584 100644 --- a/man/licd_multi.Rd +++ b/man/licd_multi.Rd @@ -18,7 +18,7 @@ licd_multi(fx, listw, zero.policy = attr(listw, "zero.policy"), adjust.n = TRUE, \item{nsim}{default 0, number of conditonal permutation simulations} \item{iseed}{default NULL, used to set the seed; the output will only be reproducible if the count of CPU cores across which computation is distributed is the same} \item{no_repeat_in_row}{default \code{FALSE}, if \code{TRUE}, sample conditionally in each row without replacements to avoid duplicate values, \url{https://github.com/r-spatial/spdep/issues/124}} - \item{control}{comp_binary=\code{TRUE} default TRUE, ignoring other weights styles than binary for composition measures, binomial_punif_alternative=\code{"greater"}, jcm_same_punif_alternative=\code{"less"}, jcm_diff_punif_alternative=\code{"greater"}, rank_ties.method=\code{"min"} default "min", for others see ?rank, check_reps=\code{FALSE}, unique_ceiling=1/3 used if check_reps TRUE, check_reps=FALSE if TRUE, check and report how many unique draws are made among the nsim draws, and if the number of unique draws is less than unique_ceiling, compute measures only for unique draws and copy out to replicated draws, pysal_rank=\code{FALSE} use rank with rank_ties.method and na.last; if TRUE, use pysal-style ranking to find the rank of sum(sims <= obs, na.rm=TRUE)+1 for pysal_sim_obs "GE", sum(sims < obs, na.rm=TRUE)+1 for the count of observed greater than "GT" the simulated values, pysal_sim_obs="GT" may also be "GE", na.last=\code{"keep"} leading to rank NA being returned if the observed joincount variance is non-positive; if TRUE joincount NAs are ranked highest when using rank, xtras=\code{FALSE} if TRUE return calculated compostion values of BW chi-squared, k-colour chi-squared, and BW Anscombe} + \item{control}{comp_binary=\code{TRUE} default TRUE, ignoring other weights styles than binary for composition measures, binomial_punif_alternative=\code{"greater"}, jcm_same_punif_alternative=\code{"less"}, jcm_diff_punif_alternative=\code{"greater"}, rank_ties.method=\code{"min"} default "min", na.last=\code{"keep"} leading to rank NA being returned if the observed joincount variance is non-positive; if TRUE joincount NAs are ranked highest when using rank, for others see ?rank, check_reps=\code{FALSE}, unique_ceiling=1/3 used if check_reps TRUE, check_reps=FALSE if TRUE, check and report how many unique draws are made among the nsim draws, and if the number of unique draws is less than unique_ceiling, compute measures only for unique draws and copy out to replicated draws, pysal_rank=\code{FALSE} use rank with rank_ties.method and na.last; if TRUE, use pysal-style ranking to find the rank of sum(sims <= obs, na.rm=TRUE)+1 for pysal_sim_obs "GE", sum(sims < obs, na.rm=TRUE)+1 for the count of observed greater than "GT" the simulated values, pysal_sim_obs="GT" may also be "GE", xtras=\code{FALSE} if TRUE return calculated compostion values of BW chi-squared, k-colour chi-squared, and BW Anscombe} } \details{The original code may be found at \doi{10.5281/zenodo.4283766}} From c0e916dbccf7cec9cc7d4f88f4203abcc50baf0f Mon Sep 17 00:00:00 2001 From: Roger Bivand Date: Fri, 22 Nov 2024 16:11:01 +0100 Subject: [PATCH 14/14] drop rgeoda --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 847f9da0..cfa47509 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -46,7 +46,7 @@ Authors@R: c(person("Roger", "Bivand", role = c("cre", "aut"), person("Danlin", "Yu", role = "ctb")) Depends: R (>= 3.3.0), methods, spData (>= 2.3.1), sf Imports: stats, deldir, boot (>= 1.3-1), graphics, utils, grDevices, units, s2, e1071, sp (>= 1.0) -Suggests: spatialreg (>= 1.2-1), Matrix, parallel, dbscan, RColorBrewer, lattice, xtable, foreign, igraph, RSpectra, knitr, classInt, tmap, spam, ggplot2, rmarkdown, tinytest, rgeoda +Suggests: spatialreg (>= 1.2-1), Matrix, parallel, dbscan, RColorBrewer, lattice, xtable, foreign, igraph, RSpectra, knitr, classInt, tmap, spam, ggplot2, rmarkdown, tinytest URL: https://github.com/r-spatial/spdep/, https://r-spatial.github.io/spdep/ BugReports: https://github.com/r-spatial/spdep/issues/ Description: A collection of functions to create spatial weights matrix