diff --git a/data_updates/R/MultiSystem entire dataset.zip b/data_updates/R/MultiSystem entire dataset.zip new file mode 100644 index 000000000..5830e505a Binary files /dev/null and b/data_updates/R/MultiSystem entire dataset.zip differ diff --git a/data_updates/R/load_mirrors.R b/data_updates/R/load_mirrors.R index 0b2670029..5407fc6fc 100644 --- a/data_updates/R/load_mirrors.R +++ b/data_updates/R/load_mirrors.R @@ -117,7 +117,6 @@ clean_dac5_file = function(){ chunk_load_table(con, dac5.table.quote, dac5_path, dac5.names) } - clean_dac1_file = function(){ dac1 <- fread(dac1_path) setnames(dac1, diff --git a/data_updates/R/mums.R b/data_updates/R/mums.R new file mode 100644 index 000000000..c80b0f3c4 --- /dev/null +++ b/data_updates/R/mums.R @@ -0,0 +1,33 @@ +list.of.packages <- c("data.table","RPostgreSQL","reshape2","here") +new.packages <- list.of.packages[!(list.of.packages %in% installed.packages()[,"Package"])] +if(length(new.packages)) install.packages(new.packages, repos="http://cran.us.r-project.org") +lapply(list.of.packages, require, character.only=T) + +# Only works while running with `Rscript` from repo root, use commented below if running manually +# script.dir <- here() +# script.dir = "/src" +# script.dir = "C:/git/ddw-analyst-ui" +# setwd("C:/git/ddw-analyst-ui/data_updates/R") +source(paste0(script.dir,"/data_updates/R/constants.R")) + +drv = dbDriver("PostgreSQL") +con = dbConnect(drv, + dbname=db.dbname + ,user=db.user + ,password=db.password + ,host=db.host + ,port=db.port) +# con = dbConnect(drv, +# dbname="analyst_ui" +# ,user="postgres") + +table.name = "mums" +table.quote = c("repo",table.name) + +# Load data, removing na strings + +unzip("MultiSystem entire dataset.zip") +mums = read.table("MultiSystem entire dataset.txt",header=T,sep="|",quote="",fill=TRUE) + +dbWriteTable(con, name = table.quote, value = mums, row.names = F, overwrite = T) +dbDisconnect(con) \ No newline at end of file diff --git a/data_updates/R/sdg_goals.R b/data_updates/R/sdg_goals.R new file mode 100644 index 000000000..8888af990 --- /dev/null +++ b/data_updates/R/sdg_goals.R @@ -0,0 +1,33 @@ +list.of.packages <- c("data.table","RPostgreSQL","reshape2","here") +new.packages <- list.of.packages[!(list.of.packages %in% installed.packages()[,"Package"])] +if(length(new.packages)) install.packages(new.packages, repos="http://cran.us.r-project.org") +lapply(list.of.packages, require, character.only=T) + +# Only works while running with `Rscript` from repo root, use commented below if running manually +script.dir <- here() +# script.dir = "/src" +# script.dir = "C:/git/ddw-analyst-ui" +source(paste0(script.dir,"/data_updates/R/constants.R")) + +drv = dbDriver("PostgreSQL") +con = dbConnect(drv, + dbname=db.dbname + ,user=db.user + ,password=db.password + ,host=db.host + ,port=db.port) +# con = dbConnect(drv, +# dbname="analyst_ui" +# ,user="postgres") + +table.name = "sdg_goals" +table.quote = c("repo",table.name) + +# Load data, removing na strings +data_url = paste0(script.dir,"/data_updates/manual/CSV/sdg_goals.csv") +sdg = read.csv(data_url) + +sdg$alpha_ordering <- tolower(LETTERS[sdg$goal]) + +dbWriteTable(con, name = table.quote, value = sdg, row.names = F, overwrite = T) +dbDisconnect(con) \ No newline at end of file diff --git a/data_updates/R/sdg_indicators.R b/data_updates/R/sdg_indicators.R new file mode 100644 index 000000000..ea852d8e2 --- /dev/null +++ b/data_updates/R/sdg_indicators.R @@ -0,0 +1,53 @@ +list.of.packages <- c("data.table","RPostgreSQL","reshape2","here","Unicode") +new.packages <- list.of.packages[!(list.of.packages %in% installed.packages()[,"Package"])] +if(length(new.packages)) install.packages(new.packages, repos="http://cran.us.r-project.org") +lapply(list.of.packages, require, character.only=T) + +# Only works while running with `Rscript` from repo root, use commented below if running manually +script.dir <- here() +# script.dir = "/src" +# script.dir = "C:/git/ddw-analyst-ui" +source(paste0(script.dir,"/data_updates/R/constants.R")) + +drv = dbDriver("PostgreSQL") +con = dbConnect(drv, + dbname=db.dbname + ,user=db.user + ,password=db.password + ,host=db.host + ,port=db.port) +# con = dbConnect(drv, +# dbname="analyst_ui" +# ,user="postgres") + +dbListTables(con) + +table.name = "sdg_indicators" +table.quote = c("repo",table.name) + +# Load data, removing na strings +data_url = paste0(script.dir,"/data_updates/manual/CSV/sdg_indicators.csv") +sdg = read.csv(data_url) + +sdg = cbind(sdg,colsplit(sdg$indicator,"\\.",names = c("goal2","target2","indicator2"))) +sdg$alpha <- NA +sdg$alpha2 <- NA +sdg$indicator2 <- as.character(sdg$indicator2) + +for (k in c(1:nrow(sdg))){ + sdg$alpha[k] <- utf8ToInt(sdg$target2[k]) + sdg$alpha2[k] <- utf8ToInt(sdg$indicator2[k]) +} + +sdg$goal3 <- tolower(LETTERS[sdg$goal2]) +sdg <- data.table(sdg)[,target3:=frank(alpha,ties.method = "dense"),by=.(goal2)] +sdg$target3 <- tolower(LETTERS[sdg$target3]) +sdg <- data.table(sdg)[,indicator3:=frank(alpha,ties.method = "dense"),by=.(goal2)] +sdg$indicator3 <- tolower(LETTERS[sdg$indicator3]) + +sdg$alpha_ordering <- paste0(sdg$goal3,sdg$target3,sdg$indicator3) + +sdg = sdg[,c(1:6,15)] + +dbWriteTable(con, name = table.quote, value = sdg, row.names = F, overwrite = T) +dbDisconnect(con) \ No newline at end of file diff --git a/data_updates/R/sdg_targets.R b/data_updates/R/sdg_targets.R new file mode 100644 index 000000000..2cbb9a800 --- /dev/null +++ b/data_updates/R/sdg_targets.R @@ -0,0 +1,46 @@ +list.of.packages <- c("data.table","RPostgreSQL","reshape2","here","Unicode") +new.packages <- list.of.packages[!(list.of.packages %in% installed.packages()[,"Package"])] +if(length(new.packages)) install.packages(new.packages, repos="http://cran.us.r-project.org") +lapply(list.of.packages, require, character.only=T) + +# Only works while running with `Rscript` from repo root, use commented below if running manually +script.dir <- here() +# script.dir = "/src" +# script.dir = "C:/git/ddw-analyst-ui" +source(paste0(script.dir,"/data_updates/R/constants.R")) + +drv = dbDriver("PostgreSQL") +con = dbConnect(drv, + dbname=db.dbname + ,user=db.user + ,password=db.password + ,host=db.host + ,port=db.port) +# con = dbConnect(drv, +# dbname="analyst_ui" +# ,user="postgres") + +table.name = "sdg_targets" +table.quote = c("repo",table.name) + +# Load data, removing na strings +data_url = paste0(script.dir,"/data_updates/manual/CSV/sdg_targets.csv") +sdg = read.csv(data_url) + +sdg = cbind(sdg,colsplit(sdg$target,"\\.",names = c("goal2","target2"))) +sdg$alpha <- NA + +for (k in c(1:nrow(sdg))){ + sdg$alpha[k] <- utf8ToInt(sdg$target2[k]) +} + +sdg$goal3 <- tolower(LETTERS[sdg$goal2]) +sdg <- data.table(sdg)[,target3:=rank(alpha),by=.(goal)] +sdg$target3 <- tolower(LETTERS[sdg$target3]) + +sdg$alpha_ordering <- paste0(sdg$goal3,sdg$target3) + +sdg = sdg[,c(1:3,9)] + +dbWriteTable(con, name = table.quote, value = sdg, row.names = F, overwrite = T) +dbDisconnect(con) \ No newline at end of file