Skip to content

Commit

Permalink
Adding more processing chunks
Browse files Browse the repository at this point in the history
Adding price, production, and area related processing
function updates
gz data updates (per recent gcamdata); also consistent with this package
  • Loading branch information
realxinzhao committed Aug 7, 2023
1 parent 6647ca6 commit 8a8586c
Show file tree
Hide file tree
Showing 22 changed files with 1,658 additions and 266 deletions.
42 changes: 7 additions & 35 deletions DESCRIPTION
Original file line number Diff line number Diff line change
@@ -1,40 +1,12 @@
Package: gcamdata
Type: Package
Title: Build the GCAM Input Datasets
Version: 5.1
Date: 2019-07-06
Authors@R: c( person("Russell", "Horowitz",
email = "[email protected]",
role = c("cre","aut"),
comment = c(ORCID = "0000-0002-0270-3127")),
person("Ben", "Bond-Lamberty", role = c( "aut"), comment = c(ORCID = "0000-0001-9525-4633")),
person("Katherine", "Calvin", role = "aut", comment = c(ORCID = "0000-0003-2191-4189")),
person("Ryna", "Cui", role = "aut", comment = c(ORCID = "0000-0002-8531-4126")),
person("Kalyn", "Dorheim", role = "aut", comment = c(ORCID = "0000-0001-8093-8397")),
person("Leyang", "Feng", role = "aut"),
person("Jill", "Horing", role = "aut", comment = c(ORCID = "0000-0003-2516-5533")),
person("Page", "Kyle", role = "aut", comment = c(ORCID = "0000-0002-1257-8358")),
person("Robert", "Link", role = "aut"),
person("Pralit", "Patel", role = "aut", comment = c(ORCID = "0000-0003-3992-1061")),
person("Christopher", "Roney", role = "aut", comment = c(ORCID = "0000-0003-0509-5259")),
person("Abigail", "Snyder", role = "aut", comment = c(ORCID = "0000-0002-9034-9948")),
person("Aaron", "Staniszweski", role = "aut"),
person("Sean", "Turner", role = "aut", comment = c(ORCID = "0000-0003-4400-9800")),
person("Matthew", "Binsted", role = "aut", comment = c(ORCID = "0000-0002-5177-7253")),
person("Zarrar", "Khan", role = "aut", comment = c(ORCID = "0000-0002-8147-8553")),
person("Corinne", "Hartin", role = "ctb", comment = c(ORCID = "0000-0003-1834-6539")),
person("Rachel", "Hoesly", role = "ctb"),
person("Mohamad", "Hejazi", role = "ctb"),
person("Gokul", "Iyer", role = "ctb"),
person("Sonny", "Kim", role = "ctb"),
person("Yaling", "Liu", role = "ctb"),
person("Haewon", "McJeon", role = "ctb"),
person("Steve", "Smith", role = "ctb"),
person("Stephanie", "Waldhoff", role = "ctb", comment = c(ORCID = "0000-0002-8073-0868")),
person("Marshall", "Wise", role = "ctb"),
person("Leon", "Clarke", role = c("rth", "ctb")),
person("Pacific Northwest National Laboratory", role = "cph"))
Description: Build the GCAM XML input files.
Title: Process FAO data for global agroeconomic modeling
Version: 1.1
Date: 2023-05-06
Authors@R: c(person("Xin", "Zhao", email = "[email protected]", role = c("cre", "aut"), comment = c(0000-0002-1801-4393)),
person("Katherine", "Calvin", role = "aut", comment = c(ORCID = "0000-0003-2191-4189")),
person("Pacific Northwest National Laboratory", role = "cph"))
Description: Clean and balance agricultural input and supply-utilization data.
License: MIT + file LICENSE
Depends: R (>= 3.1.2)
Imports:
Expand Down
2 changes: 2 additions & 0 deletions NAMESPACE
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ importFrom(dplyr,select)
importFrom(dplyr,summarise)
importFrom(dplyr,summarise_all)
importFrom(dplyr,summarize)
importFrom(dplyr,transmute)
importFrom(dplyr,ungroup)
importFrom(dplyr,vars)
importFrom(grDevices,rainbow)
Expand All @@ -64,6 +65,7 @@ importFrom(tibble,is_tibble)
importFrom(tibble,tibble)
importFrom(tidyr,complete)
importFrom(tidyr,drop_na)
importFrom(tidyr,fill)
importFrom(tidyr,gather)
importFrom(tidyr,nesting)
importFrom(tidyr,replace_na)
Expand Down
6 changes: 3 additions & 3 deletions R/module-helpers.R
Original file line number Diff line number Diff line change
Expand Up @@ -716,8 +716,8 @@ FAO_AREA_DISAGGREGATE_HIST_DISSOLUTION <-
#' @param .DF Input data frame
#' @param SUDAN2012_BREAK If T break Sudan before 2012 based on 2013- 2016 data
#' @param SUDAN2012_MERGE If T merge South Sudan into Sudan
#' @param .FAO_AREA_CODE_COL
#' @param .AREA_COL
#' @param .FAO_AREA_CODE_COL Name of col of area
#' @param .AREA_COL Name of col of area code
#'
#' @return data with historical periods of dissolved region disaggregated to small pieces.

Expand Down Expand Up @@ -842,7 +842,7 @@ FAO_AREA_DISAGGREGATE_HIST_DISSOLUTION_ALL <- function(.DF,

if (all_of(.AREA_COL) %in% names(.DF0)) {
.DF2 %>%# Get area back
left_join(.DF0 %>% distinct_at(vars(all_of(.AREA_COL), .FAO_AREA_CODE_COL)),
left_join(.DF0 %>% dplyr::distinct(across(c(.AREA_COL, .FAO_AREA_CODE_COL))) ,
by = .FAO_AREA_CODE_COL) -> .DF2
}

Expand Down
14 changes: 13 additions & 1 deletion R/xfaostat_L100_constants.R
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,16 @@
# General behavior constants ======================================================================

# having issues with package check here
DIR_RAW_DATA_FAOSTAT <- system.file("extdata", "aglu/FAO/FAOSTAT", package = "gcamdata")
# comment this line out when building package
DIR_RAW_DATA_FAOSTAT <- system.file("extdata", "aglu/FAO/FAOSTAT", package = "gcamdata")





# used in Get_SUA_TEMPLATE and SUA_bal_adjust

c("Opening stocks", "Production", "Import",
"Export", "Processed", "Food", "Feed", "Seed", "Other uses", "Loss", "Closing stocks",
"Residuals", "Regional supply", "Regional demand", "Stock Variation") ->
Bal_element_new
10 changes: 5 additions & 5 deletions R/xfaostat_L101_RawDataPreProcessing1.R
Original file line number Diff line number Diff line change
Expand Up @@ -102,12 +102,12 @@ module_xfaostat_L101_RawDataPreProcessing1 <- function(command, ...) {
# Using index to fill in missing across years
PP1 %>%
filter(element_code %in% c(5532, 5539)) %>%
select(area_code, area, item_code, item, element_code, element, year, value, unit) %>%
select(area_code, area, item_code, item, element, year, value) %>%
# Not completing year and area here
spread(element, value) %>%
left_join(
PP1 %>% filter(element_code %in% c(5532, 5539)) %>%
select(area_code, area, item_code, item, element_code, element, year, value, unit) %>%
select(area_code, area, item_code, item, element, year, value) %>%
spread(element, value) %>%
rename(pp_base = `Producer Price (USD/tonne)`,
pp_baseindex = `Producer Price Index (2014-2016 = 100)`) %>%
Expand All @@ -133,7 +133,7 @@ module_xfaostat_L101_RawDataPreProcessing1 <- function(command, ...) {
add_comments("Preprocessed FAOSTAT producer prices") ->
PP

rm(PP, PP1, PP2)
rm(PP1, PP2)

# Food balance and Supply-Utilization-Account

Expand All @@ -156,7 +156,7 @@ module_xfaostat_L101_RawDataPreProcessing1 <- function(command, ...) {
add_units("tonne") %>%
add_comments("Preprocessed FAOSTAT SCL") ->
FBS
rm(FBS, FBS1)
rm(FBS1)


## *[SCL] SUA: supply utilization accounting ----
Expand Down Expand Up @@ -185,7 +185,7 @@ module_xfaostat_L101_RawDataPreProcessing1 <- function(command, ...) {
add_units("tonne") %>%
add_comments("Preprocessed FAOSTAT SCL") ->
SCL
rm(SCL, SCL1)
rm(SCL1)


return_data(MODULE_OUTPUTS)
Expand Down
9 changes: 2 additions & 7 deletions R/xfaostat_L101_RawDataPreProcessing2.R
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,6 @@ module_xfaostat_L101_RawDataPreProcessing2 <- function(command, ...) {

QCL_area_code <- QCL_area_code_map %>% distinct(area_code) %>% pull()

# *[QCL] FAOSTAT Production and area ----

## Load raw data
FAOSTAT_load_raw_data(DATASETCODE = "QCL", DATA_FOLDER = DIR_RAW_DATA_FAOSTAT)


# Food balance and Supply-Utilization-Account

Expand Down Expand Up @@ -147,7 +142,7 @@ module_xfaostat_L101_RawDataPreProcessing2 <- function(command, ...) {
FBSH_CB

rm(SHELL_RATE_groundnuts, Mill_RATE_rice);
rm(FBSH_CB1, FBSH, CB, dup_item_code)
rm(FBSH_CB1, dup_item_code)


# *[OA]: Population ----
Expand All @@ -166,7 +161,7 @@ module_xfaostat_L101_RawDataPreProcessing2 <- function(command, ...) {
add_units("tonne") %>%
add_comments("Preprocessed FAO OA") ->
OA
rm(OA, OA1)
rm(OA1)
rm(QCL_area_code)


Expand Down
Loading

0 comments on commit 8a8586c

Please sign in to comment.