Skip to content

Commit

Permalink
quick naming updates
Browse files Browse the repository at this point in the history
  • Loading branch information
realxinzhao committed Aug 7, 2023
1 parent 9b6cb0e commit 1f1f168
Show file tree
Hide file tree
Showing 5 changed files with 319 additions and 154 deletions.
3 changes: 2 additions & 1 deletion R/xfaostat_L100_constants.R
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
DIR_RAW_DATA_FAOSTAT <- system.file("extdata", "aglu/FAO/FAOSTAT", package = "gcamdata")



# Output GCAM csv
DIR_OUT_CSV <- "inst/extdata/aglu/FAO"


# Balance elements; used in Get_SUA_TEMPLATE and SUA_bal_adjust
Expand Down
171 changes: 171 additions & 0 deletions R/xfaostat_L101_RawDataPreProcessing3.R
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
# Copyright 2019 Battelle Memorial Institute; see the LICENSE file.

#' module_xfaostat_L101_RawDataPreProcessing3
#'
#' Preprocess raw faostat data
#'
#' @param command API command to execute
#' @param ... other optional parameters, depending on command
#' @return Depends on \code{command}: either a vector of required inputs, a vector of output names, or (if
#' \code{command} is "MAKE") all the generated outputs
#' @details This chunk compiles balanced supply utilization data in primary equivalent in GCAM region and commodities.
#' @importFrom assertthat assert_that
#' @importFrom dplyr summarize bind_rows filter if_else inner_join left_join mutate rename select n group_by_at
#' first case_when vars
#' @importFrom tibble tibble
#' @importFrom tidyr complete drop_na gather nesting spread replace_na
#' @author XZ 2023
module_xfaostat_L101_RawDataPreProcessing3 <- function(command, ...) {

MODULE_INPUTS <-
c(FILE = "aglu/AGLU_ctry",
"QCL_area_code_map")

MODULE_OUTPUTS <-
c("TCL", "TM_wide" # Gross and bilateral trade
)

if(command == driver.DECLARE_INPUTS) {
return(MODULE_INPUTS)
} else if(command == driver.DECLARE_OUTPUTS) {
return(MODULE_OUTPUTS)
} else if(command == driver.MAKE) {

year <- value <- Year <- Value <- FAO_country <- iso <- NULL # silence package check.

all_data <- list(...)[[1]]

# Load required inputs ----

get_data_list(all_data, MODULE_INPUTS, strip_attributes = TRUE)



FAOSTAT_RDS <- c("TCL", "TM_wide")

DIR_PREBUILT_FAOSTAT <- "data/PREBUILT_FAOSTAT"

lapply(FAOSTAT_RDS, function(d){
assertthat::assert_that(file.exists(file.path(DIR_PREBUILT_FAOSTAT, paste0(d, ".rds"))))
assign(d, readRDS(file.path(DIR_PREBUILT_FAOSTAT, paste0(d, ".rds"))),
envir = parent.env(environment()))
})

### output TCL and clean memory ----
TCL %>%
add_title("FAO TCL") %>%
add_units("tonne") %>%
add_comments("Preprocessed FAO TCL") ->
TCL

TM_wide %>%
add_title("FAO TM") %>%
add_units("tonne") %>%
add_comments("Preprocessed FAO TM_wide") ->
TM_wide


#
# QCL_area_code <- QCL_area_code_map %>% distinct(area_code) %>% pull()
#
# # *[TCL] Gross trade ----
# FAOSTAT_load_raw_data("TCL") # Gross trade
#
# TCL %>% distinct(element, element_code, unit)
# TCL %>% distinct(item, item_code)
#
# TCL %>%
# filter(item_code <= 1700,
# # only keep quantity
# !element_code %in% c(5622 , 5922),
# area_code %in% QCL_area_code) %>%
# select(area_code, area, item_code, item, element_code, element, year, value, unit) %>%
# rm_accent("item", "area") -> TCL1
#
# ### output TCL and clean memory ----
# TCL1 %>%
# add_title("FAO TCL") %>%
# add_units("tonne") %>%
# add_comments("Preprocessed FAO TCL") ->
# TCL
#
# rm(TCL1)
#
#
#
# # *[TM] Bilateral trade ----
# #*FAO has better quality bilateral data since 1992, covering most SUA items
# FAOSTAT_load_raw_data("TM") # Bilateral trade
#
# TM %>%
# # Only keep quantities for elements with a unit of tonnes
# filter(element_code %in% c(5910, 5610),
# item_code < 1700,
# # Bilateral trade year starts from 1986 but higher quality after 1992
# # Subset data also to shrink the size
# year >= 1992,
# partner_country_code %in% QCL_area_code,
# reporter_country_code %in% QCL_area_code) %>%
# select(reporter_country_code, reporter_countries,
# partner_country_code, partner_countries,
# item_code, item, element_code, element, year, value, unit) ->
# TM1
# rm(TM)
#
#
# ## **Reconcile export and import bilateral flow ----
# # Full join export and import and use available import to fill missing and zero export
# TM1 %>% filter(element %in% c("Export Quantity")) %>% spread(element, value) %>%
# select(exporter = reporter_country_code,
# importer = partner_country_code, item_code, year, expflow = `Export Quantity`) %>%
# full_join(
# TM1 %>% filter(element %in% c("Import Quantity")) %>% spread(element, value)%>%
# select(importer = reporter_country_code,
# exporter = partner_country_code, item_code, year, impflow = `Import Quantity`),
# by = c("exporter", "importer", "item_code", "year")
# ) %>%
# # replace na with zero but use import to replace zero export later
# replace_na(list(expflow = 0, impflow = 0)) %>%
# transmute(area_code = importer, year, item_code, source_code = exporter,
# value = if_else(expflow == 0, impflow, expflow)) %>%
# mutate(element = "Import Quantity") ->
# TM2
#
#
# TM2 %>%
# # remove self-trade (per unaggregated area_code) which existed in FAO TM importing data and likely due to data processing mistakes.
# filter(area_code != source_code) %>%
# left_join(TM1 %>% distinct(item, item_code), by = c("item_code")) %>%
# left_join(TM1 %>% distinct(area = partner_countries, area_code = partner_country_code), by = c("area_code")) %>%
# left_join(TM1 %>% distinct(source = partner_countries, source_code = partner_country_code), by = c("source_code")) %>%
# rm_accent("item", "area", "source") %>%
# mutate(unit = "tonnes") ->
# TM3
# rm(TM1, TM2)
#
# TM3 %>% spread(year, value) -> TM4
#
# ### output OA and clean memory ----
# TM4 %>%
# add_title("FAO TM") %>%
# add_units("tonne") %>%
# add_comments("Preprocessed FAO TM") ->
# TM
#
# rm(TM3, TM4)
# rm(QCL_area_code)
#
# ### output TM and clean memory ----
# TM %>%
# add_title("FAO TM") %>%
# add_units("tonne") %>%
# add_comments("Preprocessed FAO TM_wide") ->
# TM_wide


return_data(MODULE_OUTPUTS)

} else {
stop("Unknown command")
}
}
7 changes: 3 additions & 4 deletions R/xfaostat_L105_DataConnectionToSUA.R
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ module_xfaostat_L105_DataConnectionToSUA <- function(command, ...) {
FILE = "aglu/FAO/Mapping_FBSH_SCL_OilCake")

MODULE_OUTPUTS <-
c("GCAMDATA_FAOSTAT_SUA_195Regs_530Items_2010to2019")
c("Bal_new_all")

if(command == driver.DECLARE_INPUTS) {
return(MODULE_INPUTS)
Expand Down Expand Up @@ -759,11 +759,10 @@ module_xfaostat_L105_DataConnectionToSUA <- function(command, ...) {

### output GCAMDATA_FAOSTAT_SUA_195Regs_530Items_2010to2019 and clean memory ----
Bal_new_all %>%
spread(year, value) %>%
add_title("GCAMDATA_FAOSTAT_SUA_195Regs_530Items_2010to2019") %>%
add_title("Bal_new_all") %>%
add_units("Ktonne") %>%
add_comments("Preprocessed FAO SUA 2010 - 2019") ->
GCAMDATA_FAOSTAT_SUA_195Regs_530Items_2010to2019
Bal_new_all


return_data(MODULE_OUTPUTS)
Expand Down
149 changes: 0 additions & 149 deletions xfaostat_L101_RawDataPreProcessing3.R

This file was deleted.

Loading

0 comments on commit 1f1f168

Please sign in to comment.