diff --git a/R/constants.R b/R/constants.R index 6591a345..5a53e182 100644 --- a/R/constants.R +++ b/R/constants.R @@ -15,7 +15,8 @@ OUTPUT_Export_CSV <- TRUE ## Fao raw data folder DIR_RAW_DATA_FAOSTAT <- "inst/extdata/aglu/FAO/FAOSTAT" ## Output GCAM csv -DIR_OUTPUT_CSV <- "inst/extdata/aglu/FAO/temp" +#DIR_OUTPUT_CSV <- "inst/extdata/aglu/FAO/temp" +DIR_OUTPUT_CSV <- "outputs/CSV" dir.create(file.path(DIR_OUTPUT_CSV), showWarnings = FALSE) diff --git a/R/xfaostat_L101_RawDataPreProc2_PP_PD_OA.R b/R/xfaostat_L101_RawDataPreProc2_PP_PD_OA.R index e119f973..f1cb3350 100644 --- a/R/xfaostat_L101_RawDataPreProc2_PP_PD_OA.R +++ b/R/xfaostat_L101_RawDataPreProc2_PP_PD_OA.R @@ -24,7 +24,8 @@ module_xfaostat_L101_RawDataPreProc2_PP_PD_OA <- function(command, ...) { FILE = "aglu/FAO/FAOSTAT/Other_supplementary/GDP_deflator_Taiwan", "QCL_area_code_map") - MODULE_OUTPUTS <- + + MODULE_OUTPUTS <- c("PP_wide", # Producer prices "PD", # GDP deflator "OA") # Population @@ -44,7 +45,6 @@ module_xfaostat_L101_RawDataPreProc2_PP_PD_OA <- function(command, ...) { get_data_list(all_data, MODULE_INPUTS, strip_attributes = TRUE) - if(Process_Raw_FAO_Data == FALSE) { # Prebuilt data is read here ---- @@ -175,7 +175,7 @@ module_xfaostat_L101_RawDataPreProc2_PP_PD_OA <- function(command, ...) { add_comments("Preprocessed FAOSTAT regional gdp deflators") %>% add_precursors("QCL_area_code_map", "aglu/FAO/FAOSTAT/Deflators_E_All_Data_(Normalized)_PalceHolder", - "aglu/fao/FAOSTAT/Other_supplementary/GDP_deflator_Taiwan") -> + "aglu/FAO/FAOSTAT/Other_supplementary/GDP_deflator_Taiwan") -> PD verify_identical_prebuilt(PD) diff --git a/R/xfaostat_L101_RawDataPreProc6_TM.R b/R/xfaostat_L101_RawDataPreProc6_TM.R index 929c2d4b..0f71b61c 100644 --- a/R/xfaostat_L101_RawDataPreProc6_TM.R +++ b/R/xfaostat_L101_RawDataPreProc6_TM.R @@ -47,21 +47,23 @@ module_xfaostat_L101_RawDataPreProc6_TM <- function(command, ...) { } else { - # Get area code ---- - QCL_area_code <- QCL_area_code_map %>% distinct(area_code) %>% pull() + FAOSTAT_RDS <- c("TM_bilateral_wide") - FAOSTAT_RDS <- c("TM_bilateral_wide") + DIR_PREBUILT_FAOSTAT <- "data" - DIR_PREBUILT_FAOSTAT <- "data/PREBUILT_FAOSTAT" + lapply(FAOSTAT_RDS, function(d){ + assertthat::assert_that(file.exists(file.path(DIR_PREBUILT_FAOSTAT, paste0(d, ".rds")))) + assign(d, readRDS(file.path(DIR_PREBUILT_FAOSTAT, paste0(d, ".rds"))), + envir = parent.env(environment())) + }) - lapply(FAOSTAT_RDS, function(d){ - assertthat::assert_that(file.exists(file.path(DIR_PREBUILT_FAOSTAT, paste0(d, ".rds")))) - assign(d, readRDS(file.path(DIR_PREBUILT_FAOSTAT, paste0(d, ".rds"))), - envir = parent.env(environment())) - }) + # # Get area code ---- + # QCL_area_code <- QCL_area_code_map %>% distinct(area_code) %>% pull() + # + # # # *[TM] Bilateral trade ---- # #*FAO has better quality bilateral data since 1992, covering most SUA items # FAOSTAT_load_raw_data("TM") # Bilateral trade @@ -123,7 +125,7 @@ module_xfaostat_L101_RawDataPreProc6_TM <- function(command, ...) { ### output TM ---- TM_bilateral_wide %>% - add_title("FAO bilateral trade (TM) ") %>% + add_title("FAO bilateral trade (TM)", overwrite = T) %>% add_units("tonne") %>% add_comments("Preprocessed FAO TM_wide") %>% add_precursors("aglu/FAO/FAOSTAT/Trade_DetailedTradeMatrix_E_All_Data_(Normalized)_PalceHolder", diff --git a/R/xfaostat_L102_ProductionArea.R b/R/xfaostat_L102_ProductionArea.R index 189a3ee4..af3dc3cb 100644 --- a/R/xfaostat_L102_ProductionArea.R +++ b/R/xfaostat_L102_ProductionArea.R @@ -45,15 +45,12 @@ module_xfaostat_L102_ProductionArea <- function(command, ...) { # wide to long QCL_wide %>% gather_years() %>% - filter(year >= min(FAOSTAT_Hist_Year_FBS)) %>% FAOSTAT_AREA_RM_NONEXIST() -> QCL FBS_wide %>% gather_years() %>% - filter(year >= min(FAOSTAT_Hist_Year_FBS)) %>% FAOSTAT_AREA_RM_NONEXIST() -> FBS FBSH_CB_wide %>% gather_years() %>% - filter(year >= min(FAOSTAT_Hist_Year_FBS)) %>% FAOSTAT_AREA_RM_NONEXIST() -> FBSH_CB @@ -372,13 +369,14 @@ module_xfaostat_L102_ProductionArea <- function(command, ...) { FBS_FISH <- FBS %>% filter(year >= 2010, element_code == 5511, # production item_code %in% c(FBS_COMM_FISH %>% pull(item_code))) %>% - replace_na(list(value = 0)) %>% bind_rows( FBSH_CB %>% filter(year < 2010, element_code == 5511, # production item_code %in% c(FBS_COMM_FISH %>% pull(item_code))) - ) %>% mutate(value = value *1000, - unit = "tonnes", - element_code = 5510 # changed here for consistency + ) %>% + replace_na(list(value = 0)) %>% + mutate(value = value *1000, + unit = "tonnes", + element_code = 5510 # changed here for consistency ) %>% select(area_code, area, item_code, item, element, element_code, year, value, unit) %>% FAOSTAT_AREA_RM_NONEXIST diff --git a/R/xfaostat_L199_CSVExportAgSUA.R b/R/xfaostat_L199_CSVExportAgSUA.R index 5fcfd408..993018f4 100644 --- a/R/xfaostat_L199_CSVExportAgSUA.R +++ b/R/xfaostat_L199_CSVExportAgSUA.R @@ -18,7 +18,8 @@ module_xfaostat_L199_ExportCSV <- function(command, ...) { MODULE_INPUTS <- - c("Bal_new_all", + c("PD", + "Bal_new_all", "FBSH_CB_wide", "QCL_PROD", "QCL_AN_LIVEANIMAL", @@ -27,11 +28,10 @@ module_xfaostat_L199_ExportCSV <- function(command, ...) { "QCL_FODDERCROP", "QCL_PRIMARY_PROD_PV", "TM_bilateral_wide", - "PD", "SUA_food_macronutrient_rate") MODULE_OUTPUTS <- - c("xfaostat_L199_DUMMY") + c("xfaostat_L199_GCAMDATA_FAOSTAT_CSV") if(command == driver.DECLARE_INPUTS) { return(MODULE_INPUTS) @@ -45,7 +45,36 @@ module_xfaostat_L199_ExportCSV <- function(command, ...) { # adding dummy output ---- - xfaostat_L199_DUMMY <- data.frame() + xfaostat_L199_GCAMDATA_FAOSTAT_CSV <- + tibble(CSV_export = c("GCAMDATA_FAOSTAT_SUA_195Regs_530Items_2010to2019", + "GCAMDATA_FAOSTAT_BiTrade_194Regs_400Items_2010to2020", + "GCAMDATA_FAOSTAT_FBSH_CB_173Regs_118Items_1973to2009", + "GCAMDATA_FAOSTAT_ProdArea_195Regs_271Prod160AreaItems_1973to2020", + "GCAMDATA_FAOSTAT_ProdArea_96Regs_16FodderItems_1973to2020", + "GCAMDATA_FAOSTAT_AnimalStock_202Regs_22Items_1973to2020", + "GCAMDATA_FAOSTAT_ProducerPrice_170Regs_185PrimaryItems_2010to2020", + "FAO_GDP_Deflators", + "GCAMDATA_FAOSTAT_MacroNutrientRate_179Regs_426Items_2010to2019Mean" + )) + + xfaostat_L199_GCAMDATA_FAOSTAT_CSV %>% + add_title("Export CSV to DIR_OUTPUT_CSV") %>% + add_units("NA") %>% + add_comments("Export CSV") %>% + add_precursors("PD", + "Bal_new_all", + "FBSH_CB_wide", + "QCL_PROD", + "QCL_AN_LIVEANIMAL", + "QCL_AN_PRIMARY_MILK", + "QCL_CROP_PRIMARY", + "QCL_FODDERCROP", + "QCL_PRIMARY_PROD_PV", + "TM_bilateral_wide", + "SUA_food_macronutrient_rate") -> + xfaostat_L199_GCAMDATA_FAOSTAT_CSV + + if (OUTPUT_Export_CSV == T) { # Load required inputs ---- @@ -60,7 +89,12 @@ module_xfaostat_L199_ExportCSV <- function(command, ...) { TM_bilateral_wide %>% gather_years() %>% filter(year >= min(FAOSTAT_Hist_Year_FBS)) %>% - filter(value > 0) -> TM_bilateral + filter(value > 0) %>% + FAOSTAT_AREA_RM_NONEXIST() %>% + rename(area_code1 = area_code, area_code = source_code) %>% + FAOSTAT_AREA_RM_NONEXIST() %>% + rename(source_code = area_code, area_code = area_code1) -> + TM_bilateral TM_bilateral %>% # only export quality data years @@ -89,7 +123,6 @@ module_xfaostat_L199_ExportCSV <- function(command, ...) { ) - # SUA and FBS ---- ## *GCAMDATA_FAOSTAT_SUA_195Regs_530Items_2010to2019 ---- diff --git a/R/xfaostat_helper_funcs.R b/R/xfaostat_helper_funcs.R index dee33392..be25c388 100644 --- a/R/xfaostat_helper_funcs.R +++ b/R/xfaostat_helper_funcs.R @@ -32,15 +32,19 @@ FAOSTAT_metadata <- function (code = NULL){ #' FAOSTAT_load_raw_data: load raw csv data #' @description Read csv data and "." in column name is substituted with "_". +#' #' @param DATASETCODE Dataset code in FAO metadata or the name of a csv file. +#' @param GET_MAPPINGCODE if NULL return data if char return other mapping files #' @param DATA_FOLDER Path to the folder storing the data. +#' #' @importFrom readr read_csv #' @importFrom magrittr %>% #' @importFrom assertthat assert_that #' @export FAOSTAT_load_raw_data <- function(DATASETCODE, - DATA_FOLDER = DIR_RAW_DATA_FAOSTAT){ + DATA_FOLDER = DIR_RAW_DATA_FAOSTAT, + GET_MAPPINGCODE = NULL){ assertthat::assert_that(is.character(DATASETCODE)) assertthat::assert_that(is.character(DATA_FOLDER)) @@ -48,25 +52,49 @@ FAOSTAT_load_raw_data <- function(DATASETCODE, metadata <- FAOSTAT_metadata() - # Loop through each code - for (CODE in DATASETCODE) { + if (is.null(GET_MAPPINGCODE)) { + # Loop through each code + for (CODE in DATASETCODE) { + + metadata %>% filter(datasetcode == CODE) -> metadata1 + + zip_file_name <- file.path(DATA_FOLDER, basename(metadata1$filelocation)) + assertthat::assert_that(file.exists(zip_file_name)) + # assuming the csv in zip has the same base name + csv_file_name <- gsub(".zip$", ".csv", basename(zip_file_name)) + + df <- readr::read_csv(unz(zip_file_name, csv_file_name), col_types = NULL) + # Lower case col names and use _ as delimiter + names(df) <- tolower(gsub("\\.| ", "_", names(df))) + # Assigned to parent env + #assign(CODE, df, envir = parent.frame()) + assign(CODE, df, envir = parent.env(environment())) + # Assigned to current env + #assign(CODE, df, envir = .GlobalEnv) + } + } else if(is.character(GET_MAPPINGCODE) == T){ - metadata %>% filter(datasetcode == CODE) -> metadata1 + for (CODE in DATASETCODE) { - zip_file_name <- file.path(DATA_FOLDER, basename(metadata1$filelocation)) - assertthat::assert_that(file.exists(zip_file_name)) - # assuming the csv in zip has the same base name - csv_file_name <- gsub(".zip$", ".csv", basename(zip_file_name)) + metadata %>% filter(datasetcode == CODE) -> metadata1 - df <- readr::read_csv(unz(zip_file_name, csv_file_name), col_types = NULL) - # Lower case col names and use _ as delimiter - names(df) <- tolower(gsub("\\.| ", "_", names(df))) - # Assigned to parent env - #assign(CODE, df, envir = parent.frame()) - assign(CODE, df, envir = parent.env(environment())) - # Assigned to current env - #assign(CODE, df, envir = .GlobalEnv) - } + zip_file_name <- file.path(DATA_FOLDER, basename(metadata1$filelocation)) + assertthat::assert_that(file.exists(zip_file_name)) + # assuming the csv in zip has the same base name + csv_file_name <- gsub(".zip$", ".csv", basename(zip_file_name)) + + csv_file_name <- gsub("All_Data_\\(Normalized\\)", GET_MAPPINGCODE, csv_file_name) + + df <- readr::read_csv(unz(zip_file_name, csv_file_name), col_types = NULL) + # Lower case col names and use _ as delimiter + names(df) <- tolower(gsub("\\.| ", "_", names(df))) + # Assigned to parent env + assign(paste0(CODE, "_", GET_MAPPINGCODE), df, envir = parent.env(environment())) + # Assigned to current env + #assign(paste0(CODE, "_", GET_MAPPINGCODE), df, envir = .GlobalEnv) + } + + } else {stop("Wrong GET_MAPPINGCODE")} } @@ -518,6 +546,7 @@ output_csv_data <- function(gcam_dataset, col_type_nonyear, #' #' @return The same dataframe with balanced world export and import. + GROSS_TRADE_ADJUST <- function(.DF, .MIN_TRADE_PROD_RATIO = 0.01, .Reg_VAR = 'area_code', @@ -533,10 +562,8 @@ GROSS_TRADE_ADJUST <- function(.DF, # Join ExportScaler and ImportScaler left_join( .DF %>% - #group_by_at(vars(all_of(.GROUP_VAR), element)) %>% - #summarise(value = sum(value, na.rm = T), .groups = "drop") %>% spread(element, value) %>% - group_by_at(vars(all_of(.GROUP_VAR))) %>% + dplyr::group_by_at(vars(dplyr::all_of(.GROUP_VAR))) %>% # filter out items with zero world trade or production # and replace na to zero later for scaler replace_na(list(Export = 0, Import = 0, Production = 0)) %>% @@ -550,9 +577,9 @@ GROSS_TRADE_ADJUST <- function(.DF, # the trade scalers will be applied to all regions mutate(ExportScaler = (sum(Export) + sum(Import))/ 2 / sum(Export), ImportScaler = (sum(Export) + sum(Import))/ 2 / sum(Import)) %>% - select(all_of(c(.Reg_VAR, .GROUP_VAR)), ExportScaler, ImportScaler) %>% + select(dplyr::all_of(c(.Reg_VAR, .GROUP_VAR)), ExportScaler, ImportScaler) %>% ungroup(), - by = c(all_of(c(.Reg_VAR, .GROUP_VAR)))) %>% + by = c(dplyr::all_of(c(.Reg_VAR, .GROUP_VAR)))) %>% replace_na(list(ExportScaler = 0, ImportScaler = 0)) %>% # If world export, import, or prod is 0, trade will be zero mutate(value = case_when( diff --git a/R/zaglu_L100.FAO_SUA_PrimaryEquivalent.R b/R/zaglu_L100.FAO_SUA_PrimaryEquivalent.R index 5930fbed..6ddcd0a3 100644 --- a/R/zaglu_L100.FAO_SUA_PrimaryEquivalent.R +++ b/R/zaglu_L100.FAO_SUA_PrimaryEquivalent.R @@ -94,7 +94,7 @@ module_aglu_L100.FAO_SUA_PrimaryEquivalent <- function(command, ...) { GCAMDATA_FAOSTAT_SUA_195Regs_530Items_2010to2019 %>% left_join_error_no_match(Area_Region_Map %>% select(area_code, GCAM_region_ID), by="area_code") %>% group_by(GCAM_region_ID, item_code, element, year) %>% - summarize(value = sum(value)) %>% + summarize(value = sum(value), .groups = "drop") %>% ungroup() -> DF_SUA_Agg @@ -104,7 +104,7 @@ module_aglu_L100.FAO_SUA_PrimaryEquivalent <- function(command, ...) { left_join_error_no_match(Area_Region_Map %>% select(source_code = area_code, source_GCAM_region_ID = GCAM_region_ID), by="source_code") %>% filter(GCAM_region_ID == source_GCAM_region_ID) %>% group_by(GCAM_region_ID, item_code, year) %>% - summarize(value = sum(value)) %>% + summarize(value = sum(value), .groups = "drop") %>% ungroup() %>% mutate(value = -value / 1000.0) -> DF_INTRA_REG_TRADE @@ -140,7 +140,7 @@ module_aglu_L100.FAO_SUA_PrimaryEquivalent <- function(command, ...) { GrossTradeRM %>% mutate(element = All_Bal_element[All_Bal_element == "Import"]), DF_SUA_Agg_TradeAdj) %>% group_by(GCAM_region_ID, item_code, element, year) %>% - summarize(value = sum(value)) %>% + summarize(value = sum(value), .groups = "drop") %>% ungroup() -> DF_SUA_Agg_TradeAdj_TriagAdj @@ -208,7 +208,7 @@ module_aglu_L100.FAO_SUA_PrimaryEquivalent <- function(command, ...) { select(-nest_level) %>% bind_rows(DF_CURR_NEST %>% filter(element == "Production" | element == "Export")) %>% dplyr::group_by_at(vars(-item_code, -value)) %>% - summarize(value=sum(value)) %>% + summarize(value=sum(value), .groups = "drop") %>% ungroup() %>% complete(GCAM_region_ID = GCAM_region_names$GCAM_region_ID, nesting(element, year, APE_comm), fill=list(value=0)) %>% spread(element, value, fill = 0.0) %>% @@ -257,7 +257,7 @@ module_aglu_L100.FAO_SUA_PrimaryEquivalent <- function(command, ...) { spread(is_import, value, fill=0.0) %>% group_by(APE_comm, GCAM_region_ID, year, item_code) %>% summarize(import = sum(`TRUE`), - import_demand = sum(`FALSE`)) %>% + import_demand = sum(`FALSE`), .groups = "drop") %>% ungroup() %>% mutate(Import_Demand_Share = import / import_demand, # replace NA and inf @@ -289,7 +289,7 @@ module_aglu_L100.FAO_SUA_PrimaryEquivalent <- function(command, ...) { group_by(APE_comm, GCAM_region_ID, year, item_code, bal_source) %>% # Clean the bal items summarize(`Regional supply` = sum(value[is_supply]), - `Regional demand` = sum(value[is_demand])) %>% + `Regional demand` = sum(value[is_demand]), .groups = "drop") %>% ungroup() %>% mutate(`Residuals` = `Regional supply` - `Regional demand`) %>% gather(element, value, `Regional supply`, `Regional demand`, `Residuals`) %>% @@ -322,7 +322,7 @@ module_aglu_L100.FAO_SUA_PrimaryEquivalent <- function(command, ...) { spread(is_opening, value, fill=0.0) %>% group_by(APE_comm, GCAM_region_ID, year, item_code) %>% summarize(Ostock = sum(`TRUE`), - Ostock_demand = sum(`FALSE`)) %>% + Ostock_demand = sum(`FALSE`), .groups = "drop") %>% ungroup() %>% mutate(Ostock_Demand_Share = Ostock / Ostock_demand, # The share should be small than 1 @@ -356,7 +356,8 @@ module_aglu_L100.FAO_SUA_PrimaryEquivalent <- function(command, ...) { summarize(`Regional supply` = sum(value[is_supply]), `Regional demand` = sum(value[is_demand]), # using max to guard against missing Closing stocks row - `Stock Variation` = max(value[element == "Closing stocks"], 0) - max(value[element == "Opening stocks"], 0)) %>% + `Stock Variation` = max(value[element == "Closing stocks"], 0) - max(value[element == "Opening stocks"], 0), + .groups = "drop") %>% ungroup() %>% mutate(`Residuals` = `Regional supply` - `Regional demand`) %>% gather(element, value, `Regional supply`, `Regional demand`, `Stock Variation`, `Residuals`) %>% @@ -457,7 +458,7 @@ module_aglu_L100.FAO_SUA_PrimaryEquivalent <- function(command, ...) { item_code = source_item_code) %>% select(-share) %>% group_by(nest_level, APE_comm, GCAM_region_ID, year, item_code, element) %>% - summarize(value = sum(value)) %>% + summarize(value = sum(value), .groups = "drop") %>% ungroup() %>% complete(element=All_Bal_element[All_Bal_element %in% c("Prodution", "Processed")], nesting(nest_level, APE_comm, GCAM_region_ID, year, item_code), fill=list(value=0)) %>% group_by(nest_level, APE_comm, GCAM_region_ID, year, item_code) %>% @@ -477,7 +478,7 @@ module_aglu_L100.FAO_SUA_PrimaryEquivalent <- function(command, ...) { DF_ALL[DF_ALL$nest_level == nest_i, "data", drop=TRUE][[1]], df3_nested[df3_nested$nest_level == nest_i, "data", drop=TRUE][[1]]) %>% group_by(APE_comm, GCAM_region_ID, year, item_code, element) %>% - summarize(value = sum(value)) %>% + summarize(value = sum(value), .groups = "drop") %>% ungroup() -> AGG DF_ALL %>% @@ -495,7 +496,7 @@ module_aglu_L100.FAO_SUA_PrimaryEquivalent <- function(command, ...) { DF_ALL %>% tidyr::unnest(c("data")) %>% group_by(GCAM_region_ID, APE_comm, element, year) %>% - summarize(value = sum(value)) %>% + summarize(value = sum(value), .groups = "drop") %>% ungroup() %>% spread(element, value, fill = 0.0) %>% # Do a final balance cleaning @@ -512,7 +513,7 @@ module_aglu_L100.FAO_SUA_PrimaryEquivalent <- function(command, ...) { left_join_error_no_match(Mapping_SUA_PrimaryEquivalent %>% select(GCAM_commodity, APE_comm) %>% distinct(), by = c("APE_comm")) %>% group_by(GCAM_region_ID, GCAM_commodity, element, year) %>% - summarize(value = sum(value)) %>% + summarize(value = sum(value), .groups = "drop") %>% ungroup() %>% left_join_error_no_match(GCAM_region_names, by=c("GCAM_region_ID")) %>% mutate(element = as.character(element)) %>% @@ -556,56 +557,8 @@ module_aglu_L100.FAO_SUA_PrimaryEquivalent <- function(command, ...) { # 3.1. Helper functions ---- - #' Balance gross trade - #' @description Scale gross export and import in all regions to make them equal at the world level. - #' @param .DF An input dataframe with an element col including Import and Export - #' @param .MIN_TRADE_PROD_RATIO Trade will be removed if world total export or import over production is smaller than .MIN_TRADE_PROD_RATIO (1% default value) - #' @param .Reg_VAR Region variable name; default is ("area_code") - #' @param .GROUP_VAR Group variable; default is ("item_code", "year") - #' @return The same dataframe with balanced world export and import. - - GROSS_TRADE_ADJUST <- function(.DF, - .MIN_TRADE_PROD_RATIO = 0.01, - .Reg_VAR = 'area_code', - .GROUP_VAR = c("item_code", "year")){ - - # assert .DF structure - assertthat::assert_that(all(c("element", .GROUP_VAR) %in% names(.DF))) - assertthat::assert_that(dplyr::is.grouped_df(.DF) == F) - assertthat::assert_that(all(c("Import", "Export", "Production") %in% - c(.DF %>% distinct(element) %>% pull))) - - .DF %>% - # Join ExportScaler and ImportScaler - left_join( - .DF %>% - spread(element, value) %>% - dplyr::group_by_at(vars(dplyr::all_of(.GROUP_VAR))) %>% - # filter out items with zero world trade or production - # and replace na to zero later for scaler - replace_na(list(Export = 0, Import = 0, Production = 0)) %>% - filter(sum(Export) != 0, sum(Import) != 0, sum(Production) != 0) %>% - # world trade should be later than .MIN_TRADE_PROD_RATIO to have meaningful data - # depending on item group, .MIN_TRADE_PROD_RATIO can be set differently - filter(sum(Export) / sum(Production) > .MIN_TRADE_PROD_RATIO) %>% - filter(sum(Import) / sum(Production) > .MIN_TRADE_PROD_RATIO) %>% - # finally, - # use average gross trade value to calculate trade scaler - # the trade scalers will be applied to all regions - mutate(ExportScaler = (sum(Export) + sum(Import))/ 2 / sum(Export), - ImportScaler = (sum(Export) + sum(Import))/ 2 / sum(Import)) %>% - select(dplyr::all_of(c(.Reg_VAR, .GROUP_VAR)), ExportScaler, ImportScaler) %>% - ungroup(), - by = c(dplyr::all_of(c(.Reg_VAR, .GROUP_VAR)))) %>% - replace_na(list(ExportScaler = 0, ImportScaler = 0)) %>% - # If world export, import, or prod is 0, trade will be zero - mutate(value = case_when( - element %in% c("Export") ~ value * ExportScaler, - element %in% c("Import") ~ value * ImportScaler, - TRUE ~ value)) %>% - select(-ExportScaler, -ImportScaler) + # GROSS_TRADE_ADJUST() moved to xfaostat_helper_func.R - } # 3.2. Execution ---- ## a. FBSH_CB aggregate to GCAM commodity and region---- @@ -658,6 +611,7 @@ module_aglu_L100.FAO_SUA_PrimaryEquivalent <- function(command, ...) { # Primary production could be different due to aggregation or inconsistency QCL_PROD_GCAM %>% + filter(year >= min(unique(FBSH_CB_GCAM$year))) %>% # Complete elements in QCL_PROD # also GCAM_commodity because no pork production in Pakistan complete(area, year, GCAM_commodity, diff --git a/data/GCAM_DATA_MAP.rda b/data/GCAM_DATA_MAP.rda index a4d1dd54..2fe1a3b7 100644 Binary files a/data/GCAM_DATA_MAP.rda and b/data/GCAM_DATA_MAP.rda differ diff --git a/data/PREBUILT_DATA.rda b/data/PREBUILT_DATA.rda index 903f15a7..728ba5ef 100644 Binary files a/data/PREBUILT_DATA.rda and b/data/PREBUILT_DATA.rda differ