diff --git a/results/alpha_ecs_unc_nmse.txt b/results/alpha_ecs_unc_nmse.txt new file mode 100644 index 0000000..764b6ec --- /dev/null +++ b/results/alpha_ecs_unc_nmse.txt @@ -0,0 +1,46 @@ +parameters values +beta 0.732 +q10_rh 2.64 +diff 2.2 +S 5 +alpha 1.15015020270593 + +Objective Function Value: 0.000451 +Counts: 23 +Counts: 23 +Convergence: 0 +Messages: CONVERGENCE: REL_REDUCTION_OF_F <= FACTR*EPSMCH + +CO2 MSE: 50.3 +T MSE: 0.0457 +RMSE: 7.09 +T MSE accounting for unc: 0.017 + +***Key Metrics*** +TCRE: 2.4 +TCR: 2.37 + +***Historical Warming and ERF*** +GSAT Warming: 0.705 +Ocean Heat Content Change: 459 +Total Aerosol ERF: -1.24 +WMGHG ERF: 3.66 +Methane ERF: 0.539 + +***Future Warming*** +scenario start end GSAT +ssp119 2021 2040 1 +ssp119 2041 2060 1.35 +ssp119 2081 2100 1.4 +ssp126 2021 2040 1.01 +ssp126 2041 2060 1.57 +ssp126 2081 2100 1.89 +ssp245 2021 2040 1 +ssp245 2041 2060 1.78 +ssp245 2081 2100 2.98 +ssp370 2021 2040 1 +ssp370 2041 2060 1.91 +ssp370 2081 2100 4.14 +ssp585 2021 2040 1.17 +ssp585 2041 2060 2.37 +ssp585 2081 2100 5.36 diff --git a/results/alpha_ecs_unc_nmse_big_box.txt b/results/alpha_ecs_unc_nmse_big_box.txt new file mode 100644 index 0000000..7b0b512 --- /dev/null +++ b/results/alpha_ecs_unc_nmse_big_box.txt @@ -0,0 +1,46 @@ +parameters values +beta 1.196 +q10_rh 3.52 +diff 2 +S 5 +alpha 0.947871668212818 + +Objective Function Value: 0.000403 +Counts: 28 +Counts: 28 +Convergence: 0 +Messages: CONVERGENCE: REL_REDUCTION_OF_F <= FACTR*EPSMCH + +CO2 MSE: 293 +T MSE: 0.0419 +RMSE: 17.1 +T MSE accounting for unc: 0.0149 + +***Key Metrics*** +TCRE: 2.36 +TCR: 2.42 + +***Historical Warming and ERF*** +GSAT Warming: 0.708 +Ocean Heat Content Change: 435 +Total Aerosol ERF: -1.24 +WMGHG ERF: 3.26 +Methane ERF: 0.538 + +***Future Warming*** +scenario start end GSAT +ssp119 2021 2040 0.869 +ssp119 2041 2060 1.1 +ssp119 2081 2100 1.07 +ssp126 2021 2040 0.887 +ssp126 2041 2060 1.31 +ssp126 2081 2100 1.5 +ssp245 2021 2040 0.899 +ssp245 2041 2060 1.57 +ssp245 2081 2100 2.57 +ssp370 2021 2040 0.912 +ssp370 2041 2060 1.74 +ssp370 2081 2100 3.82 +ssp585 2021 2040 1.05 +ssp585 2041 2060 2.15 +ssp585 2081 2100 4.97 diff --git a/results/alpha_ecs_unc_nmse_very_big_box.txt b/results/alpha_ecs_unc_nmse_very_big_box.txt new file mode 100644 index 0000000..58ba0ba --- /dev/null +++ b/results/alpha_ecs_unc_nmse_very_big_box.txt @@ -0,0 +1,46 @@ +parameters values +beta 1.66 +q10_rh 4.4 +diff 1.8 +S 6 +alpha 0.83005939428505 + +Objective Function Value: 0.00031 +Counts: 38 +Counts: 38 +Convergence: 0 +Messages: CONVERGENCE: REL_REDUCTION_OF_F <= FACTR*EPSMCH + +CO2 MSE: 531 +T MSE: 0.0338 +RMSE: 23 +T MSE accounting for unc: 0.0111 + +***Key Metrics*** +TCRE: 2.69 +TCR: 2.7 + +***Historical Warming and ERF*** +GSAT Warming: 0.773 +Ocean Heat Content Change: 438 +Total Aerosol ERF: -1.24 +WMGHG ERF: 2.99 +Methane ERF: 0.539 + +***Future Warming*** +scenario start end GSAT +ssp119 2021 2040 0.855 +ssp119 2041 2060 1.05 +ssp119 2081 2100 1.05 +ssp126 2021 2040 0.877 +ssp126 2041 2060 1.28 +ssp126 2081 2100 1.48 +ssp245 2021 2040 0.903 +ssp245 2041 2060 1.57 +ssp245 2081 2100 2.61 +ssp370 2021 2040 0.926 +ssp370 2041 2060 1.77 +ssp370 2081 2100 4.01 +ssp585 2021 2040 1.06 +ssp585 2041 2060 2.18 +ssp585 2081 2100 5.21 diff --git a/results/alpha_initial_comparison_plots.jpeg b/results/alpha_initial_comparison_plots.jpeg new file mode 100644 index 0000000..ec18dff Binary files /dev/null and b/results/alpha_initial_comparison_plots.jpeg differ diff --git a/results/alpha_sens.rda b/results/alpha_sens.rda new file mode 100644 index 0000000..370d62e Binary files /dev/null and b/results/alpha_sens.rda differ diff --git a/results/alpha_sens_plots.jpeg b/results/alpha_sens_plots.jpeg new file mode 100644 index 0000000..5114a4e Binary files /dev/null and b/results/alpha_sens_plots.jpeg differ diff --git a/scripts/error_functions.R b/scripts/error_functions.R index fd167e2..7000a9b 100644 --- a/scripts/error_functions.R +++ b/scripts/error_functions.R @@ -111,6 +111,24 @@ nmse_unc <- function(x, x_upper, x_lower, y) { sum((x[!is.na(x)])^2)) } +# mvsse - function to find the mean variance-standardized squared error +# +# args: +# x - observed values +# sd - standard deviation of observed values +# y - predicted values +# +# returns: numeric vector length 1 containing the NMSE between predicted and +# observed values +mvsse <- function(x, sd, y) { + vsse <- (x - y)^2 / (sd)^2 + return(mean(vsse)) +} + + +####################################### +### Single-Variable Error Functions ### +####################################### # get_var_mse: function to find MSE between observed and predicted data for # a given variable @@ -156,6 +174,35 @@ get_var_mse_unc <- function(obs_data, hector_data, var, yrs, mse_fn) { return(mse_fn(x = x, x_upper = x_upper, x_lower = x_lower, y = y)) } + +# get_var_mvsse: function to find MVSSE between observed and predicted data +# for a given variable +# +# args: +# obs_data - data frame of observed data formatted like Hector data frame +# hector_data - data frame outputted by Hector +# var - variable name +# yrs - vector of years for finding MSE +# +# Returns: MSE between predicted and observed data for var +# +# Note: Assumes observed data contains symmetric upper and lower bounds 1 SD +# away from actual value +get_var_mse_unc <- function(obs_data, hector_data, var, yrs, mse_fn) { + x <- filter(obs_data, year %in% yrs & variable == var)$value + x_upper <- filter(obs_data, year %in% yrs & variable == var)$upper + sd <- x_upper - x + y <- filter(hector_data, year %in% yrs & variable == var)$value + + return(mvsse(x = x, sd = sd, y = y)) +} + + +####################################### +### Error Functions for Optim Usage ### +####################################### + + # mean_T_CO2_mse: function to find the mean of the temperature and CO2 MSEs # between observed and predicted data for a given variable # diff --git a/scripts/graph_comparison_plots.R b/scripts/graph_comparison_plots.R index 96ae0f5..78dfbcd 100644 --- a/scripts/graph_comparison_plots.R +++ b/scripts/graph_comparison_plots.R @@ -20,9 +20,9 @@ TEMP_PATH <- "HadCRUT.5.0.2.0.analysis.summary_series.global.annual.csv") INI_FILE <- system.file("input/hector_ssp245.ini", package = "hector") -PARAMS <- c(BETA(), Q10_RH(), DIFFUSIVITY(), ECS()) +PARAMS <- c(BETA(), Q10_RH(), DIFFUSIVITY(), ECS(), AERO_SCALE()) -OUTPUT <- file.path(RESULTS_DIR, "ecs_initial_comparison_plots.jpeg") +OUTPUT <- file.path(RESULTS_DIR, "alpha_initial_comparison_plots.jpeg") source(file.path(SCRIPTS_DIR, "major_functions.R")) @@ -45,36 +45,57 @@ default_data <- run_hector(ini_file = INI_FILE, vars = c(GMST(), CONCENTRATIONS_CO2())) default_data$scenario <- "Hector - Default Fit" -nmse_data <- run_hector(ini_file = INI_FILE, - params = PARAMS, - vals = c(0.732, 2.64, 2.4, 3), - yrs = 1750:2014, - vars = c(GMST(), CONCENTRATIONS_CO2())) -nmse_data$scenario <- "Hector - Fit to NMSEs w/ unc" - -nmse_bb_data <- run_hector(ini_file = INI_FILE, - params = PARAMS, - vals = c(1.084, 3.52, 2, 3), - yrs = 1750:2014, - vars = c(GMST(), CONCENTRATIONS_CO2())) -nmse_bb_data$scenario <- "Hector - Fit to NMSEs w/ unc, big box" +# nmse_data <- run_hector(ini_file = INI_FILE, +# params = PARAMS, +# vals = c(0.732, 2.64, 2.4, 3, 1), +# yrs = 1750:2014, +# vars = c(GMST(), CONCENTRATIONS_CO2())) +# nmse_data$scenario <- "Hector - Fit to NMSEs w/ unc" +# +# nmse_bb_data <- run_hector(ini_file = INI_FILE, +# params = PARAMS, +# vals = c(1.084, 3.52, 2, 3, 1), +# yrs = 1750:2014, +# vars = c(GMST(), CONCENTRATIONS_CO2())) +# nmse_bb_data$scenario <- "Hector - Fit to NMSEs w/ unc, big box" nmse_ecs_data <- run_hector(ini_file = INI_FILE, params = PARAMS, - vals = c(0.732, 2.24, 2.4, 5), + vals = c(0.732, 2.24, 2.4, 5, 1), yrs = 1750:2014, vars = c(GMST(), CONCENTRATIONS_CO2())) nmse_ecs_data$scenario <- "Hector - NMSEs w/ unc & Tuning S" nmse_bb_ecs_data <- run_hector(ini_file = INI_FILE, params = PARAMS, - vals = c(1.069, 3.52, 2, 5), + vals = c(1.069, 3.52, 2, 5, 1), yrs = 1750:2014, vars = c(GMST(), CONCENTRATIONS_CO2())) nmse_bb_ecs_data$scenario <- "Hector - NMSEs w/ unc, big box & Tuning S" -hector_data <- rbind(default_data, nmse_data, nmse_bb_data, nmse_ecs_data, nmse_bb_ecs_data) +alpha_data <- run_hector(ini_file = INI_FILE, + params = PARAMS, + vals = c(0.732, 2.24, 2.4, 5, 1.15), + yrs = 1750:2014, + vars = c(GMST(), CONCENTRATIONS_CO2())) +alpha_data$scenario <- "Hector - NMSEs w/ unc & Tuning S and Alpha" + +alpha_bb_data <- run_hector(ini_file = INI_FILE, + params = PARAMS, + vals = c(1.196, 3.52, 2, 5, 0.948), + yrs = 1750:2014, + vars = c(GMST(), CONCENTRATIONS_CO2())) +alpha_bb_data$scenario <- "Hector - NMSEs w/ unc, big box & Tuning S and Alpha" + +alpha_vbb_data <- run_hector(ini_file = INI_FILE, + params = PARAMS, + vals = c(1.66, 4.4, 1.8, 6, 0.83), + yrs = 1750:2014, + vars = c(GMST(), CONCENTRATIONS_CO2())) +alpha_vbb_data$scenario <- "Hector - NMSEs w/ unc, very big box & Tuning S and Alpha" + +hector_data <- rbind(default_data, nmse_ecs_data, nmse_bb_ecs_data, alpha_data, alpha_bb_data, alpha_vbb_data) hector_data$lower <- hector_data$value hector_data$upper <- hector_data$value @@ -89,4 +110,4 @@ ggplot(data = comb_data, aes(x = year, y = value, color = scenario)) + geom_line() + facet_wrap(~ variable, scales = "free") + ggtitle("Comparing Parameterizations") -ggsave(OUTPUT, width = 15) \ No newline at end of file +ggsave(OUTPUT, width = 16) \ No newline at end of file diff --git a/scripts/major_functions.R b/scripts/major_functions.R index 12c413c..779378c 100644 --- a/scripts/major_functions.R +++ b/scripts/major_functions.R @@ -253,7 +253,7 @@ run_hector <- function(ini_file, params, vals, yrs, vars, include_unc = F) { shutdown(core) # Rescaling temperatures (if applicable) - if (GMST() %in% params) { + if (GMST() %in% vars) { data <- rel_to_interval(data = data, var = GMST(), start = 1961, end = 1990) } diff --git a/scripts/more_params_unc_nmse.R b/scripts/more_params_unc_nmse.R new file mode 100644 index 0000000..a292d41 --- /dev/null +++ b/scripts/more_params_unc_nmse.R @@ -0,0 +1,76 @@ +# Script to use normalized the T and CO2 MSEs while accounting for T uncertainty +# Also includes ECS and alpha as params to optimize over +# Author: Peter Scully +# Date: 6/20/24 + +### Constants and Imports ### + +# Importing libraries +library(hector) + +# Setting up file paths +COMP_DATA_DIR <- file.path(here::here(), "comparison_data") +SCRIPTS_DIR <- file.path(here::here(), "scripts") +RESULTS_DIR <- file.path(here::here(), "results") + +CO2_PATH <- file.path(COMP_DATA_DIR, + "Supplementary_Table_UoM_GHGConcentrations-1-1-0_annualmeans_v23March2017.csv") +TEMP_PATH <- + file.path(COMP_DATA_DIR, + "HadCRUT.5.0.2.0.analysis.summary_series.global.annual.csv") + +INI_FILE <- system.file("input/hector_ssp245.ini", package = "hector") +PARAMS <- c(BETA(), Q10_RH(), DIFFUSIVITY(), ECS(), AERO_SCALE()) + +OUTPUT <- file.path(RESULTS_DIR, "alpha_ecs_unc_nmse_very_big_box.txt") + + +source(file.path(SCRIPTS_DIR, "major_functions.R")) + +### Getting observational data ### +co2_data <- get_co2_data(CO2_PATH, include_unc = TRUE) +temp_data <- get_temp_data(TEMP_PATH, include_unc = TRUE) +obs_data <- rbind(co2_data, temp_data) + +### Calling optim ### +best_pars <- run_optim(obs_data = obs_data, + ini_file = INI_FILE, + params = PARAMS, + lower = c(0, 2.2 - 0.44 * 5, 2.3 - 0.1 * 5, 2 - 1, 0), + upper = c(0.5 + 0.232 * 5, 2.2 + 0.44 * 5, 2.3 + 0.1 * 5, 5 + 1, 3), + yrs = 1750:2014, + vars = c(GMST(), CONCENTRATIONS_CO2()), + error_fn = mean_T_CO2_nmse_unc, + include_unc = T, + method = "L-BFGS-B", + output_file = OUTPUT) + +### Outputting individual MSEs ### +hector_data <- run_hector(ini_file = INI_FILE, + params = PARAMS, + vals = best_pars, + yrs = 1750:2014, + vars = c(GMST(), CONCENTRATIONS_CO2())) + +T_mse <- get_var_mse(obs_data = obs_data, + hector_data = hector_data, + var = GMST(), + yrs = 1850:2014) +CO2_mse <- get_var_mse(obs_data = obs_data, + hector_data = hector_data, + var = CONCENTRATIONS_CO2(), + yrs = c(1750, 1850:2014)) +T_mse_unc <- get_var_mse_unc(obs_data = obs_data, + hector_data = hector_data, + var = GMST(), + yrs = 1850:2014, + mse_fn = mse_unc) + +write_metric("CO2 MSE:", CO2_mse, OUTPUT) +write_metric("T MSE: ", T_mse, OUTPUT) +write_metric("RMSE: ", sqrt(mean(CO2_mse, T_mse)), OUTPUT) # not 100% sure this is how we want to calculate this +write_metric("T MSE accounting for unc:", T_mse_unc, OUTPUT) +write("", OUTPUT, append = TRUE) + +### Outputting table metrics ### +calc_table_metrics(PARAMS, best_pars, OUTPUT) \ No newline at end of file diff --git a/scripts/other/alpha_testing.R b/scripts/other/alpha_testing.R new file mode 100644 index 0000000..c3c412a --- /dev/null +++ b/scripts/other/alpha_testing.R @@ -0,0 +1,65 @@ +# Script to get data from ~30 runs with different alpha values +# Author: Peter Scully +# Date: 6/21/24 + +### Constants and Imports ### + +# Importing libraries +library(hector) +library(ggplot2) + +# Setting up file paths +COMP_DATA_DIR <- file.path(here::here(), "comparison_data") +SCRIPTS_DIR <- file.path(here::here(), "scripts") +RESULTS_DIR <- file.path(here::here(), "results") + +CO2_PATH <- file.path(COMP_DATA_DIR, + "Supplementary_Table_UoM_GHGConcentrations-1-1-0_annualmeans_v23March2017.csv") +TEMP_PATH <- + file.path(COMP_DATA_DIR, + "HadCRUT.5.0.2.0.analysis.summary_series.global.annual.csv") + +INI_FILE <- system.file("input/hector_ssp245.ini", package = "hector") + +OUTPUT <- file.path(RESULTS_DIR, "alpha_sens_plots.jpeg") + + +source(file.path(SCRIPTS_DIR, "major_functions.R")) + +### Getting observational data ### +co2_data <- get_co2_data(CO2_PATH) +temp_data <- get_temp_data(TEMP_PATH) +temp_data <- filter(temp_data, year <= 2014) + +obs_data <- rbind(co2_data, temp_data) + +### Running Hector ### +all_data <- run_hector(ini_file = INI_FILE, + params = NULL, + vals = NULL, + yrs = 1750:2014, + vars = c(GMST(), CONCENTRATIONS_CO2())) +all_data$scenario <- "default" + +# This is probably pretty inefficient +for (alpha in seq(0, 3, by=0.1)) { + curr_data <- run_hector(ini_file = INI_FILE, + params = AERO_SCALE(), + vals = alpha, + yrs = 1750:2014, + vars = c(GMST(), CONCENTRATIONS_CO2())) + curr_data$scenario <- paste("Alpha =", alpha) + all_data <- rbind(all_data, curr_data) +} + +# Saving data +save(all_data, file = file.path(RESULTS_DIR, "alpha_sens.rda")) + +# Plotting +comb_data <- rbind(obs_data, all_data) + +ggplot(data = comb_data, aes(x = year, y = value, color = scenario)) + + geom_line() + + facet_wrap(~ variable, scales = "free") + + ggtitle("Comparing Alpha Values") +ggsave(OUTPUT, width = 15) \ No newline at end of file diff --git a/tests/error_tests.R b/tests/error_tests.R index e34385b..ce8b278 100644 --- a/tests/error_tests.R +++ b/tests/error_tests.R @@ -73,7 +73,7 @@ mse_unc_tests <- function() { } # Trying to use nmse_unc -mse_unc_tests <- function() { +nmse_unc_tests <- function() { assert_that(nmse_unc(10, 15, 5, 10) == 0) assert_that(nmse_unc(10, 15, 5, 0) == .25) assert_that(nmse_unc(10, 15, 5, 20) == .25) @@ -87,6 +87,9 @@ mse_unc_tests <- function() { c(10, 7, 28)) == 9 / 625) } + +# Calling all testing functions mse_tests() get_var_mse_tests() -mse_unc_tests() \ No newline at end of file +mse_unc_tests() +nmse_unc_tests() \ No newline at end of file