Merge pull request #27 from kdorheim/further_param_experiments

Trying to Tune Alpha and Change Parameter Bounds

results/alpha_ecs_unc_nmse.txt

@@ -0,0 +1,46 @@
+parameters	values
+beta	0.732
+q10_rh	2.64
+diff	2.2
+S	5
+alpha	1.15015020270593
+
+Objective Function Value: 0.000451
+Counts: 23
+Counts: 23
+Convergence:  0
+Messages: CONVERGENCE: REL_REDUCTION_OF_F <= FACTR*EPSMCH
+
+CO2 MSE: 50.3
+T MSE:   0.0457
+RMSE:    7.09
+T MSE accounting for unc: 0.017
+
+***Key Metrics***
+TCRE: 2.4
+TCR:  2.37
+
+***Historical Warming and ERF***
+GSAT Warming:              0.705
+Ocean Heat Content Change: 459
+Total Aerosol ERF:         -1.24
+WMGHG ERF:                 3.66
+Methane ERF:               0.539
+
+***Future Warming***
+scenario	start	end	GSAT
+ssp119	2021	2040	1
+ssp119	2041	2060	1.35
+ssp119	2081	2100	1.4
+ssp126	2021	2040	1.01
+ssp126	2041	2060	1.57
+ssp126	2081	2100	1.89
+ssp245	2021	2040	1
+ssp245	2041	2060	1.78
+ssp245	2081	2100	2.98
+ssp370	2021	2040	1
+ssp370	2041	2060	1.91
+ssp370	2081	2100	4.14
+ssp585	2021	2040	1.17
+ssp585	2041	2060	2.37
+ssp585	2081	2100	5.36

results/alpha_ecs_unc_nmse_big_box.txt

@@ -0,0 +1,46 @@
+parameters	values
+beta	1.196
+q10_rh	3.52
+diff	2
+S	5
+alpha	0.947871668212818
+
+Objective Function Value: 0.000403
+Counts: 28
+Counts: 28
+Convergence:  0
+Messages: CONVERGENCE: REL_REDUCTION_OF_F <= FACTR*EPSMCH
+
+CO2 MSE: 293
+T MSE:   0.0419
+RMSE:    17.1
+T MSE accounting for unc: 0.0149
+
+***Key Metrics***
+TCRE: 2.36
+TCR:  2.42
+
+***Historical Warming and ERF***
+GSAT Warming:              0.708
+Ocean Heat Content Change: 435
+Total Aerosol ERF:         -1.24
+WMGHG ERF:                 3.26
+Methane ERF:               0.538
+
+***Future Warming***
+scenario	start	end	GSAT
+ssp119	2021	2040	0.869
+ssp119	2041	2060	1.1
+ssp119	2081	2100	1.07
+ssp126	2021	2040	0.887
+ssp126	2041	2060	1.31
+ssp126	2081	2100	1.5
+ssp245	2021	2040	0.899
+ssp245	2041	2060	1.57
+ssp245	2081	2100	2.57
+ssp370	2021	2040	0.912
+ssp370	2041	2060	1.74
+ssp370	2081	2100	3.82
+ssp585	2021	2040	1.05
+ssp585	2041	2060	2.15
+ssp585	2081	2100	4.97

results/alpha_ecs_unc_nmse_very_big_box.txt

@@ -0,0 +1,46 @@
+parameters	values
+beta	1.66
+q10_rh	4.4
+diff	1.8
+S	6
+alpha	0.83005939428505
+
+Objective Function Value: 0.00031
+Counts: 38
+Counts: 38
+Convergence:  0
+Messages: CONVERGENCE: REL_REDUCTION_OF_F <= FACTR*EPSMCH
+
+CO2 MSE: 531
+T MSE:   0.0338
+RMSE:    23
+T MSE accounting for unc: 0.0111
+
+***Key Metrics***
+TCRE: 2.69
+TCR:  2.7
+
+***Historical Warming and ERF***
+GSAT Warming:              0.773
+Ocean Heat Content Change: 438
+Total Aerosol ERF:         -1.24
+WMGHG ERF:                 2.99
+Methane ERF:               0.539
+
+***Future Warming***
+scenario	start	end	GSAT
+ssp119	2021	2040	0.855
+ssp119	2041	2060	1.05
+ssp119	2081	2100	1.05
+ssp126	2021	2040	0.877
+ssp126	2041	2060	1.28
+ssp126	2081	2100	1.48
+ssp245	2021	2040	0.903
+ssp245	2041	2060	1.57
+ssp245	2081	2100	2.61
+ssp370	2021	2040	0.926
+ssp370	2041	2060	1.77
+ssp370	2081	2100	4.01
+ssp585	2021	2040	1.06
+ssp585	2041	2060	2.18
+ssp585	2081	2100	5.21

scripts/error_functions.R

@@ -111,6 +111,24 @@ nmse_unc <- function(x, x_upper, x_lower, y) {
            sum((x[!is.na(x)])^2))
 }
 
+# mvsse - function to find the mean variance-standardized squared error
+#
+# args: 
+#   x  - observed values
+#   sd - standard deviation of observed values
+#   y  - predicted values
+#
+# returns: numeric vector length 1 containing the NMSE between predicted and
+#          observed values
+mvsse <- function(x, sd, y) {
+  vsse <- (x - y)^2 / (sd)^2
+  return(mean(vsse))
+}
+
+
+#######################################
+### Single-Variable Error Functions ###
+#######################################
 
 # get_var_mse: function to find MSE between observed and predicted data for
 #              a given variable
@@ -156,6 +174,35 @@ get_var_mse_unc <- function(obs_data, hector_data, var, yrs, mse_fn) {
   return(mse_fn(x = x, x_upper = x_upper, x_lower = x_lower, y = y))
 }
 
+
+# get_var_mvsse: function to find MVSSE between observed and predicted data
+#                  for a given variable
+#
+# args: 
+#   obs_data    - data frame of observed data formatted like Hector data frame
+#   hector_data - data frame outputted by Hector
+#   var         - variable name
+#   yrs         - vector of years for finding MSE
+#
+# Returns: MSE between predicted and observed data for var
+#
+# Note: Assumes observed data contains symmetric upper and lower bounds 1 SD 
+#       away from actual value
+get_var_mse_unc <- function(obs_data, hector_data, var, yrs, mse_fn) {
+  x       <- filter(obs_data, year %in% yrs & variable == var)$value
+  x_upper <- filter(obs_data, year %in% yrs & variable == var)$upper
+  sd      <- x_upper - x
+  y       <- filter(hector_data, year %in% yrs & variable == var)$value
+
+  return(mvsse(x = x, sd = sd, y = y))
+}
+
+
+#######################################
+### Error Functions for Optim Usage ###
+#######################################
+
+
 # mean_T_CO2_mse: function to find the mean of the temperature and CO2 MSEs 
 #                 between observed and predicted data for a given variable
 #

scripts/graph_comparison_plots.R

@@ -20,9 +20,9 @@ TEMP_PATH <-
             "HadCRUT.5.0.2.0.analysis.summary_series.global.annual.csv")
 
 INI_FILE <- system.file("input/hector_ssp245.ini", package = "hector")
-PARAMS <- c(BETA(), Q10_RH(), DIFFUSIVITY(), ECS())
+PARAMS <- c(BETA(), Q10_RH(), DIFFUSIVITY(), ECS(), AERO_SCALE())
 
-OUTPUT <- file.path(RESULTS_DIR, "ecs_initial_comparison_plots.jpeg")
+OUTPUT <- file.path(RESULTS_DIR, "alpha_initial_comparison_plots.jpeg")
 
 
 source(file.path(SCRIPTS_DIR, "major_functions.R"))
@@ -45,36 +45,57 @@ default_data <- run_hector(ini_file = INI_FILE,
                            vars = c(GMST(), CONCENTRATIONS_CO2()))
 default_data$scenario <- "Hector - Default Fit"
 
-nmse_data <- run_hector(ini_file = INI_FILE,
-                        params = PARAMS,
-                        vals = c(0.732, 2.64, 2.4, 3),
-                        yrs = 1750:2014,
-                        vars = c(GMST(), CONCENTRATIONS_CO2()))
-nmse_data$scenario <- "Hector - Fit to NMSEs w/ unc"
-
-nmse_bb_data <- run_hector(ini_file = INI_FILE,
-                        params = PARAMS,
-                        vals = c(1.084, 3.52, 2, 3),
-                        yrs = 1750:2014,
-                        vars = c(GMST(), CONCENTRATIONS_CO2()))
 
-nmse_bb_data$scenario <- "Hector - Fit to NMSEs w/ unc, big box"
+# nmse_data <- run_hector(ini_file = INI_FILE,
+#                         params = PARAMS,
+#                         vals = c(0.732, 2.64, 2.4, 3, 1),
+#                         yrs = 1750:2014,
+#                         vars = c(GMST(), CONCENTRATIONS_CO2()))
+# nmse_data$scenario <- "Hector - Fit to NMSEs w/ unc"
+# 
+# nmse_bb_data <- run_hector(ini_file = INI_FILE,
+#                         params = PARAMS,
+#                         vals = c(1.084, 3.52, 2, 3, 1),
+#                         yrs = 1750:2014,
+#                         vars = c(GMST(), CONCENTRATIONS_CO2()))
+# nmse_bb_data$scenario <- "Hector - Fit to NMSEs w/ unc, big box"
 
 nmse_ecs_data <- run_hector(ini_file = INI_FILE,
                         params = PARAMS,
-                        vals = c(0.732, 2.24, 2.4, 5),
+                        vals = c(0.732, 2.24, 2.4, 5, 1),
                         yrs = 1750:2014,
                         vars = c(GMST(), CONCENTRATIONS_CO2()))
 nmse_ecs_data$scenario <- "Hector - NMSEs w/ unc & Tuning S"
 
 nmse_bb_ecs_data <- run_hector(ini_file = INI_FILE,
                                   params = PARAMS,
-                                  vals = c(1.069, 3.52, 2, 5),
+                                  vals = c(1.069, 3.52, 2, 5, 1),
                                   yrs = 1750:2014,
                                   vars = c(GMST(), CONCENTRATIONS_CO2()))
 nmse_bb_ecs_data$scenario <- "Hector - NMSEs w/ unc, big box & Tuning S"
 
-hector_data <- rbind(default_data, nmse_data, nmse_bb_data, nmse_ecs_data, nmse_bb_ecs_data)
+alpha_data <- run_hector(ini_file = INI_FILE,
+                               params = PARAMS,
+                               vals = c(0.732, 2.24, 2.4, 5, 1.15),
+                               yrs = 1750:2014,
+                               vars = c(GMST(), CONCENTRATIONS_CO2()))
+alpha_data$scenario <- "Hector - NMSEs w/ unc & Tuning S and Alpha"
+
+alpha_bb_data <- run_hector(ini_file = INI_FILE,
+                               params = PARAMS,
+                               vals = c(1.196, 3.52, 2, 5, 0.948),
+                               yrs = 1750:2014,
+                               vars = c(GMST(), CONCENTRATIONS_CO2()))
+alpha_bb_data$scenario <- "Hector - NMSEs w/ unc, big box & Tuning S and Alpha"
+
+alpha_vbb_data <- run_hector(ini_file = INI_FILE,
+                            params = PARAMS,
+                            vals = c(1.66, 4.4, 1.8, 6, 0.83),
+                            yrs = 1750:2014,
+                            vars = c(GMST(), CONCENTRATIONS_CO2()))
+alpha_vbb_data$scenario <- "Hector - NMSEs w/ unc, very big box & Tuning S and Alpha"
+
+hector_data <- rbind(default_data, nmse_ecs_data, nmse_bb_ecs_data, alpha_data, alpha_bb_data, alpha_vbb_data)
 hector_data$lower <- hector_data$value
 hector_data$upper <- hector_data$value
 
@@ -89,4 +110,4 @@ ggplot(data = comb_data, aes(x = year, y = value, color = scenario)) +
   geom_line() +
   facet_wrap(~ variable, scales = "free") +
   ggtitle("Comparing Parameterizations")
-ggsave(OUTPUT, width = 15)
+ggsave(OUTPUT, width = 16)

scripts/major_functions.R

@@ -253,7 +253,7 @@ run_hector <- function(ini_file, params, vals, yrs, vars, include_unc = F) {
   shutdown(core)
 
   # Rescaling temperatures (if applicable)
-  if (GMST() %in% params) {
+  if (GMST() %in% vars) {
     data <- rel_to_interval(data = data, var = GMST(), start = 1961, end = 1990)
   }
 

scripts/more_params_unc_nmse.R

@@ -0,0 +1,76 @@
+# Script to use normalized the T and CO2 MSEs while accounting for T uncertainty
+# Also includes ECS and alpha as params to optimize over
+# Author: Peter Scully
+# Date: 6/20/24
+
+### Constants and Imports ###
+
+# Importing libraries
+library(hector)
+
+# Setting up file paths
+COMP_DATA_DIR <- file.path(here::here(), "comparison_data")
+SCRIPTS_DIR <- file.path(here::here(), "scripts")
+RESULTS_DIR <- file.path(here::here(), "results")
+
+CO2_PATH <- file.path(COMP_DATA_DIR,
+                      "Supplementary_Table_UoM_GHGConcentrations-1-1-0_annualmeans_v23March2017.csv")
+TEMP_PATH <-
+  file.path(COMP_DATA_DIR,
+            "HadCRUT.5.0.2.0.analysis.summary_series.global.annual.csv")
+
+INI_FILE <- system.file("input/hector_ssp245.ini", package = "hector")
+PARAMS <- c(BETA(), Q10_RH(), DIFFUSIVITY(), ECS(), AERO_SCALE())
+
+OUTPUT <- file.path(RESULTS_DIR, "alpha_ecs_unc_nmse_very_big_box.txt")
+
+
+source(file.path(SCRIPTS_DIR, "major_functions.R"))
+
+### Getting observational data ###
+co2_data <- get_co2_data(CO2_PATH, include_unc = TRUE)
+temp_data <- get_temp_data(TEMP_PATH, include_unc = TRUE)
+obs_data <- rbind(co2_data, temp_data)
+
+### Calling optim ###
+best_pars <- run_optim(obs_data = obs_data,
+                       ini_file = INI_FILE,
+                       params = PARAMS,
+                       lower = c(0,               2.2 - 0.44 * 5, 2.3 - 0.1 * 5, 2 - 1, 0),
+                       upper = c(0.5 + 0.232 * 5, 2.2 + 0.44 * 5, 2.3 + 0.1 * 5, 5 + 1, 3),
+                       yrs = 1750:2014,
+                       vars = c(GMST(), CONCENTRATIONS_CO2()),
+                       error_fn = mean_T_CO2_nmse_unc,
+                       include_unc = T,
+                       method = "L-BFGS-B",
+                       output_file = OUTPUT)
+
+### Outputting individual MSEs ###
+hector_data <- run_hector(ini_file = INI_FILE, 
+                          params = PARAMS, 
+                          vals = best_pars, 
+                          yrs = 1750:2014, 
+                          vars = c(GMST(), CONCENTRATIONS_CO2()))
+
+T_mse <- get_var_mse(obs_data = obs_data, 
+                     hector_data = hector_data, 
+                     var = GMST(), 
+                     yrs = 1850:2014)
+CO2_mse <- get_var_mse(obs_data = obs_data, 
+                       hector_data = hector_data, 
+                       var = CONCENTRATIONS_CO2(), 
+                       yrs = c(1750, 1850:2014))
+T_mse_unc <- get_var_mse_unc(obs_data = obs_data,
+                             hector_data = hector_data,
+                             var = GMST(),
+                             yrs = 1850:2014,
+                             mse_fn = mse_unc)
+
+write_metric("CO2 MSE:", CO2_mse, OUTPUT)
+write_metric("T MSE:  ", T_mse, OUTPUT)
+write_metric("RMSE:   ", sqrt(mean(CO2_mse, T_mse)), OUTPUT) # not 100% sure this is how we want to calculate this
+write_metric("T MSE accounting for unc:", T_mse_unc, OUTPUT)
+write("", OUTPUT, append = TRUE)
+
+### Outputting table metrics ###
+calc_table_metrics(PARAMS, best_pars, OUTPUT)