Skip to content

Commit

Permalink
Merge pull request #34 from kdorheim/try_mvsse
Browse files Browse the repository at this point in the history 
Try mvsse
  • Loading branch information
@ptrscll
ptrscll authored Jul 3, 2024
2 parents 39e2bc5 + 1ae22d4 commit 7da1e04
Show file tree
Hide file tree
Showing 7 changed files with 230 additions and 13 deletions.
48 changes: 48 additions & 0 deletions results/13_alpha_ecs_ohc_mvsse.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
parameters values
beta 0.526231809997589
q10_rh 2.31055396561195
diff 1.042
S 2.82788331898928
alpha 1.40507672116753

Objective Function Value: 91.3
Counts: 104
Counts: 104
Convergence: 0
Messages: CONVERGENCE: REL_REDUCTION_OF_F <= FACTR*EPSMCH

CO2 MSE: 3.4
T MSE: 0.102
RMSE: 1.84
CO2 MVSSE: 236
T MVSSE: 37.3
OHC MVSSE: 0.385

***Key Metrics***
TCRE: 1.62
TCR: 1.92

***Historical Warming and ERF***
GSAT Warming: 0.502
Ocean Heat Content Change: 291
Total Aerosol ERF: -1.24
WMGHG ERF: 3.88
Methane ERF: 0.536

***Future Warming***
scenario start end GSAT
ssp119 2021 2040 0.904
ssp119 2041 2060 1.13
ssp119 2081 2100 0.968
ssp126 2021 2040 0.907
ssp126 2041 2060 1.33
ssp126 2081 2100 1.36
ssp245 2021 2040 0.872
ssp245 2041 2060 1.49
ssp245 2081 2100 2.3
ssp370 2021 2040 0.854
ssp370 2041 2060 1.57
ssp370 2081 2100 3.23
ssp585 2021 2040 1.05
ssp585 2041 2060 2.04
ssp585 2081 2100 4.3
13 changes: 5 additions & 8 deletions results/default_stats.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,11 @@ Unnamed Hector core NA beta 0.53 (unitless)
Unnamed Hector core NA q10_rh 1.76 (unitless)
Unnamed Hector core NA diff 2.38 cm2/s

Mean of T, CO2 MSEs: 2.31
Mean of T, CO2 NMSEs: 0.000948
Mean of T, CO2 NMSEs (with unc): 0.000313
scenario year variable value units
Unnamed Hector core NA beta 0.53 (unitless)
Unnamed Hector core NA q10_rh 1.76 (unitless)
Unnamed Hector core NA diff 2.38 cm2/s

Mean of T, CO2 MSEs: 2.31
Mean of T, CO2 NMSEs: 0.000948
Mean of T, CO2 NMSEs (with unc): 0.000313
Mean of T, CO2, OHC NMSEs (with unc): 0.00022
Mean of T, CO2, OHC MVSSEs: 113

Mean of smooth T, CO2 MSEs (k = 3 ): 2.3
Mean of smooth T, CO2 NMSEs (k = 3 ): 0.000864
Expand All @@ -33,6 +26,10 @@ T NMSE: 0.0019
T NMSE with unc: 0.000625
OHC NMSE with unc: 3.43e-05

CO2 MVSSE: 318
T MVSSE: 19.5
OHC MVSSE: 1.02

***Key Metrics***
TCRE: 1.51
TCR: 1.77
Expand Down
96 changes: 96 additions & 0 deletions scripts/alpha_ohc_mvsse.R
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
# Script for experiment 13
# Script to optimize over T, OHC and CO2 MVSSEs
# Also includes ECS and alpha as params to optimize over
# Uses OHC range from Matilda table rather than manuscript script
# Author: Peter Scully
# Date: 6/25/24

### Constants and Imports ###

# Importing libraries
library(hector)

# Setting up file paths
COMP_DATA_DIR <- file.path(here::here(), "comparison_data")
SCRIPTS_DIR <- file.path(here::here(), "scripts")
RESULTS_DIR <- file.path(here::here(), "results")

CO2_PATH <- file.path(COMP_DATA_DIR,
"Supplementary_Table_UoM_GHGConcentrations-1-1-0_annualmeans_v23March2017.csv")
TEMP_PATH <-
file.path(COMP_DATA_DIR,
"HadCRUT.5.0.2.0.analysis.summary_series.global.annual.csv")

OHC_PATH <- file.path(COMP_DATA_DIR, "OHC_ensemble_Kuhlbrodt_etal_2022.csv")


INI_FILE <- system.file("input/hector_ssp245.ini", package = "hector")
PARAMS <- c(BETA(), Q10_RH(), DIFFUSIVITY(), ECS(), AERO_SCALE())

OUTPUT <- file.path(RESULTS_DIR, "13_alpha_ecs_ohc_mvsse.txt")


source(file.path(SCRIPTS_DIR, "major_functions.R"))

### Getting observational data ###
co2_data <- get_co2_data(CO2_PATH, include_unc = TRUE)
temp_data <- get_temp_data(TEMP_PATH, include_unc = TRUE)
ohc_data <- get_ohc_data(OHC_PATH, include_unc = T)
obs_data <- rbind(co2_data, temp_data, ohc_data)

### Calling optim ###
best_pars <- run_optim(obs_data = obs_data,
ini_file = INI_FILE,
params = PARAMS,
lower = c(0.5 - 0.232, 2.2 - 0.44, 1.16 - 0.118, 2, 0),
upper = c(0.5 + 0.232, 2.2 + 0.44, 1.16 + 0.118, 5, 3),
yrs = 1750:2014,
vars = c(GMST(), CONCENTRATIONS_CO2(), HEAT_FLUX()),
error_fn = mean_T_CO2_OHC_mvsse,
include_unc = T,
method = "L-BFGS-B",
output_file = OUTPUT)

### Outputting individual MSEs ###
hector_data <- run_hector(ini_file = INI_FILE,
params = PARAMS,
vals = best_pars,
yrs = 1750:2014,
vars = c(GMST(), CONCENTRATIONS_CO2(), HEAT_FLUX()))

T_mse <- get_var_mse(obs_data = obs_data,
hector_data = hector_data,
var = GMST(),
yrs = 1850:2014)
CO2_mse <- get_var_mse(obs_data = obs_data,
hector_data = hector_data,
var = CONCENTRATIONS_CO2(),
yrs = c(1750, 1850:2014))

# Getting MVSSEs
T_mvsse <- get_var_mvsse(obs_data = obs_data,
hector_data = hector_data,
var = GMST(),
yrs = 1850:2014,
mse_fn = mvsse)
CO2_mvsse <- get_var_mvsse(obs_data = obs_data,
hector_data = hector_data,
var = CONCENTRATIONS_CO2(),
yrs = c(1750, 1850:2014),
mse_fn = mvsse)
OHC_mvsse <- get_var_mvsse(obs_data = obs_data,
hector_data = hector_data,
var = "OHC",
yrs = 1957:2014,
mse_fn = mvsse)

write_metric("CO2 MSE:", CO2_mse, OUTPUT)
write_metric("T MSE: ", T_mse, OUTPUT)
write_metric("RMSE: ", sqrt(mean(CO2_mse, T_mse)), OUTPUT) # not 100% sure this is how we want to calculate this
write_metric("CO2 MVSSE:", CO2_mvsse, OUTPUT)
write_metric("T MVSSE: ", T_mvsse, OUTPUT)
write_metric("OHC MVSSE:", OHC_mvsse, OUTPUT)
write("", OUTPUT, append = TRUE)

### Outputting table metrics ###
calc_table_metrics(PARAMS, best_pars, OUTPUT)
27 changes: 27 additions & 0 deletions scripts/default_stats.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,9 @@ write_metric("Mean of T, CO2 NMSEs (with unc):",
write_metric("Mean of T, CO2, OHC NMSEs (with unc):",
mean_T_CO2_OHC_nmse_unc(obs_data, hector_data),
OUTPUT)
write_metric("Mean of T, CO2, OHC MVSSEs:",
mean_T_CO2_OHC_mvsse(obs_data, hector_data),
OUTPUT)
write("", OUTPUT, append = TRUE)

# Outputting smoothed MSEs
Expand Down Expand Up @@ -134,6 +137,26 @@ OHC_nmse_unc <- get_var_mse_unc(obs_data = obs_data,
yrs = 1957:2014,
mse_fn = nmse_unc)




# Getting MVSSEs
T_mvsse <- get_var_mvsse(obs_data = obs_data,
hector_data = hector_data,
var = GMST(),
yrs = 1850:2014,
mse_fn = mvsse)
CO2_mvsse <- get_var_mvsse(obs_data = obs_data,
hector_data = hector_data,
var = CONCENTRATIONS_CO2(),
yrs = c(1750, 1850:2014),
mse_fn = mvsse)
OHC_mvsse <- get_var_mvsse(obs_data = obs_data,
hector_data = hector_data,
var = "OHC",
yrs = 1957:2014,
mse_fn = mvsse)

write_metric("CO2 MSE: ", CO2_mse, OUTPUT)
write_metric("T MSE: ", T_mse, OUTPUT)
write_metric("T MSE with unc:", T_mse_unc, OUTPUT)
Expand All @@ -144,6 +167,10 @@ write_metric("T NMSE: ", T_nmse, OUTPUT)
write_metric("T NMSE with unc:", T_nmse_unc, OUTPUT)
write_metric("OHC NMSE with unc:", OHC_nmse_unc, OUTPUT)
write("", OUTPUT, append = TRUE)
write_metric("CO2 MVSSE:", CO2_mvsse, OUTPUT)
write_metric("T MVSSE: ", T_mvsse, OUTPUT)
write_metric("OHC MVSSE:", OHC_mvsse, OUTPUT)
write("", OUTPUT, append = TRUE)

### Outputting table metrics ###
calc_table_metrics(NULL, NULL, OUTPUT)
38 changes: 36 additions & 2 deletions scripts/error_functions.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,10 +189,17 @@ get_var_mse_unc <- function(obs_data, hector_data, var, yrs, mse_fn) {
# Note: Assumes observed data contains symmetric upper and lower bounds 1 SD
# away from actual value
get_var_mvsse <- function(obs_data, hector_data, var, yrs, mse_fn) {

# Getting x and sd
x <- filter(obs_data, year %in% yrs & variable == var)$value
x_upper <- filter(obs_data, year %in% yrs & variable == var)$upper
sd <- x_upper - x
y <- filter(hector_data, year %in% yrs & variable == var)$value
if (var == GMST()) {
sd <- (x_upper - x) / 1.96 # Accounting for 95% confidence interval
} else {
sd <- x_upper - x
}

y <- filter(hector_data, year %in% yrs & variable == var)$value

return(mvsse(x = x, sd = sd, y = y))
}
Expand Down Expand Up @@ -297,6 +304,33 @@ mean_T_CO2_OHC_nmse_unc <- function(obs_data, hector_data) {
return(mean(c(T_mse, CO2_mse, OHC_mse)))
}

# mean_T_CO2_OHC_mvsse: function to find the mean of the temperature, CO2,
# and OHC MVSSEs between observed and predicted data
#
# args:
# obs_data - data frame of observed data formatted like Hector data frame
# hector_data - data frame outputted by Hector
#
# Returns: Average NMSE between predicted and observed data
mean_T_CO2_OHC_mvsse <- function(obs_data, hector_data) {
T_mse <- get_var_mvsse(obs_data = obs_data,
hector_data = hector_data,
var = GMST(),
yrs = 1850:2014,
mse_fn = mvsse)
CO2_mse <- get_var_mvsse(obs_data = obs_data,
hector_data = hector_data,
var = CONCENTRATIONS_CO2(),
yrs = c(1750, 1850:2014),
mse_fn = mvsse)
OHC_mse <- get_var_mvsse(obs_data = obs_data,
hector_data = hector_data,
var = "OHC",
yrs = 1957:2014,
mse_fn = mvsse)
return(mean(c(T_mse, CO2_mse, OHC_mse)))
}

# smooth_T_CO2_mse: function to find the mean of smoothed temperature and CO2
# MSEs between observed & predicted data for a given variable
#
Expand Down
7 changes: 5 additions & 2 deletions scripts/major_functions.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ INPUT_TCR <- file.path(IDEAL_RUNS, "hector_1pctCO2.ini")
OCEAN_AREA <- 5100656e8 * (1 - 0.29) # The total area of the ocean
W_TO_ZJ <- 3.155693e-14 # Watts to ZJ

# Other Numeric Constants
CO2_OBS_SD <- 0.12

# Constants for TCRE and ERF calculations
EMISSIONS_VARS <- c(FFI_EMISSIONS(), LUC_EMISSIONS())
UPTAKE_VARS <- c(LUC_UPTAKE(), DACCS_UPTAKE())
Expand Down Expand Up @@ -64,8 +67,8 @@ get_co2_data <- function(file, scenario = "historical", include_unc = F) {
co2_data$units <- " ppmv CO2"

if (include_unc) {
co2_data$upper <- co2_data$value
co2_data$lower <- co2_data$value
co2_data$upper <- co2_data$value + CO2_OBS_SD
co2_data$lower <- co2_data$value - CO2_OBS_SD
}

return(co2_data)
Expand Down
14 changes: 13 additions & 1 deletion tests/error_tests.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,9 +87,21 @@ nmse_unc_tests <- function() {
c(10, 7, 28)) == 9 / 625)
}

# Trying to use mvsse
mvsse_tests <- function () {
# Just using mse tests and sd of 1
assert_that(mvsse(0, 1, 0) == 0)
assert_that(mvsse(c(0, 1, 2), 1, c(1, 2, 1)) == 1)
assert_that(mvsse(c(0, 1, 2), c(1, 1, 1), c(0, 1, -1)) == 3)
assert_that(mvsse(c(-2, 1, 2), 1, c(0, 3, 0)) == 4)

# Other tests
assert_that(mvsse(c(0, 1, 2, 3), c(2, 3, 4, 1), c(-1, 4, 4, 3)) == 0.375)
}

# Calling all testing functions
mse_tests()
get_var_mse_tests()
mse_unc_tests()
nmse_unc_tests()
nmse_unc_tests()
mvsse_tests()

0 comments on commit 7da1e04

Please sign in to comment.