From 18964eae80c7432fe1a41df5684d481eb2fe348a Mon Sep 17 00:00:00 2001 From: Duncan Watson-Parris Date: Fri, 14 Oct 2022 16:54:09 +0100 Subject: [PATCH 1/7] Add support for daily output frequency --- pangeo_forge_esgf/dynamic_kwargs.py | 18 ++++++++++++++---- pangeo_forge_esgf/recipe_inputs.py | 2 +- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/pangeo_forge_esgf/dynamic_kwargs.py b/pangeo_forge_esgf/dynamic_kwargs.py index 29fd894..ec1c757 100644 --- a/pangeo_forge_esgf/dynamic_kwargs.py +++ b/pangeo_forge_esgf/dynamic_kwargs.py @@ -13,21 +13,31 @@ # TODO: I might not want to allow this in the ocean and ice fields. Lets see ) +daily_divisors = sorted( + [1, 30, 360, 365, 3653] + # TODO: It might be more efficient to default to the underlying file chunks? +) + allowed_divisors = { "Omon": monthly_divisors, "SImon": monthly_divisors, "Amon": monthly_divisors, + "day": daily_divisors, } # Add table_ids and allowed divisors as needed def get_timesteps_simple(dates, table_id): - assert ( - "mon" in table_id - ) # this needs some more careful treatment for other timefrequencies. - timesteps = [ + months = [ (int(d[1][0:4]) - int(d[0][0:4])) * 12 + (int(d[1][4:6]) - int(d[0][4:6]) + 1) for d in dates ] + if "mon" in table_id: + timesteps = months + elif "day" in table_id: + timesteps = [m * 30 for m in months] # This is close enough without worrying about calendars etc. + else: + # this needs some more careful treatment for other timefrequencies. + raise NotImplemented("Only daily and monthly time frequencies are currently supported") return timesteps diff --git a/pangeo_forge_esgf/recipe_inputs.py b/pangeo_forge_esgf/recipe_inputs.py index 91ff059..6401ffb 100644 --- a/pangeo_forge_esgf/recipe_inputs.py +++ b/pangeo_forge_esgf/recipe_inputs.py @@ -20,6 +20,7 @@ # Data nodes in preferred order (from naomis code here: https://github.com/naomi-henderson/cmip6collect2/blob/main/myconfig.py) data_nodes = [ + "noresg.nird.sigma2.no", "esgf-data1.llnl.gov", "esgf-data2.llnl.gov", "aims3.llnl.gov", @@ -49,7 +50,6 @@ "esgf.nci.org.au", "esgf.rcec.sinica.edu.tw", "esgf3.dkrz.de", - "noresg.nird.sigma2.no", "polaris.pknu.ac.kr", "vesg.ipsl.upmc.fr", ] From a3f687335dc70cc66277970fd4b05d7a3a7c1c2a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 14 Oct 2022 15:59:20 +0000 Subject: [PATCH 2/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pangeo_forge_esgf/dynamic_kwargs.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pangeo_forge_esgf/dynamic_kwargs.py b/pangeo_forge_esgf/dynamic_kwargs.py index ec1c757..9d02cc2 100644 --- a/pangeo_forge_esgf/dynamic_kwargs.py +++ b/pangeo_forge_esgf/dynamic_kwargs.py @@ -34,10 +34,14 @@ def get_timesteps_simple(dates, table_id): if "mon" in table_id: timesteps = months elif "day" in table_id: - timesteps = [m * 30 for m in months] # This is close enough without worrying about calendars etc. + timesteps = [ + m * 30 for m in months + ] # This is close enough without worrying about calendars etc. else: # this needs some more careful treatment for other timefrequencies. - raise NotImplemented("Only daily and monthly time frequencies are currently supported") + raise NotImplemented( + "Only daily and monthly time frequencies are currently supported" + ) return timesteps From 1391ba7e9c5b21edfe595b960b4ba48b6697c0b7 Mon Sep 17 00:00:00 2001 From: Duncan Watson-Parris Date: Fri, 14 Oct 2022 17:02:02 +0100 Subject: [PATCH 3/7] Fix exception style --- pangeo_forge_esgf/dynamic_kwargs.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pangeo_forge_esgf/dynamic_kwargs.py b/pangeo_forge_esgf/dynamic_kwargs.py index 9d02cc2..72674cb 100644 --- a/pangeo_forge_esgf/dynamic_kwargs.py +++ b/pangeo_forge_esgf/dynamic_kwargs.py @@ -39,9 +39,7 @@ def get_timesteps_simple(dates, table_id): ] # This is close enough without worrying about calendars etc. else: # this needs some more careful treatment for other timefrequencies. - raise NotImplemented( - "Only daily and monthly time frequencies are currently supported" - ) + raise NotImplementedError("Only daily and monthly time frequencies are currently supported") return timesteps From 625a9118b573f5905fb66f02e4c0dd34a1242914 Mon Sep 17 00:00:00 2001 From: Duncan Watson-Parris Date: Fri, 14 Oct 2022 17:02:17 +0100 Subject: [PATCH 4/7] Revert unintended change --- pangeo_forge_esgf/recipe_inputs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pangeo_forge_esgf/recipe_inputs.py b/pangeo_forge_esgf/recipe_inputs.py index 6401ffb..91ff059 100644 --- a/pangeo_forge_esgf/recipe_inputs.py +++ b/pangeo_forge_esgf/recipe_inputs.py @@ -20,7 +20,6 @@ # Data nodes in preferred order (from naomis code here: https://github.com/naomi-henderson/cmip6collect2/blob/main/myconfig.py) data_nodes = [ - "noresg.nird.sigma2.no", "esgf-data1.llnl.gov", "esgf-data2.llnl.gov", "aims3.llnl.gov", @@ -50,6 +49,7 @@ "esgf.nci.org.au", "esgf.rcec.sinica.edu.tw", "esgf3.dkrz.de", + "noresg.nird.sigma2.no", "polaris.pknu.ac.kr", "vesg.ipsl.upmc.fr", ] From 7fbe7631cc4a6643208f5818c431d6347c54df0a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 14 Oct 2022 16:03:29 +0000 Subject: [PATCH 5/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pangeo_forge_esgf/dynamic_kwargs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pangeo_forge_esgf/dynamic_kwargs.py b/pangeo_forge_esgf/dynamic_kwargs.py index 72674cb..17221da 100644 --- a/pangeo_forge_esgf/dynamic_kwargs.py +++ b/pangeo_forge_esgf/dynamic_kwargs.py @@ -39,7 +39,9 @@ def get_timesteps_simple(dates, table_id): ] # This is close enough without worrying about calendars etc. else: # this needs some more careful treatment for other timefrequencies. - raise NotImplementedError("Only daily and monthly time frequencies are currently supported") + raise NotImplementedError( + "Only daily and monthly time frequencies are currently supported" + ) return timesteps From 2ba4f64deafdbfe90746aa26c1015f1e8fa3d874 Mon Sep 17 00:00:00 2001 From: Duncan Watson-Parris Date: Mon, 23 Jan 2023 21:14:17 +0000 Subject: [PATCH 6/7] Update pangeo_forge_esgf/dynamic_kwargs.py Co-authored-by: Julius Busecke --- pangeo_forge_esgf/dynamic_kwargs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pangeo_forge_esgf/dynamic_kwargs.py b/pangeo_forge_esgf/dynamic_kwargs.py index 17221da..457920a 100644 --- a/pangeo_forge_esgf/dynamic_kwargs.py +++ b/pangeo_forge_esgf/dynamic_kwargs.py @@ -15,7 +15,6 @@ daily_divisors = sorted( [1, 30, 360, 365, 3653] - # TODO: It might be more efficient to default to the underlying file chunks? ) allowed_divisors = { From 49035f01704c13a91224b616c7c820b7dd7c51e1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 23 Jan 2023 21:15:08 +0000 Subject: [PATCH 7/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pangeo_forge_esgf/dynamic_kwargs.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pangeo_forge_esgf/dynamic_kwargs.py b/pangeo_forge_esgf/dynamic_kwargs.py index 457920a..5ebb8b9 100644 --- a/pangeo_forge_esgf/dynamic_kwargs.py +++ b/pangeo_forge_esgf/dynamic_kwargs.py @@ -13,9 +13,7 @@ # TODO: I might not want to allow this in the ocean and ice fields. Lets see ) -daily_divisors = sorted( - [1, 30, 360, 365, 3653] -) +daily_divisors = sorted([1, 30, 360, 365, 3653]) allowed_divisors = { "Omon": monthly_divisors,