From cb042e3a26fe6bd2fab9da355d2a8acc59cb9a19 Mon Sep 17 00:00:00 2001 From: Jacob Mims <122570226+jtmims@users.noreply.github.com> Date: Fri, 13 Sep 2024 12:31:00 -0500 Subject: [PATCH 01/19] Create container_config_demo.jsonc --- .../container_config_demo.jsonc | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 diagnostics/example_multicase/container_config_demo.jsonc diff --git a/diagnostics/example_multicase/container_config_demo.jsonc b/diagnostics/example_multicase/container_config_demo.jsonc new file mode 100644 index 000000000..cffb8ec80 --- /dev/null +++ b/diagnostics/example_multicase/container_config_demo.jsonc @@ -0,0 +1,117 @@ +// This a template for configuring MDTF to run PODs that analyze multi-run/ensemble data +// +// Copy this file, rename it, and customize the settings as needed +// Pass your file to the framework using the -f/--input-file flag. +// Any other explicit command line options will override what's listed here. +// +// All text to the right of an unquoted "//" is a comment and ignored, as well +// as blank lines (JSONC quasi-standard.) +// +// Remove your test config file, or any changes you make to this template if you do not rename it, +// from your remote repository before you submit a PR for review. +// To generate CMIP synthetic data in the example dataset, run the following: +// > mamba env create --force -q -f ./src/conda/_env_synthetic_data.yml +// > conda activate _MDTF_synthetic_data +// > pip install mdtf-test-data +// > cd /mdtf +// > mkdir mdtf_test_data && cd mdtf_test_data +// > mdtf_synthetic.py -c CMIP --startyear 1980 --nyears 5 +// > mdtf_synthetic.py -c CMIP --startyear 1985 --nyears 5 +// Note that MODEL_DATA_ROOT assumes that mdtf_test_data is one directory above MDTF-diagnostics +// in this sample config file +{ + // Run each ensemble on the example POD. + // Add other PODs that work on ensemble datasets to the pod_list as needed + "pod_list" : [ + //"example" + "example_multicase" + ], + // Each case corresponds to a different simulation/output dataset + // startdate, enddate: either YYYY-MM-DD, YYYYMMDD:HHMMSS, or YYYY-MM-DD:HHMMSS + "case_list": + { + "CMIP_Synthetic_r1i1p1f1_gr1_19800101-19841231": + { + "model": "test", + "convention": "CMIP", + "startdate": "19800101", + "enddate": "19841231" + } + , + "CMIP_Synthetic_r1i1p1f1_gr1_19850101-19891231": + { + "model": "test", + "convention": "CMIP", + "startdate": "19850101", + "enddate": "19891231" + } + }, + // PATHS --------------------------------------------------------------------- + // Location of supporting data downloaded when the framework was installed. + // If a relative path is given, it's resolved relative to the MDTF-diagnostics + // code directory. Environment variables (eg, $HOME) can be referenced with a + // "$" and will be expended to their current values when the framework runs. + // Full or relative path to model data ESM-intake catalog header file + + "DATA_CATALOG": "/nbhome/Jacob.Mims/syn_data/esm_catalog_CMIP_synthetic_r1i1p1f1_gr1.json", + + // Parent directory containing observational data used by individual PODs. + "OBS_DATA_ROOT": "../inputdata/obs_data", + + // Working directory. + "WORK_DIR": "../wkdir", + + // Directory to write output. The results of each run of the framework will be + // put in a subdirectory of this directory. Defaults to WORKING_DIR if blank. + "OUTPUT_DIR": "../wkdir", + + // Location of the Anaconda/miniconda or micromamba installation to use for managing + // dependencies (path returned by running `conda info --base` or `micromamba info`.) + "conda_root": "/local/home/Jacob.Mims/miniconda3", + + // Directory containing the framework-specific conda environments. This should + // be equal to the "--env_dir" flag passed to conda_env_setup.sh. If left + // blank, the framework will look for its environments in conda_root/envs + "conda_env_root": "/local/home/Jacob.Mims/miniconda3/envs", + + // Location of the micromamba executable. Required if using micromamba + "micromamba_exe": "", + + // SETTINGS ------------------------------------------------------------------ + // Any command-line option recognized by the mdtf script (type `mdtf --help`) + // can be set here, in the form "flag name": "desired setting". + + // Settings affecting what output is generated: + // Set to true to run the preprocessor; default true: + "run_pp": true, + // Set to true to perform data translation; default false: + "translate_data": true, + // Set to true to have PODs save postscript figures in addition to bitmaps. + "save_ps": false, + + // Set to true for files > 4 GB + "large_file": false, + + // If true, leave pp data in OUTPUT_DIR after preprocessing; if false, delete pp data after PODs + // run to completion + "save_pp_data": true, + + // Set to true to save HTML and bitmap plots in a .tar file. + "make_variab_tar": false, + + // Generate html output for multiple figures per case + "make_multicase_figure_html": false, + + // Set to true to overwrite results in OUTPUT_DIR; otherwise results saved + // under a unique name. + "overwrite": false, + + // List with custom preprocessing script(s) to run on data + // Place these scripts in the user_scripts directory of your copy of the MDTF-diagnostics repository + "user_pp_scripts" : [], + + // Settings used in debugging: + + // Log verbosity level. + "verbose": 1 +} From e43260b74564d9816414d5e18748aacc828b87d8 Mon Sep 17 00:00:00 2001 From: Jacob Mims <122570226+jtmims@users.noreply.github.com> Date: Fri, 13 Sep 2024 12:44:54 -0500 Subject: [PATCH 02/19] Create container_cat.csv --- diagnostics/example_multicase/container_cat.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 diagnostics/example_multicase/container_cat.csv diff --git a/diagnostics/example_multicase/container_cat.csv b/diagnostics/example_multicase/container_cat.csv new file mode 100644 index 000000000..d3db31f86 --- /dev/null +++ b/diagnostics/example_multicase/container_cat.csv @@ -0,0 +1,3 @@ +activity_id,branch_method,branch_time_in_child,branch_time_in_parent,experiment,experiment_id,frequency,grid,grid_label,institution_id,nominal_resolution,parent_activity_id,parent_experiment_id,parent_source_id,parent_time_units,parent_variant_label,product,realm,source_id,source_type,sub_experiment,sub_experiment_id,table_id,variable_id,variant_label,member_id,standard_name,long_name,units,vertical_levels,init_year,start_time,end_time,time_range,path,version +CMIP,standard,,,,synthetic,day,,gr,,,CMIP,,,days since 1980-01-01,r1i1p1f1,,atmos,,,none,none,day,tas,r1i1p1f1,r1i1p1f1,air_temperature,Near-Surface Air Temperature,K,1,,1980-01-01,1984-12-31,1980-01-01-1984-12-31,/proj/MDTF-diagnostics/mdtf_test_data/CMIP_Synthetic_r1i1p1f1_gr1_19800101-19841231/day/CMIP_Synthetic_r1i1p1f1_gr1_19800101-19841231.tas.day.nc,none +CMIP,standard,,,,synthetic,day,,gr,,,CMIP,,,days since 1985-01-01,r1i1p1f1,,atmos,,,none,none,day,tas,r1i1p1f1,r1i1p1f1,air_temperature,Near-Surface Air Temperature,K,1,,1985-01-01,1989-12-31,1985-01-01-1989-12-31,/proj/MDTF-diagnostics/mdtf_test_data/CMIP_Synthetic_r1i1p1f1_gr1_19850101-19891231/day/CMIP_Synthetic_r1i1p1f1_gr1_19850101-19891231.tas.day.nc,none From 99c25035ae8d189c9882af0b95918ab3a8982284 Mon Sep 17 00:00:00 2001 From: Jacob Mims <122570226+jtmims@users.noreply.github.com> Date: Fri, 13 Sep 2024 12:46:00 -0500 Subject: [PATCH 03/19] Create container_cat.json --- .../example_multicase/container_cat.json | 181 ++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 diagnostics/example_multicase/container_cat.json diff --git a/diagnostics/example_multicase/container_cat.json b/diagnostics/example_multicase/container_cat.json new file mode 100644 index 000000000..6917f854e --- /dev/null +++ b/diagnostics/example_multicase/container_cat.json @@ -0,0 +1,181 @@ +{ + "esmcat_version": "0.0.1", + "attributes": [ + { + "column_name": "activity_id", + "vocabulary": "" + }, + { + "column_name": "branch_method", + "vocabulary": "" + }, + { + "column_name": "branch_time_in_child", + "vocabulary": "" + }, + { + "column_name": "branch_time_in_parent", + "vocabulary": "" + }, + { + "column_name": "experiment", + "vocabulary": "" + }, + { + "column_name": "experiment_id", + "vocabulary": "" + }, + { + "column_name": "frequency", + "vocabulary": "" + }, + { + "column_name": "grid", + "vocabulary": "" + }, + { + "column_name": "grid_label", + "vocabulary": "" + }, + { + "column_name": "institution_id", + "vocabulary": "" + }, + { + "column_name": "nominal_resolution", + "vocabulary": "" + }, + { + "column_name": "parent_activity_id", + "vocabulary": "" + }, + { + "column_name": "parent_experiment_id", + "vocabulary": "" + }, + { + "column_name": "parent_source_id", + "vocabulary": "" + }, + { + "column_name": "parent_time_units", + "vocabulary": "" + }, + { + "column_name": "parent_variant_label", + "vocabulary": "" + }, + { + "column_name": "product", + "vocabulary": "" + }, + { + "column_name": "realm", + "vocabulary": "" + }, + { + "column_name": "source_id", + "vocabulary": "" + }, + { + "column_name": "source_type", + "vocabulary": "" + }, + { + "column_name": "sub_experiment", + "vocabulary": "" + }, + { + "column_name": "sub_experiment_id", + "vocabulary": "" + }, + { + "column_name": "table_id", + "vocabulary": "" + }, + { + "column_name": "variable_id", + "vocabulary": "" + }, + { + "column_name": "variant_label", + "vocabulary": "" + }, + { + "column_name": "member_id", + "vocabulary": "" + }, + { + "column_name": "standard_name", + "vocabulary": "" + }, + { + "column_name": "long_name", + "vocabulary": "" + }, + { + "column_name": "units", + "vocabulary": "" + }, + { + "column_name": "vertical_levels", + "vocabulary": "" + }, + { + "column_name": "init_year", + "vocabulary": "" + }, + { + "column_name": "start_time", + "vocabulary": "" + }, + { + "column_name": "end_time", + "vocabulary": "" + }, + { + "column_name": "time_range", + "vocabulary": "" + }, + { + "column_name": "path", + "vocabulary": "" + }, + { + "column_name": "version", + "vocabulary": "" + } + ], + "assets": { + "column_name": "path", + "format": "netcdf", + "format_column_name": null + }, + "aggregation_control": { + "variable_column_name": "variable_id", + "groupby_attrs": [ + "activity_id", + "institution_id", + "source_id", + "experiment_id", + "frequency", + "member_id", + "table_id", + "grid_label", + "realm", + "variant_label" + ], + "aggregations": [ + { + "type": "union", + "attribute_name": "variable_id", + "options": {} + } + ] + }, + "id": "esm_catalog_CMIP_synthetic_r1i1p1f1_gr1.csv", + "description": null, + "title": null, + "last_updated": "2023-06-01", + "catalog_file": "file:/proj/MDTF-diagnostics/diagnostics/example_multicase/container_cat.csv" +} From 4a0072e065ccd3cbf4194cde108737b8be41e3ee Mon Sep 17 00:00:00 2001 From: Jacob Mims <122570226+jtmims@users.noreply.github.com> Date: Fri, 13 Sep 2024 12:48:59 -0500 Subject: [PATCH 04/19] Update container_config_demo.jsonc --- .../example_multicase/container_config_demo.jsonc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/diagnostics/example_multicase/container_config_demo.jsonc b/diagnostics/example_multicase/container_config_demo.jsonc index cffb8ec80..081896643 100644 --- a/diagnostics/example_multicase/container_config_demo.jsonc +++ b/diagnostics/example_multicase/container_config_demo.jsonc @@ -53,29 +53,29 @@ // "$" and will be expended to their current values when the framework runs. // Full or relative path to model data ESM-intake catalog header file - "DATA_CATALOG": "/nbhome/Jacob.Mims/syn_data/esm_catalog_CMIP_synthetic_r1i1p1f1_gr1.json", + "DATA_CATALOG": "/proj/MDTF-diagnostics/diagnostics/example_multicase/container_cat.json", // Parent directory containing observational data used by individual PODs. "OBS_DATA_ROOT": "../inputdata/obs_data", // Working directory. - "WORK_DIR": "../wkdir", + "WORK_DIR": "/proj/wkdir", // Directory to write output. The results of each run of the framework will be // put in a subdirectory of this directory. Defaults to WORKING_DIR if blank. - "OUTPUT_DIR": "../wkdir", + "OUTPUT_DIR": "/proj/wkdir", // Location of the Anaconda/miniconda or micromamba installation to use for managing // dependencies (path returned by running `conda info --base` or `micromamba info`.) - "conda_root": "/local/home/Jacob.Mims/miniconda3", + "conda_root": "/opt/conda", // Directory containing the framework-specific conda environments. This should // be equal to the "--env_dir" flag passed to conda_env_setup.sh. If left // blank, the framework will look for its environments in conda_root/envs - "conda_env_root": "/local/home/Jacob.Mims/miniconda3/envs", + "conda_env_root": "/opt/conda/envs", // Location of the micromamba executable. Required if using micromamba - "micromamba_exe": "", + "micromamba_exe": "/bin/micromaba", // SETTINGS ------------------------------------------------------------------ // Any command-line option recognized by the mdtf script (type `mdtf --help`) From 125fad257bd47c3233f0570e3a51da52ac7d9df9 Mon Sep 17 00:00:00 2001 From: Jacob Mims <122570226+jtmims@users.noreply.github.com> Date: Fri, 13 Sep 2024 14:06:12 -0400 Subject: [PATCH 05/19] docs --- doc/sphinx/ref_container.rst | 85 ++++++++++++++++++++++++++++++++++++ doc/sphinx/ref_toc.rst | 1 + 2 files changed, 86 insertions(+) create mode 100644 doc/sphinx/ref_container.rst diff --git a/doc/sphinx/ref_container.rst b/doc/sphinx/ref_container.rst new file mode 100644 index 000000000..fb38aad34 --- /dev/null +++ b/doc/sphinx/ref_container.rst @@ -0,0 +1,85 @@ +.. role:: code-rst(code) + :language: reStructuredText +.. _ref-container: +Container Reference +=============================== +This section provides basic directions for downloading, +installing, and running the example_multicase POD in the +Model Diagnostics Task Force (MDTF) container. + +Getting the Container +------------------------------- +The container assumes that the MDTF GitHub repo is located on your local machine. +If you have not already, please clone the repo to your local machine with: + + .. code-block:: bash + + git clone https://github.com/NOAA-GFDL/MDTF-diagnostics.git + +In the new MDTF-Diagnostics directory, you can build the container +using the Dockerfile. If using podman (as required internally at the GFDL), +please build with the command: + + .. code-block:: bash + + podman build . --format docker -t mdtf + +:code-rst:`--format docker` is essential to have your copy commands work and +have the expected permissions in your container. +The container can also just be pulled from the GitHub +container registry with the command: + + .. code-block:: bash + + docker pull ghcr.io/aradhakrishnangfdl/mdtf-diagnostics:container + +or what the equivalent command in your container software happens to be. + +Launching the Container +------------------------------- +The container it self can be launched with Docker using: + + .. code-block:: bash + + docker run -it -v {DIAG_DIR}:/proj/MDTF-diagnostics/diagnostics/:z -v {WKDIR}:/proj/wkdir:z mdtf + +wherein: + * :code-rst:`{DIAG_DIR}` is the path to the diagnostics directory on your local machine. + This volume is not required, but heavily recommended. + * :code-rst:`{WKDIR}` is where you would like to store the output on your local machine. + This allows the output HTML to be reachable without having to open a port to the container. + +For the case of the example, we can launch with only the wkdir mounted volume. + +Generating Synthetic Data +------------------------------- +Now that we are in the container, we can get to making some data to run the POD with. +The MDTF has a synthetic data generator for just this case. First, move into the proj dir: + + .. code-block:: bash + + cd /proj/MDTF-diagnostics/ + +We can now use the following to generate our synthetic data: + + .. code-block:: bash + + micromamba activate _MDTF_synthetic_data + pip install mdtf-test-data + mkdir mdtf_test_data && cd mdtf_test_data + mdtf_synthetic.py -c CMIP --startyear 1980 --nyears 5 + mdtf_synthetic.py -c CMIP --startyear 1985 --nyears 5 + +Now would be a good time to generate a catalog for the synthetic data, but, in the sake +of testing, we provide a catalog for the files needed to run the example POD. + +Running the POD +------------------------------- +The POD can now be ran using: + + .. code-block:: bash + + micromamba activate _MDTF_base + mdtf_framework.py -f /proj/MDTF-diagnostics/diagnostics/example_multicase/container_config_demo.jsonc + +The results can be found in :code-rst:`/proj/wkdir/` \ No newline at end of file diff --git a/doc/sphinx/ref_toc.rst b/doc/sphinx/ref_toc.rst index 13188ccb3..eea25a836 100644 --- a/doc/sphinx/ref_toc.rst +++ b/doc/sphinx/ref_toc.rst @@ -11,3 +11,4 @@ Framework reference ref_envvars ref_output ref_submodules + ref_container From 346f6fed6baf2d2a8942be78d5a9acf71f51c871 Mon Sep 17 00:00:00 2001 From: Jacob Mims <122570226+jtmims@users.noreply.github.com> Date: Fri, 13 Sep 2024 13:07:17 -0500 Subject: [PATCH 06/19] Update ref_container.rst --- doc/sphinx/ref_container.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/ref_container.rst b/doc/sphinx/ref_container.rst index fb38aad34..83f00564c 100644 --- a/doc/sphinx/ref_container.rst +++ b/doc/sphinx/ref_container.rst @@ -82,4 +82,4 @@ The POD can now be ran using: micromamba activate _MDTF_base mdtf_framework.py -f /proj/MDTF-diagnostics/diagnostics/example_multicase/container_config_demo.jsonc -The results can be found in :code-rst:`/proj/wkdir/` \ No newline at end of file +The results can be found in :code-rst:`/proj/wkdir/`. From 024a3909880b0053a4e7e7e8af869a1430637944 Mon Sep 17 00:00:00 2001 From: Jacob Mims <122570226+jtmims@users.noreply.github.com> Date: Fri, 13 Sep 2024 13:08:01 -0500 Subject: [PATCH 07/19] Update ref_container.rst --- doc/sphinx/ref_container.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/ref_container.rst b/doc/sphinx/ref_container.rst index 83f00564c..a25ccde1e 100644 --- a/doc/sphinx/ref_container.rst +++ b/doc/sphinx/ref_container.rst @@ -33,7 +33,7 @@ container registry with the command: docker pull ghcr.io/aradhakrishnangfdl/mdtf-diagnostics:container -or what the equivalent command in your container software happens to be. +or the equivalent command in your container software happens to be. Launching the Container ------------------------------- From 9125a75ee4ae9e94cc6fa6403e9ed6da0d96c47b Mon Sep 17 00:00:00 2001 From: Jacob Mims <122570226+jtmims@users.noreply.github.com> Date: Fri, 13 Sep 2024 13:08:21 -0500 Subject: [PATCH 08/19] Update ref_container.rst --- doc/sphinx/ref_container.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/ref_container.rst b/doc/sphinx/ref_container.rst index a25ccde1e..aed6bf880 100644 --- a/doc/sphinx/ref_container.rst +++ b/doc/sphinx/ref_container.rst @@ -33,7 +33,7 @@ container registry with the command: docker pull ghcr.io/aradhakrishnangfdl/mdtf-diagnostics:container -or the equivalent command in your container software happens to be. +or the equivalent command in your container software. Launching the Container ------------------------------- From 573fb77fbb93cc268ba0515ebdfc2382e8ac1851 Mon Sep 17 00:00:00 2001 From: Jacob Mims <122570226+jtmims@users.noreply.github.com> Date: Fri, 13 Sep 2024 13:08:46 -0500 Subject: [PATCH 09/19] Update ref_container.rst --- doc/sphinx/ref_container.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/ref_container.rst b/doc/sphinx/ref_container.rst index aed6bf880..84f272121 100644 --- a/doc/sphinx/ref_container.rst +++ b/doc/sphinx/ref_container.rst @@ -37,7 +37,7 @@ or the equivalent command in your container software. Launching the Container ------------------------------- -The container it self can be launched with Docker using: +The container can be launched with Docker using: .. code-block:: bash From ef4ddf243ead0945eea697727b8cd9a922e73973 Mon Sep 17 00:00:00 2001 From: Jacob Mims <122570226+jtmims@users.noreply.github.com> Date: Tue, 17 Sep 2024 17:05:18 -0400 Subject: [PATCH 10/19] Update ref_container.rst --- doc/sphinx/ref_container.rst | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/doc/sphinx/ref_container.rst b/doc/sphinx/ref_container.rst index 84f272121..dc733fc69 100644 --- a/doc/sphinx/ref_container.rst +++ b/doc/sphinx/ref_container.rst @@ -16,32 +16,22 @@ If you have not already, please clone the repo to your local machine with: git clone https://github.com/NOAA-GFDL/MDTF-diagnostics.git -In the new MDTF-Diagnostics directory, you can build the container -using the Dockerfile. If using podman (as required internally at the GFDL), -please build with the command: - - .. code-block:: bash - - podman build . --format docker -t mdtf - -:code-rst:`--format docker` is essential to have your copy commands work and -have the expected permissions in your container. -The container can also just be pulled from the GitHub +The container can then be pulled from the GitHub container registry with the command: .. code-block:: bash - docker pull ghcr.io/aradhakrishnangfdl/mdtf-diagnostics:container + docker pull ghcr.io/noaa-gfdl/mdtf-diagnostics:container -or the equivalent command in your container software. +or with the equivalent command in your container software. Launching the Container ------------------------------- -The container can be launched with Docker using: +The container itself can be launched with Docker using: .. code-block:: bash - docker run -it -v {DIAG_DIR}:/proj/MDTF-diagnostics/diagnostics/:z -v {WKDIR}:/proj/wkdir:z mdtf + docker run -it -v {DIAG_DIR}:/proj/MDTF-diagnostics/diagnostics/ -v {WKDIR}:/proj/wkdir mdtf wherein: * :code-rst:`{DIAG_DIR}` is the path to the diagnostics directory on your local machine. @@ -82,4 +72,17 @@ The POD can now be ran using: micromamba activate _MDTF_base mdtf_framework.py -f /proj/MDTF-diagnostics/diagnostics/example_multicase/container_config_demo.jsonc -The results can be found in :code-rst:`/proj/wkdir/`. +The results can be found in :code-rst:`/proj/wkdir/` + +Building the Container +-------------------------------- +If you would like, you can build the container using the Dockerfile found in the GitHub repo. +If using podman (as required internally at the GFDL), +please build with the command: + + .. code-block:: bash + + podman build . --format docker -t mdtf + +:code-rst:`--format docker` is essential to have your copy commands work and +have the expected permissions in your container. From ed06142d2494d4ef024b2ae7fb520e5a0b2cf4ca Mon Sep 17 00:00:00 2001 From: Jacob Mims <122570226+jtmims@users.noreply.github.com> Date: Tue, 17 Sep 2024 17:05:59 -0400 Subject: [PATCH 11/19] Update dev_start.rst --- doc/sphinx/dev_start.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/sphinx/dev_start.rst b/doc/sphinx/dev_start.rst index cb2be8c03..529c88cb5 100644 --- a/doc/sphinx/dev_start.rst +++ b/doc/sphinx/dev_start.rst @@ -26,6 +26,10 @@ Developers may download the code from GitHub as described in :ref:`ref-download` clone the repo in order to keep up with changes in the main branch, and to simplify submitting pull requests with your POD's code. Instructions for how to do this are given in :doc:`dev_git_intro`. +It is also possible to download the MDTF container to bypass worrying about dependencies. +the details can be found in :doc:`ref_container`. The container is a new addition. +So, Beta Testers are very much welcome! + Installing dependencies with Conda ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 0c82db36c526de079d283b5733fca634dbb1cea4 Mon Sep 17 00:00:00 2001 From: Jacob Mims <122570226+jtmims@users.noreply.github.com> Date: Wed, 18 Sep 2024 11:46:45 -0400 Subject: [PATCH 12/19] Update ref_container.rst --- doc/sphinx/ref_container.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/sphinx/ref_container.rst b/doc/sphinx/ref_container.rst index dc733fc69..bfe9767c1 100644 --- a/doc/sphinx/ref_container.rst +++ b/doc/sphinx/ref_container.rst @@ -24,6 +24,7 @@ container registry with the command: docker pull ghcr.io/noaa-gfdl/mdtf-diagnostics:container or with the equivalent command in your container software. +If you do not have a container software, Docker can be downloaded from `here `_. Launching the Container ------------------------------- From b65bd95a2e467cc4bb3496cec5d06245e26b9b06 Mon Sep 17 00:00:00 2001 From: Jacob Mims <122570226+jtmims@users.noreply.github.com> Date: Wed, 18 Sep 2024 12:10:16 -0400 Subject: [PATCH 13/19] Update dev_start.rst --- doc/sphinx/dev_start.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/dev_start.rst b/doc/sphinx/dev_start.rst index 529c88cb5..d538b9702 100644 --- a/doc/sphinx/dev_start.rst +++ b/doc/sphinx/dev_start.rst @@ -27,7 +27,7 @@ clone the repo in order to keep up with changes in the main branch, and to simpl POD's code. Instructions for how to do this are given in :doc:`dev_git_intro`. It is also possible to download the MDTF container to bypass worrying about dependencies. -the details can be found in :doc:`ref_container`. The container is a new addition. +Further details can be found in :doc:`ref_container`. The container is a new addition. So, Beta Testers are very much welcome! Installing dependencies with Conda From 8548a2bbd6b51ae8a1749a3ea2f2acbfaed39dad Mon Sep 17 00:00:00 2001 From: Jacob Mims <122570226+jtmims@users.noreply.github.com> Date: Wed, 18 Sep 2024 12:14:34 -0400 Subject: [PATCH 14/19] Update ref_container.rst --- doc/sphinx/ref_container.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx/ref_container.rst b/doc/sphinx/ref_container.rst index bfe9767c1..06d67fde6 100644 --- a/doc/sphinx/ref_container.rst +++ b/doc/sphinx/ref_container.rst @@ -40,12 +40,12 @@ wherein: * :code-rst:`{WKDIR}` is where you would like to store the output on your local machine. This allows the output HTML to be reachable without having to open a port to the container. -For the case of the example, we can launch with only the wkdir mounted volume. +These happen to be the only required volumes. Further volumes may need to be mounted including volumes such as data storage. Generating Synthetic Data ------------------------------- Now that we are in the container, we can get to making some data to run the POD with. -The MDTF has a synthetic data generator for just this case. First, move into the proj dir: +The MDTF has a synthetic data generator for just this case. First, move into the MDTF-diagnostics dir: .. code-block:: bash From 4a5845fa884c1d5a7aa9dd91f7d94493c6da66f4 Mon Sep 17 00:00:00 2001 From: Jacob Mims <122570226+jtmims@users.noreply.github.com> Date: Wed, 18 Sep 2024 12:32:42 -0400 Subject: [PATCH 15/19] Update doc/sphinx/dev_start.rst Co-authored-by: Jess <20195932+wrongkindofdoctor@users.noreply.github.com> --- doc/sphinx/dev_start.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/dev_start.rst b/doc/sphinx/dev_start.rst index d538b9702..a2c0042a1 100644 --- a/doc/sphinx/dev_start.rst +++ b/doc/sphinx/dev_start.rst @@ -28,7 +28,7 @@ POD's code. Instructions for how to do this are given in :doc:`dev_git_intro`. It is also possible to download the MDTF container to bypass worrying about dependencies. Further details can be found in :doc:`ref_container`. The container is a new addition. -So, Beta Testers are very much welcome! +Beta Testers are very much welcome! Installing dependencies with Conda ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From dab50765cb6003a11c6ebeb60b1cbdbd164963d1 Mon Sep 17 00:00:00 2001 From: Jacob Mims <122570226+jtmims@users.noreply.github.com> Date: Wed, 18 Sep 2024 12:32:49 -0400 Subject: [PATCH 16/19] Update doc/sphinx/ref_container.rst Co-authored-by: Jess <20195932+wrongkindofdoctor@users.noreply.github.com> --- doc/sphinx/ref_container.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/ref_container.rst b/doc/sphinx/ref_container.rst index 06d67fde6..7c9d59558 100644 --- a/doc/sphinx/ref_container.rst +++ b/doc/sphinx/ref_container.rst @@ -9,7 +9,7 @@ Model Diagnostics Task Force (MDTF) container. Getting the Container ------------------------------- -The container assumes that the MDTF GitHub repo is located on your local machine. +The container assumes that the MDTF-diangnostics GitHub repo is located on your local machine. If you have not already, please clone the repo to your local machine with: .. code-block:: bash From e80a435307a3a8373244695b1213d8611ffcf7c8 Mon Sep 17 00:00:00 2001 From: Jacob Mims <122570226+jtmims@users.noreply.github.com> Date: Wed, 18 Sep 2024 12:32:57 -0400 Subject: [PATCH 17/19] Update doc/sphinx/ref_container.rst Co-authored-by: Jess <20195932+wrongkindofdoctor@users.noreply.github.com> --- doc/sphinx/ref_container.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/ref_container.rst b/doc/sphinx/ref_container.rst index 7c9d59558..128ae3ee7 100644 --- a/doc/sphinx/ref_container.rst +++ b/doc/sphinx/ref_container.rst @@ -44,7 +44,7 @@ These happen to be the only required volumes. Further volumes may need to be mou Generating Synthetic Data ------------------------------- -Now that we are in the container, we can get to making some data to run the POD with. +Now that we are in the container, we can create some data to run the POD with. The MDTF has a synthetic data generator for just this case. First, move into the MDTF-diagnostics dir: .. code-block:: bash From ac430f6fb302d4e17650d2d5f976a535c7a298b9 Mon Sep 17 00:00:00 2001 From: Jacob Mims <122570226+jtmims@users.noreply.github.com> Date: Wed, 18 Sep 2024 12:33:05 -0400 Subject: [PATCH 18/19] Update doc/sphinx/ref_container.rst Co-authored-by: Jess <20195932+wrongkindofdoctor@users.noreply.github.com> --- doc/sphinx/ref_container.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/ref_container.rst b/doc/sphinx/ref_container.rst index 128ae3ee7..f7efbb428 100644 --- a/doc/sphinx/ref_container.rst +++ b/doc/sphinx/ref_container.rst @@ -51,7 +51,7 @@ The MDTF has a synthetic data generator for just this case. First, move into the cd /proj/MDTF-diagnostics/ -We can now use the following to generate our synthetic data: +We generate our synthetic data by running: .. code-block:: bash From 8a70d15bd5ee88f5ae7b29aa48b52d8de4627a04 Mon Sep 17 00:00:00 2001 From: Jacob Mims <122570226+jtmims@users.noreply.github.com> Date: Wed, 18 Sep 2024 12:33:12 -0400 Subject: [PATCH 19/19] Update doc/sphinx/dev_start.rst Co-authored-by: Jess <20195932+wrongkindofdoctor@users.noreply.github.com> --- doc/sphinx/dev_start.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/dev_start.rst b/doc/sphinx/dev_start.rst index a2c0042a1..f6464a26b 100644 --- a/doc/sphinx/dev_start.rst +++ b/doc/sphinx/dev_start.rst @@ -26,7 +26,7 @@ Developers may download the code from GitHub as described in :ref:`ref-download` clone the repo in order to keep up with changes in the main branch, and to simplify submitting pull requests with your POD's code. Instructions for how to do this are given in :doc:`dev_git_intro`. -It is also possible to download the MDTF container to bypass worrying about dependencies. +Users may also install and run the MDTF-diagnostics Docker container that includes pre-built base, python3_base, and synthetic_data Conda environments (NCL is not compatible with Docker). Further details can be found in :doc:`ref_container`. The container is a new addition. Beta Testers are very much welcome!