From cb91734fe10fb4dcc02504ba401008fa93c6dbba Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Tue, 25 Apr 2023 14:39:29 +0200 Subject: [PATCH] Refactored toolkit so that download is top level (#30), and adjusted runs to use the new layout --- .github/workflows/run_algoa-bay-forecast.yml | 26 ++++++++--------- .github/workflows/run_false-bay-forecast.yml | 26 ++++++++--------- .vscode/.filedump | 6 ++++ toolkit/README.md | 22 +++++++------- toolkit/__main__.py | 5 +++- toolkit/cli/applications/__init__.py | 4 +-- .../{ops => }/download/__init__.py | 4 +-- .../{ops => }/download/gfs/__init__.py | 2 +- .../{ops => }/download/gfs/functions.py | 0 .../{ops => }/download/mercator/__init__.py | 0 toolkit/cli/applications/ops/__init__.py | 3 -- toolkit/cli/define/__init__.py | 4 +-- .../cli/define/{ops => download}/__init__.py | 29 +++++++------------ toolkit/cli/parse/__init__.py | 4 +-- toolkit/cli/parse/download/__init__.py | 10 +++++++ toolkit/cli/parse/ops/__init__.py | 10 ------- 16 files changed, 73 insertions(+), 82 deletions(-) create mode 100644 .vscode/.filedump rename toolkit/cli/applications/{ops => }/download/__init__.py (91%) rename toolkit/cli/applications/{ops => }/download/gfs/__init__.py (98%) rename toolkit/cli/applications/{ops => }/download/gfs/functions.py (100%) rename toolkit/cli/applications/{ops => }/download/mercator/__init__.py (100%) delete mode 100644 toolkit/cli/applications/ops/__init__.py rename toolkit/cli/define/{ops => download}/__init__.py (55%) create mode 100644 toolkit/cli/parse/download/__init__.py delete mode 100644 toolkit/cli/parse/ops/__init__.py diff --git a/.github/workflows/run_algoa-bay-forecast.yml b/.github/workflows/run_algoa-bay-forecast.yml index 04742047..10fb0c5e 100644 --- a/.github/workflows/run_algoa-bay-forecast.yml +++ b/.github/workflows/run_algoa-bay-forecast.yml @@ -247,13 +247,12 @@ jobs: -e COPERNICUS_USERNAME=${{ env.COPERNICUS_USERNAME }} \ -e COPERNICUS_PASSWORD=${{ env.COPERNICUS_PASSWORD }} \ ${{ needs.toolkit.outputs.image }}:${{ env.SHA }} \ - ops \ - download \ - --provider gfs \ - --workdir /tmp/somisana/current/forcing-inputs \ - --matlab-env /tmp/somisana/current/.env \ - --download-date ${{ env.MODEL_RUN_DATE }} \ - --domain 22,31,-37,-31 + download \ + --provider gfs \ + --workdir /tmp/somisana/current/forcing-inputs \ + --matlab-env /tmp/somisana/current/.env \ + --download-date ${{ env.MODEL_RUN_DATE }} \ + --domain 22,31,-37,-31 - name: Download Mercator uses: nick-fields/retry@master with: @@ -272,13 +271,12 @@ jobs: -e COPERNICUS_USERNAME=${{ env.COPERNICUS_USERNAME }} \ -e COPERNICUS_PASSWORD=${{ env.COPERNICUS_PASSWORD }} \ ${{ needs.toolkit.outputs.image }}:${{ env.SHA }} \ - ops \ - download \ - --provider mercator \ - --workdir /tmp/somisana/current/forcing-inputs \ - --matlab-env /tmp/somisana/current/.env \ - --download-date ${{ env.MODEL_RUN_DATE }} \ - --domain 22,31,-37,-31 + download \ + --provider mercator \ + --workdir /tmp/somisana/current/forcing-inputs \ + --matlab-env /tmp/somisana/current/.env \ + --download-date ${{ env.MODEL_RUN_DATE }} \ + --domain 22,31,-37,-31 # CROCOTOOLS is a collection of MatLab scripts for converting environmental data (i.e. the boundary data downloaded previously) # into NetCDF files that can be used as input to the CROCO model. https://www.croco-ocean.org/documentation/crocotools-documentation/ diff --git a/.github/workflows/run_false-bay-forecast.yml b/.github/workflows/run_false-bay-forecast.yml index a73bf5e4..d97a731c 100644 --- a/.github/workflows/run_false-bay-forecast.yml +++ b/.github/workflows/run_false-bay-forecast.yml @@ -223,13 +223,12 @@ jobs: -e COPERNICUS_USERNAME=${{ env.COPERNICUS_USERNAME }} \ -e COPERNICUS_PASSWORD=${{ env.COPERNICUS_PASSWORD }} \ ${{ needs.toolkit.outputs.image }} \ - ops \ - download \ - --provider gfs \ - --workdir /tmp/somisana/current/forcing-inputs \ - --matlab-env /tmp/somisana/current/.env \ - --download-date ${{ env.MODEL_RUN_DATE }} \ - --domain 15,24.5,-37,-31 + download \ + --provider gfs \ + --workdir /tmp/somisana/current/forcing-inputs \ + --matlab-env /tmp/somisana/current/.env \ + --download-date ${{ env.MODEL_RUN_DATE }} \ + --domain 15,24.5,-37,-31 - name: Download Mercator uses: nick-fields/retry@master with: @@ -248,13 +247,12 @@ jobs: -e COPERNICUS_USERNAME=${{ env.COPERNICUS_USERNAME }} \ -e COPERNICUS_PASSWORD=${{ env.COPERNICUS_PASSWORD }} \ ${{ needs.toolkit.outputs.image }} \ - ops \ - download \ - --provider mercator \ - --workdir /tmp/somisana/current/forcing-inputs \ - --matlab-env /tmp/somisana/current/.env \ - --download-date ${{ env.MODEL_RUN_DATE }} \ - --domain 15,24.5,-37,-31 + download \ + --provider mercator \ + --workdir /tmp/somisana/current/forcing-inputs \ + --matlab-env /tmp/somisana/current/.env \ + --download-date ${{ env.MODEL_RUN_DATE }} \ + --domain 15,24.5,-37,-31 # CROCOTOOLS is a collection of MatLab scripts for converting environmental data (i.e. the boundary data downloaded previously) # into NetCDF files that can be used as input to the CROCO model. https://www.croco-ocean.org/documentation/crocotools-documentation/ diff --git a/.vscode/.filedump b/.vscode/.filedump new file mode 100644 index 00000000..febcd7c5 --- /dev/null +++ b/.vscode/.filedump @@ -0,0 +1,6 @@ + + begin + f = addfile("/home/zach/code/saeon/repositories/somisana/toolkit/.output/mercator_20230425.nc","r") + print(f) + end + \ No newline at end of file diff --git a/toolkit/README.md b/toolkit/README.md index a4a7b8f4..a472ae16 100644 --- a/toolkit/README.md +++ b/toolkit/README.md @@ -81,24 +81,22 @@ cd $SOMISANA_DIR ```sh somisana \ - ops \ - download \ - --workdir $WORKDIR/forcing-inputs \ - --matlab-env $WORKDIR/.env \ - --provider gfs \ - --domain 22,31,-37,-31 + download \ + --workdir $WORKDIR/forcing-inputs \ + --matlab-env $WORKDIR/.env \ + --provider gfs \ + --domain 22,31,-37,-31 ``` **_(3) Download Mercator boundary data_** ```sh somisana \ - ops \ - download \ - --workdir $WORKDIR/forcing-inputs \ - --matlab-env $WORKDIR/.env \ - --provider mercator \ - --domain 22,31,-37,-31 + download \ + --workdir $WORKDIR/forcing-inputs \ + --matlab-env $WORKDIR/.env \ + --provider mercator \ + --domain 22,31,-37,-31 ``` **_(4) Create forcing files_** diff --git a/toolkit/__main__.py b/toolkit/__main__.py index 765abdc4..36adf101 100644 --- a/toolkit/__main__.py +++ b/toolkit/__main__.py @@ -21,7 +21,10 @@ def main(): # (1) Build applications app_objects = { - "ops": (getattr(getattr(define, "ops"), "build")(module_parser), apps.ops), + "download": ( + getattr(getattr(define, "download"), "build")(module_parser), + apps.download, + ), "mhw": (getattr(getattr(define, "mhw"), "build")(module_parser), apps.mhw), "lacce": ( getattr(getattr(define, "lacce"), "build")(module_parser), diff --git a/toolkit/cli/applications/__init__.py b/toolkit/cli/applications/__init__.py index 5e87e605..48c44c9b 100644 --- a/toolkit/cli/applications/__init__.py +++ b/toolkit/cli/applications/__init__.py @@ -1,3 +1,3 @@ -from cli.applications import lacce, mhw, ops, kerchunk, update, pg, croco +from cli.applications import lacce, mhw, kerchunk, update, pg, croco, download -__all__ = ["lacce", "mhw", "ops", "kerchunk", "update", "pg", "croco"] +__all__ = ["lacce", "mhw", "kerchunk", "update", "pg", "croco", "download"] diff --git a/toolkit/cli/applications/ops/download/__init__.py b/toolkit/cli/applications/download/__init__.py similarity index 91% rename from toolkit/cli/applications/ops/download/__init__.py rename to toolkit/cli/applications/download/__init__.py index 3d583b23..caf9e16b 100644 --- a/toolkit/cli/applications/ops/download/__init__.py +++ b/toolkit/cli/applications/download/__init__.py @@ -1,8 +1,8 @@ import os from datetime import datetime, timedelta from pathlib import Path -from cli.applications.ops.download.mercator import download as mercator_download -from cli.applications.ops.download.gfs import download as gfs_download +from cli.applications.download.mercator import download as mercator_download +from cli.applications.download.gfs import download as gfs_download def download(args): diff --git a/toolkit/cli/applications/ops/download/gfs/__init__.py b/toolkit/cli/applications/download/gfs/__init__.py similarity index 98% rename from toolkit/cli/applications/ops/download/gfs/__init__.py rename to toolkit/cli/applications/download/gfs/__init__.py index e8dd0df9..fe5bb9bd 100644 --- a/toolkit/cli/applications/ops/download/gfs/__init__.py +++ b/toolkit/cli/applications/download/gfs/__init__.py @@ -1,6 +1,6 @@ import asyncio from datetime import datetime, timedelta, time -from cli.applications.ops.download.gfs.functions import ( +from cli.applications.download.gfs.functions import ( download_file, get_latest_available_dt, set_params, diff --git a/toolkit/cli/applications/ops/download/gfs/functions.py b/toolkit/cli/applications/download/gfs/functions.py similarity index 100% rename from toolkit/cli/applications/ops/download/gfs/functions.py rename to toolkit/cli/applications/download/gfs/functions.py diff --git a/toolkit/cli/applications/ops/download/mercator/__init__.py b/toolkit/cli/applications/download/mercator/__init__.py similarity index 100% rename from toolkit/cli/applications/ops/download/mercator/__init__.py rename to toolkit/cli/applications/download/mercator/__init__.py diff --git a/toolkit/cli/applications/ops/__init__.py b/toolkit/cli/applications/ops/__init__.py deleted file mode 100644 index 5fa043de..00000000 --- a/toolkit/cli/applications/ops/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from cli.applications.ops.download import download - -__all__ = ["download"] diff --git a/toolkit/cli/define/__init__.py b/toolkit/cli/define/__init__.py index a856e4d6..87986e15 100644 --- a/toolkit/cli/define/__init__.py +++ b/toolkit/cli/define/__init__.py @@ -1,3 +1,3 @@ -from cli.define import lacce, mhw, ops, kerchunk, update, pg, croco +from cli.define import lacce, mhw, download, kerchunk, update, pg, croco -__all__ = ["lacce", "mhw", "ops", "kerchunk", "update", "pg", "croco"] +__all__ = ["lacce", "mhw", "download", "kerchunk", "update", "pg", "croco"] diff --git a/toolkit/cli/define/ops/__init__.py b/toolkit/cli/define/download/__init__.py similarity index 55% rename from toolkit/cli/define/ops/__init__.py rename to toolkit/cli/define/download/__init__.py index 6df1159c..b6b7dfde 100644 --- a/toolkit/cli/define/ops/__init__.py +++ b/toolkit/cli/define/download/__init__.py @@ -4,49 +4,40 @@ def build(module_parser): - ops = module_parser.add_parser("ops", help="CROCO Operational Models module") - ops_parser = ops.add_subparsers( - title="Operational ocean forecasting", - description="Run localised, high resolution ocean forecasts using the CROCO modelling suite", - dest="ops_command", - metavar="Available commands", + download = module_parser.add_parser( + "download", help="Download boundary conditions and forcing files" ) - - # DOWNLOAD - ops_download = ops_parser.add_parser( - "download", help="Download forcing input files" - ) - ops_download.add_argument( + download.add_argument( "--workdir", default=".output", help="Directory output of forcing files" ) - ops_download.add_argument( + download.add_argument( "--matlab-env", default=".output/.env", help="Path to MatLab configuration file" ) - ops_download.add_argument( + download.add_argument( "--download-date", default=NOW, help="Download date (yyyymmdd)" ) - ops_download.add_argument( + download.add_argument( "--provider", type=str, choices=["gfs", "mercator"], help="Forcing data provider", required=True, ) - ops_download.add_argument( + download.add_argument( "--hdays", type=int, default=5, help="Hindcast download days", ) - ops_download.add_argument( + download.add_argument( "--fdays", type=int, default=5, help="Forecast download days", ) - ops_download.add_argument( + download.add_argument( "--domain", help="Bounding box in 4326 projection (i.e. min_long,max_long,min_lat,max_lat)", required=True, ) - return ops + return download diff --git a/toolkit/cli/parse/__init__.py b/toolkit/cli/parse/__init__.py index 1bef0bbc..695a741e 100644 --- a/toolkit/cli/parse/__init__.py +++ b/toolkit/cli/parse/__init__.py @@ -1,3 +1,3 @@ -from cli.parse import lacce, mhw, ops, kerchunk, update, pg, croco +from cli.parse import lacce, mhw, download, kerchunk, update, pg, croco -__all__ = ["lacce", "mhw", "ops", "kerchunk", "update", "pg", "croco"] +__all__ = ["lacce", "mhw", "download", "kerchunk", "update", "pg", "croco"] diff --git a/toolkit/cli/parse/download/__init__.py b/toolkit/cli/parse/download/__init__.py new file mode 100644 index 00000000..19be0aa2 --- /dev/null +++ b/toolkit/cli/parse/download/__init__.py @@ -0,0 +1,10 @@ +def parse(cmd, args, module): + def e(*args): + print(cmd.format_help()) + exit() + + try: + return module.download + except Exception as e: + print(e) + return e diff --git a/toolkit/cli/parse/ops/__init__.py b/toolkit/cli/parse/ops/__init__.py deleted file mode 100644 index d44c6183..00000000 --- a/toolkit/cli/parse/ops/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -def parse(cmd, args, module): - if args.ops_command == "download": - return module.download - else: - - def e(*args): - print(cmd.format_help()) - exit() - - return e