From 46f37249cc7b3621a0eb107bf7f0a329074d6a03 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Tue, 25 Apr 2023 14:19:30 +0200 Subject: [PATCH] refactored away a cli arg for ppv1 (the date can be determined by looking inside the croco output file) --- .../croco/load_pp_v1_output_to_pg/__init__.py | 2 +- .../load_pp_v1_output_to_pg/upsert_model_run/__init__.py | 7 ++++++- .../upsert_model_run/upsert_values/__init__.py | 6 +++--- .../cli/applications/croco/post_process_v1/__init__.py | 7 ++++--- toolkit/cli/define/croco/__init__.py | 8 -------- .../croco/load_pp_v1_output_to_pg => }/models.json | 0 6 files changed, 14 insertions(+), 16 deletions(-) rename toolkit/{cli/applications/croco/load_pp_v1_output_to_pg => }/models.json (100%) diff --git a/toolkit/cli/applications/croco/load_pp_v1_output_to_pg/__init__.py b/toolkit/cli/applications/croco/load_pp_v1_output_to_pg/__init__.py index 184d9453..70c444c6 100644 --- a/toolkit/cli/applications/croco/load_pp_v1_output_to_pg/__init__.py +++ b/toolkit/cli/applications/croco/load_pp_v1_output_to_pg/__init__.py @@ -14,9 +14,9 @@ upsert_model_run, ) +models_file = os.path.abspath("models.json") current_dir = os.path.dirname(os.path.abspath(__file__)) -models_file = os.path.join(current_dir, "models.json") merge_details_file = os.path.join(current_dir, "upsert_model_info/upsert-details.sql") merge_coordinates_file = os.path.join( current_dir, "upsert_model_info/upsert-coordinates.sql" diff --git a/toolkit/cli/applications/croco/load_pp_v1_output_to_pg/upsert_model_run/__init__.py b/toolkit/cli/applications/croco/load_pp_v1_output_to_pg/upsert_model_run/__init__.py index fb3dea9a..7cf17ddf 100644 --- a/toolkit/cli/applications/croco/load_pp_v1_output_to_pg/upsert_model_run/__init__.py +++ b/toolkit/cli/applications/croco/load_pp_v1_output_to_pg/upsert_model_run/__init__.py @@ -78,7 +78,12 @@ async def upsert_model_run(pool, id, run_date, ds, input, model, parallelization # Upsert values datetimes = ds.time.values total_depth_levels = ds.sizes["depth"] - await upsert_values(runid, datetimes, total_depth_levels, parallelization) + total_timesteps = ds.sizes["time"] + log("Total depth levels", total_depth_levels) + log("Total timesteps", total_timesteps) + await upsert_values( + runid, datetimes, total_depth_levels, parallelization, total_timesteps + ) # Finalize the run async with pool.acquire() as conn: diff --git a/toolkit/cli/applications/croco/load_pp_v1_output_to_pg/upsert_model_run/upsert_values/__init__.py b/toolkit/cli/applications/croco/load_pp_v1_output_to_pg/upsert_model_run/upsert_values/__init__.py index 08962e08..4e3dd94e 100644 --- a/toolkit/cli/applications/croco/load_pp_v1_output_to_pg/upsert_model_run/upsert_values/__init__.py +++ b/toolkit/cli/applications/croco/load_pp_v1_output_to_pg/upsert_model_run/upsert_values/__init__.py @@ -5,8 +5,6 @@ import asyncio import asyncpg -total_timesteps = 240 - async def load_worker(queue, async_pool, runid, datetimes, total_depth_levels): while True: @@ -18,7 +16,9 @@ async def load_worker(queue, async_pool, runid, datetimes, total_depth_levels): queue.task_done() -async def upsert_values(runid, datetimes, total_depth_levels, parallelization): +async def upsert_values( + runid, datetimes, total_depth_levels, parallelization, total_timesteps +): depth_levels = [*range(int(1), int(total_depth_levels) + 1, 1)] async_pool = await asyncpg.create_pool( database=PG_DB, diff --git a/toolkit/cli/applications/croco/post_process_v1/__init__.py b/toolkit/cli/applications/croco/post_process_v1/__init__.py index e7583315..f70e37e4 100644 --- a/toolkit/cli/applications/croco/post_process_v1/__init__.py +++ b/toolkit/cli/applications/croco/post_process_v1/__init__.py @@ -26,14 +26,12 @@ def post_process_v1(args): grid = os.path.abspath(args.grid) input = os.path.abspath(args.input) output = os.path.abspath(args.output) - run_date = args.run_date log("Running CROCO output post-processing (v1)") log("CONFIG::id", id) log("CONFIG::input", input) log("CONFIG::grid", grid) log("CONFIG::output", output) - log("CONFIG::run_date", run_date) # Ensure the directory for the specified output exists os.makedirs(os.path.dirname(output), exist_ok=True) @@ -52,7 +50,10 @@ def post_process_v1(args): date_now = REFERENCE_DATE + timedelta(seconds=np.float64(t)) date_round = hour_rounder(date_now) time_steps.append(date_round) - log("Generated time steps") + + # Get the run_date from the CROCO output NetCDF file + run_date = time_steps[119].strftime("%Y%m%d") + log("CONFIG::run_date", run_date) # Variables used in the visualisations temperature = data.temp.values diff --git a/toolkit/cli/define/croco/__init__.py b/toolkit/cli/define/croco/__init__.py index 2fe8d3a3..3e5cfe68 100644 --- a/toolkit/cli/define/croco/__init__.py +++ b/toolkit/cli/define/croco/__init__.py @@ -1,8 +1,3 @@ -from datetime import datetime - -NOW = datetime.now().strftime("%Y%m%d") - - def build(module_parser): croco = module_parser.add_parser("croco", help="CROCO module") croco_parser = croco.add_subparsers( @@ -50,9 +45,6 @@ def build(module_parser): help="Path of processed output path", default=".output/croco/post-process-v1-output.nc", ) - croco_post_process_v1.add_argument( - "--run-date", default=NOW, help="Run date (yyyymmdd)" - ) """ Post-processing V2: diff --git a/toolkit/cli/applications/croco/load_pp_v1_output_to_pg/models.json b/toolkit/models.json similarity index 100% rename from toolkit/cli/applications/croco/load_pp_v1_output_to_pg/models.json rename to toolkit/models.json