Merge branch 'ti' into mapx-dtype-updates

NOAA-OWP · Dec 19, 2024 · 1ab54df · 1ab54df
2 parents 9d5afb7 + 242ced9
commit 1ab54df
Show file tree

Hide file tree

Showing 33 changed files with 4,650 additions and 106 deletions.
diff --git a/.gitignore b/.gitignore
@@ -53,4 +53,8 @@ terraform.rc
 # Ignore all jupyter checkpoint folders
 .ipynb_checkpoints
 
-Dev/env.yaml
+Dev/env.yaml
+
+# Ignore pycachefiles
+**/__pycache__/*
+**/__pycache__
diff --git a/...s_10day_peak_flow_arrival_time_alaska.sql → ...f_gfs_10day_peak_flow_arrival_time_ak.sql b/...s_10day_peak_flow_arrival_time_alaska.sql → ...f_gfs_10day_peak_flow_arrival_time_ak.sql
@@ -1,29 +1,21 @@
 DROP TABLE IF EXISTS publish.mrf_gfs_10day_peak_flow_arrival_time_alaska;
 
-WITH arrival_time AS(
-    SELECT 
-           forecasts.feature_id, 
-           max(forecasts.forecast_hour)+1 AS below_bank_return_hour
-    FROM ingest.nwm_channel_rt_mrf_gfs_ak_mem1 AS forecasts
-    GROUP BY forecasts.feature_id
-)
-
 SELECT
     forecasts.feature_id,
     forecasts.feature_id::TEXT AS feature_id_str,
+    channels.name,
+    channels.strm_order::integer,
     min(forecasts.forecast_hour) AS peak_flow_arrival_hour,
     to_char(forecasts.reference_time::timestamp without time zone + INTERVAL '1 hour' * min(forecasts.forecast_hour), 'YYYY-MM-DD HH24:MI:SS UTC') AS peak_flow_arrival_time,
+    channels.huc6,
+    'AK' as state,
     forecasts.nwm_vers,
     forecasts.reference_time,
     max_flows.discharge_cfs AS max_flow_cfs,
+    rf.high_water_threshold,
     arrival_time.below_bank_return_hour,
-    to_char(forecasts.reference_time::timestamp without time zone + INTERVAL '1 hour' * arrival_time.below_bank_return_hour, 'YYYY-MM-DD HH24:MI:SS UTC') AS below_bank_return_time,
+    arrival_time.below_bank_return_time,
     to_char(now()::timestamp without time zone, 'YYYY-MM-DD HH24:MI:SS UTC') AS update_time,
-    channels.strm_order::integer,
-    channels.name,
-    channels.huc6,
-    'AK' as state,
-    -9999.0 as high_water_threshold,
     channels.geom
 
 INTO publish.mrf_gfs_10day_peak_flow_arrival_time_alaska
@@ -34,9 +26,10 @@ JOIN cache.max_flows_mrf_gfs_10day_ak AS max_flows
     ON forecasts.feature_id = max_flows.feature_id AND round((forecasts.streamflow*35.315)::numeric, 2) = max_flows.discharge_cfs
 
 -- Join in channels data to get reach metadata and geometry
-JOIN derived.channels_alaska as channels ON forecasts.feature_id = channels.feature_id::bigint
+JOIN derived.channels_ak as channels ON forecasts.feature_id = channels.feature_id::bigint
 
--- Join in arrival_time 
-JOIN arrival_time ON forecasts.feature_id = arrival_time.feature_id
+-- Join in high water arrival time for return time (the yaml config file ensures that arrival time finishes first for this, but we'll join on reference_time as well to ensure)
+JOIN publish.mrf_gfs_10day_high_water_arrival_time_ak AS arrival_time ON forecasts.feature_id = arrival_time.feature_id and forecasts.reference_time = arrival_time.reference_time
 
+WHERE round((forecasts.streamflow*35.315)::numeric, 2) >= rf.high_water_threshold
 GROUP BY forecasts.feature_id, forecasts.reference_time, forecasts.nwm_vers, forecasts.streamflow, max_flows.discharge_cfs, arrival_time.below_bank_return_hour, channels.geom, channels.strm_order, channels.name, channels.huc6;
diff --git a/...m_10day_peak_flow_arrival_time_alaska.sql → ...f_nbm_10day_peak_flow_arrival_time_ak.sql b/...m_10day_peak_flow_arrival_time_alaska.sql → ...f_nbm_10day_peak_flow_arrival_time_ak.sql
@@ -1,29 +1,21 @@
 DROP TABLE IF EXISTS publish.mrf_nbm_10day_peak_flow_arrival_time_alaska;
 
-WITH arrival_time AS(
-    SELECT 
-           forecasts.feature_id, 
-           max(forecasts.forecast_hour)+1 AS below_bank_return_hour
-    FROM ingest.nwm_channel_rt_mrf_nbm_ak AS forecasts
-    GROUP BY forecasts.feature_id
-)
-
 SELECT
     forecasts.feature_id,
     forecasts.feature_id::TEXT AS feature_id_str,
+    channels.name,
+    channels.strm_order::integer,
     min(forecasts.forecast_hour) AS peak_flow_arrival_hour,
     to_char(forecasts.reference_time::timestamp without time zone + INTERVAL '1 hour' * min(forecasts.forecast_hour), 'YYYY-MM-DD HH24:MI:SS UTC') AS peak_flow_arrival_time,
+    channels.huc6,
+    'AK' as state,
     forecasts.nwm_vers,
     forecasts.reference_time,
     max_flows.discharge_cfs AS max_flow_cfs,
+    rf.high_water_threshold,
     arrival_time.below_bank_return_hour,
-    to_char(forecasts.reference_time::timestamp without time zone + INTERVAL '1 hour' * arrival_time.below_bank_return_hour, 'YYYY-MM-DD HH24:MI:SS UTC') AS below_bank_return_time,
+    arrival_time.below_bank_return_time,
     to_char(now()::timestamp without time zone, 'YYYY-MM-DD HH24:MI:SS UTC') AS update_time,
-    channels.strm_order::integer,
-    channels.name,
-    channels.huc6,
-    'AK' as state,
-    -9999.0 as high_water_threshold,
     channels.geom
 
 INTO publish.mrf_nbm_10day_peak_flow_arrival_time_alaska
@@ -36,7 +28,8 @@ JOIN cache.max_flows_mrf_nbm_10day_ak AS max_flows
 -- Join in channels data to get reach metadata and geometry
 JOIN derived.channels_alaska as channels ON forecasts.feature_id = channels.feature_id::bigint
 
--- Join in arrival_time 
-JOIN arrival_time ON forecasts.feature_id = arrival_time.feature_id
+-- Join in high water arrival time for return time (the yaml config file ensures that arrival time finishes first for this, but we'll join on reference_time as well to ensure)
+JOIN publish.mrf_nbm_10day_high_water_arrival_time_ak AS arrival_time ON forecasts.feature_id = arrival_time.feature_id and forecasts.reference_time = arrival_time.reference_time
 
+WHERE round((forecasts.streamflow*35.315)::numeric, 2) >= rf.high_water_threshold
 GROUP BY forecasts.feature_id, forecasts.reference_time, forecasts.nwm_vers, forecasts.streamflow, max_flows.discharge_cfs, arrival_time.below_bank_return_hour, channels.geom, channels.strm_order, channels.name, channels.huc6;
diff --git a/...nbm_10day_rapid_onset_flooding_alaska.sql → ...mrf_nbm_10day_rapid_onset_flooding_ak.sql b/...nbm_10day_rapid_onset_flooding_alaska.sql → ...mrf_nbm_10day_rapid_onset_flooding_ak.sql
diff --git a/...f_15hr_high_water_arrival_time_alaska.sql → ...a/srf_15hr_high_water_arrival_time_ak.sql b/...f_15hr_high_water_arrival_time_alaska.sql → ...a/srf_15hr_high_water_arrival_time_ak.sql
diff --git a/...rf_15hr_peak_flow_arrival_time_alaska.sql → ...ka/srf_15hr_peak_flow_arrival_time_ak.sql b/...rf_15hr_peak_flow_arrival_time_alaska.sql → ...ka/srf_15hr_peak_flow_arrival_time_ak.sql
@@ -34,4 +34,5 @@ JOIN derived.recurrence_flows_ak AS rf ON forecasts.feature_id = rf.feature_id
 -- Join in high water arrival time for return time (the yaml config file ensures that arrival time finishes first for this, but we'll join on reference_time as well to ensure)
 JOIN publish.srf_15hr_high_water_arrival_time_alaska AS arrival_time ON forecasts.feature_id = arrival_time.feature_id and forecasts.reference_time = arrival_time.reference_time
 
+WHERE round((forecasts.streamflow*35.315)::numeric, 2) >= rf.high_water_threshold
 GROUP BY forecasts.feature_id, forecasts.reference_time, forecasts.nwm_vers, arrival_time.below_bank_return_hour, arrival_time.below_bank_return_time, max_flows.discharge_cfs, channels.geom, channels.strm_order, channels.name, channels.huc6, rf.high_water_threshold;
diff --git a/...line/product_configs/medium_range_alaska_mem1/mrf_gfs_10day_peak_flow_arrival_time_ak.yml b/...line/product_configs/medium_range_alaska_mem1/mrf_gfs_10day_peak_flow_arrival_time_ak.yml
@@ -18,8 +18,8 @@ db_max_flows:
     max_flows_sql_file: mrf_gfs_10day_max_flows_ak
 
 postprocess_sql:
-  - sql_file: mrf_gfs_10day_peak_flow_arrival_time_alaska
+  - sql_file: mrf_gfs_10day_peak_flow_arrival_time_ak
     target_table: publish.mrf_gfs_10day_peak_flow_arrival_time_alaska
 
 services:
-  - mrf_gfs_10day_peak_flow_arrival_time_alaska_noaa
+  - mrf_gfs_10day_peak_flow_arrival_time_ak_noaa
diff --git a/...ine/product_configs/medium_range_blend_alaska/mrf_nbm_10day_peak_flow_arrival_time_ak.yml b/...ine/product_configs/medium_range_blend_alaska/mrf_nbm_10day_peak_flow_arrival_time_ak.yml
@@ -19,8 +19,8 @@ db_max_flows:
     max_flows_sql_file: mrf_nbm_10day_max_flows_ak
 
 postprocess_sql:
-  - sql_file: mrf_nbm_10day_peak_flow_arrival_time_alaska
+  - sql_file: mrf_nbm_10day_peak_flow_arrival_time_ak
     target_table: publish.mrf_nbm_10day_peak_flow_arrival_time_alaska
 
 services:
-  - mrf_nbm_10day_peak_flow_arrival_time_alaska_noaa
+  - mrf_nbm_10day_peak_flow_arrival_time_ak_noaa
diff --git a/...eline/product_configs/medium_range_blend_alaska/mrf_nbm_10day_rapid_onset_flooding_ak.yml b/...eline/product_configs/medium_range_blend_alaska/mrf_nbm_10day_rapid_onset_flooding_ak.yml
@@ -19,7 +19,7 @@ db_max_flows:
     max_flows_sql_file: mrf_nbm_10day_max_flows_ak
 
 postprocess_sql:
-  - sql_file: mrf_nbm_10day_rapid_onset_flooding_alaska
+  - sql_file: mrf_nbm_10day_rapid_onset_flooding_ak
     target_table: publish.mrf_nbm_10day_rapid_onset_flooding_alaska
 
 product_summaries:

diff --git a/...alize_pipeline/product_configs/short_range_alaska/srf_15hr_high_water_arrival_time_ak.yml b/...alize_pipeline/product_configs/short_range_alaska/srf_15hr_high_water_arrival_time_ak.yml
@@ -1,4 +1,4 @@
-product: srf_15hr_high_water_arrival_time_alaska
+product: srf_15hr_high_water_arrival_time_ak
 configuration: short_range_alaska
 product_type: "vector"
 run: true
@@ -11,8 +11,8 @@ ingest_files:
       target_keys: (feature_id, streamflow)
 
 postprocess_sql:
-  - sql_file: srf_15hr_high_water_arrival_time_alaska
+  - sql_file: srf_15hr_high_water_arrival_time_ak
     target_table: publish.srf_15hr_high_water_arrival_time_alaska
 
 services:
-  - srf_15hr_high_water_arrival_time_alaska_noaa
+  - srf_15hr_high_water_arrival_time_ak_noaa
diff --git a/...alize_pipeline/product_configs/short_range_alaska/srf_15hr_max_high_flow_magnitude_ak.yml b/...alize_pipeline/product_configs/short_range_alaska/srf_15hr_max_high_flow_magnitude_ak.yml
@@ -1,4 +1,4 @@
-product: srf_15hr_max_high_flow_magnitude_alaska
+product: srf_15hr_max_high_flow_magnitude_ak
 configuration: short_range_alaska
 product_type: "vector"
 run: true
@@ -22,4 +22,4 @@ postprocess_sql:
     target_table: publish.srf_15hr_max_high_flow_magnitude_ak
 
 services:
-  - srf_15hr_max_high_flow_magnitude_alaska_noaa
+  - srf_15hr_max_high_flow_magnitude_ak_noaa
diff --git a/...ialize_pipeline/product_configs/short_range_alaska/srf_15hr_peak_flow_arrival_time_ak.yml b/...ialize_pipeline/product_configs/short_range_alaska/srf_15hr_peak_flow_arrival_time_ak.yml
@@ -18,8 +18,8 @@ db_max_flows:
     max_flows_sql_file: srf_max_flows_ak
 
 postprocess_sql:
-  - sql_file: srf_15hr_peak_flow_arrival_time_alaska
+  - sql_file: srf_15hr_peak_flow_arrival_time_ak
     target_table: publish.srf_15hr_peak_flow_arrival_time_alaska
 
 services:
-  - srf_15hr_peak_flow_arrival_time_alaska_noaa
+  - srf_15hr_peak_flow_arrival_time_ak_noaa
diff --git a/...itialize_pipeline/product_configs/short_range_alaska/srf_15hr_rapid_onset_flooding_ak.yml b/...itialize_pipeline/product_configs/short_range_alaska/srf_15hr_rapid_onset_flooding_ak.yml
@@ -22,4 +22,4 @@ postprocess_sql:
     target_table: publish.srf_15hr_rapid_onset_flooding_ak
 
 services:
-  - srf_15hr_rapid_onset_flooding_alaska_noaa
+  - srf_15hr_rapid_onset_flooding_ak_noaa