Feat flow dag

.github/workflows/ci.yaml

@@ -179,6 +179,10 @@ jobs:
           pip -V
           pip list
 
+      - name: Get GIS files for tests
+        run: |
+          python pywatershed/utils/gis_files.py
+
       - name: domainless - run tests no requiring domain data
         working-directory: autotest
         run: pytest -m domainless -n=auto -vv
@@ -200,7 +204,7 @@ jobs:
         run: pytest
           -vv
           -n=auto
-          -m domain
+          -m "not domainless"
           --domain=sagehen_5yr
           --control_pattern=sagehen_no_cascades.control
           --durations=0
@@ -225,15 +229,15 @@ jobs:
         run: pytest
           -vv
           -n=auto
-          -m domain
+          -m "not domainless"
           --domain=hru_1
           --control_pattern=nhm.control
           --durations=0
           --cov=pywatershed
           --cov-report=xml
           --junitxml=pytest_hru_1_nhm.xml
 
-      - name: drb_2yr with and without dprst - generate and manage test data
+      - name: drb_2yr with and without dprst and obsin - generate and manage test data
         working-directory: autotest
         run: |
           python generate_test_data.py \
@@ -255,7 +259,7 @@ jobs:
         run: pytest
           -vv
           -n=auto
-          -m domain
+          -m "not domainless"
           --domain=drb_2yr
           --control_pattern=nhm.control
           --durations=0
@@ -274,14 +278,30 @@ jobs:
           test_prms_below_snow.py
           -vv
           -n=auto
-          -m domain
+          -m "not domainless"
           --domain=drb_2yr
           --control_pattern=no_dprst
           --durations=0
           --cov=pywatershed
           --cov-report=xml
           --junitxml=pytest_drb_2yr_no_dprst.xml
 
+      # Specific tests not redundant with dprst
+      - name: drb_2yr_obsin - pywatershed tests
+        working-directory: autotest
+        run: pytest
+          test_obsin_flow_node.py
+          -vv
+          -n=0
+          -m "not domainless"
+          --domain=drb_2yr
+          --control_pattern=nhm_obsin.control
+          --durations=0
+          --cov=pywatershed
+          --cov-report=xml
+          --junitxml=pytest_drb_2yr_no_dprst.xml
+
+
       - name: ucb_2yr_nhm - generate and manage test data
         working-directory: autotest
         run: |
@@ -298,7 +318,7 @@ jobs:
         working-directory: autotest
         run: pytest
           -vv
-          -m domain
+          -m "not domainless"
           --domain=ucb_2yr
           --control_pattern=nhm.control
           --durations=0

.github/workflows/ci_examples.yaml

@@ -37,12 +37,6 @@ jobs:
           compiler: gcc
           version: 11
 
-      - name: Download GIS files
-        working-directory: examples
-        run: |
-          curl -LO https://github.com/EC-USGS/pywatershed/releases/download/v2022.0.1/pynhm_gis.zip
-          unzip pynhm_gis.zip
-
       - name: Set environment variables
         run: |
           echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV
@@ -78,6 +72,14 @@ jobs:
         run: |
           python -c "import pywatershed"
 
+      - name: Get GIS files for tests
+        run: |
+          python pywatershed/utils/gis_files.py
+
+      - name: Get additional domain files for notebooks
+        run: |
+          python pywatershed/utils/addtl_domain_files.py
+
       - name: Run tests
         working-directory: autotest_exs
         run: |

DISCLAIMER.md

@@ -1,3 +1,11 @@
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents**  *generated with [DocToc](https://github.com/thlorenz/doctoc)*
+
+- [Disclaimer](#disclaimer)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
 Disclaimer
 ==========
 

asv_benchmarks/README.md

@@ -1,3 +1,11 @@
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents**  *generated with [DocToc](https://github.com/thlorenz/doctoc)*
+
+- [ASV Benchmarking](#asv-benchmarking)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
 # ASV Benchmarking
 
 [https://github.com/airspeed-velocity/asv/](https://github.com/airspeed-velocity/asv/)

autotest/conftest.py

@@ -144,10 +144,10 @@ def pytest_generate_tests(metafunc):
 
     control_pattern_list = metafunc.config.getoption("control_pattern")
 
-    if (
-        "domainless" not in metafunc.config.option.markexpr
-        and "simulation" in metafunc.fixturenames
-    ):
+    if "domainless" == metafunc.config.option.markexpr:
+        return
+
+    if "simulation" in metafunc.fixturenames:
         simulations = collect_simulations(domain_list, control_pattern_list)
 
         # Put --print_ans in the domain fixture as it applies only to the
@@ -165,3 +165,5 @@ def pytest_generate_tests(metafunc):
         metafunc.parametrize(
             "simulation", simulations.values(), ids=simulations.keys()
         )
+
+    return

autotest/test_budget.py

@@ -53,6 +53,7 @@ def test_budget(control_simple):
         time_unit="D",
         description="simple_test",
         units="m*3/D",
+        verbose=False,
     )
     budget.set(terms)
 

autotest/test_cbh_to_netcdf.py

@@ -82,7 +82,6 @@ def params_and_none(request, simulation, control):
 }
 
 
-@pytest.mark.domain
 @pytest.mark.parametrize("var", [var_cases[0]])
 def test_cbh_files_to_df(simulation, var, params):
     the_file = simulation["dir"] / f"{var}.cbh"
@@ -96,7 +95,6 @@ def test_cbh_files_to_df(simulation, var, params):
     return
 
 
-@pytest.mark.domain
 def test_cbh_file_to_netcdf(simulation, params, tmp_path):
     input_files_dict = {
         ff.with_suffix("").name: ff for ff in simulation["dir"].glob("*.cbh")

autotest/test_control.py

@@ -154,14 +154,12 @@ def test_control_advance(control_simple, params_simple):
         #     )
 
 
-@pytest.mark.domain
 def test_init_load(simulation):
     with pytest.warns(RuntimeWarning):
         _ = Control.load_prms(simulation["control_file"])
     return None
 
 
-@pytest.mark.domain
 def test_deepcopy(simulation):
     ctl = Control.load_prms(
         simulation["control_file"], warn_unused_options=False
@@ -178,7 +176,6 @@ def test_deepcopy(simulation):
     return None
 
 
-@pytest.mark.domain
 def test_setitem_setattr(simulation):
     ctl = Control.load_prms(
         simulation["control_file"], warn_unused_options=False
@@ -204,7 +201,6 @@ def test_setitem_setattr(simulation):
         ctl.options = None
 
 
-@pytest.mark.domain
 def test_yaml_roundtrip(simulation, tmp_path):
     ctl = Control.load_prms(
         simulation["control_file"], warn_unused_options=False

autotest/test_control_read.py

@@ -34,7 +34,6 @@
 )
 
 
-@pytest.mark.domain
 def test_control_read(simulation):
     domain_name = simulation["name"].split(":")[0]
     if domain_name not in test_answers.keys():

autotest/test_netcdf_output.py

@@ -75,7 +75,6 @@ def check_vars():
 check_budget_sum_vars_params = [False, True, "some"]
 
 
-@pytest.mark.domain
 @pytest.mark.filterwarnings("ignore:Budget for")
 @pytest.mark.parametrize(
     "budget_sum_param",
@@ -184,12 +183,12 @@ def test_process_budgets(
             for bb in check_budget_sum_vars:
                 if tt == 0:
                     # use the output data to figure out the shape
-                    check_dict[pp][bb] = np.zeros(
-                        (
-                            n_time_steps,
-                            model.processes[pp].budget[f"_{bb}"].shape[0],
-                        )
-                    )
+                    shp = model.processes[pp].budget[f"_{bb}"].shape
+                    if len(shp):
+                        shp = shp[0]
+                    else:
+                        shp = 1
+                    check_dict[pp][bb] = np.zeros((n_time_steps, shp))
 
                 check_dict[pp][bb][tt, :] = model.processes[pp].budget[
                     f"_{bb}"
@@ -245,7 +244,6 @@ def sep_vars(request):
     return (request.param[0], request.param[1])
 
 
-@pytest.mark.domain
 def test_separate_together_var_list(
     simulation, control, params, tmp_path, sep_vars, check_vars
 ):

autotest/test_netcdf_read.py

@@ -1,9 +1,6 @@
-import pytest
-
 from pywatershed.utils.netcdf_utils import NetCdfRead
 
 
-@pytest.mark.domain
 def test_netcdf(simulation):
     variable = "gwres_stor"
     output_dir = simulation["output_dir"]