From 1ddaff0db12ff2521c794812ec944d13651c6197 Mon Sep 17 00:00:00 2001 From: Emily Soth Date: Thu, 28 Mar 2024 12:49:22 -0700 Subject: [PATCH 01/11] move gdal constraint into constraints file --- constraints_tests.txt | 5 +++++ requirements.txt | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 constraints_tests.txt diff --git a/constraints_tests.txt b/constraints_tests.txt new file mode 100644 index 0000000000..7599c49196 --- /dev/null +++ b/constraints_tests.txt @@ -0,0 +1,5 @@ +# This file contains package constraints needed to run the invest test suite + +# A gdal bug caused our test suite to fail, but this issue is unlikely to +# occur with regular use of invest. https://github.com/OSGeo/gdal/issues/8497 +GDAL!=3.6.*,!=3.7.* diff --git a/requirements.txt b/requirements.txt index cad4fa0e7f..d20cae2cfa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ # scripts/convert-requirements-to-conda-yml.py as though it can only be found # on pip. -GDAL>=3.4.2,<3.6.0 +GDAL>=3.4.2 Pyro4==4.77 # pip-only pandas>=1.2.1 numpy>=1.11.0,!=1.16.0 From 91ececb664dc2c1e53a741a1aed57266395ab197 Mon Sep 17 00:00:00 2001 From: Emily Soth Date: Thu, 28 Mar 2024 15:48:36 -0700 Subject: [PATCH 02/11] try adding constraints to requirements for github actions tests --- .github/workflows/build-and-test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index dc61a18bd4..a8617da384 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -79,7 +79,6 @@ jobs: with: fetch-depth: 0 # Fetch complete history for accurate versioning - # NOTE: It takes twice as long to save the sample data cache # as it does to do a fresh clone (almost 5 minutes vs. 2.5 minutes) # Test data is way, way faster by contrast (on the order of a few @@ -94,7 +93,7 @@ jobs: uses: ./.github/actions/setup_env with: python-version: ${{ matrix.python-version }} - requirements-files: requirements.txt requirements-dev.txt + requirements-files: requirements.txt requirements-dev.txt constraints-test.txt requirements: ${{ env.CONDA_DEFAULT_DEPENDENCIES }} - name: Download previous conda environment.yml From 316e4ddc1d93d43328547533d754c7f464262f7a Mon Sep 17 00:00:00 2001 From: Emily Soth Date: Thu, 28 Mar 2024 15:50:21 -0700 Subject: [PATCH 03/11] add exception for FutureWarnings to pytest config --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fed1937a6c..554e0ccc2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,4 +78,4 @@ where = ["src"] [tool.pytest.ini_options] # raise warnings to errors, except for deprecation warnings -filterwarnings = ["error", "default::DeprecationWarning"] +filterwarnings = ["error", "default::DeprecationWarning", "default::FutureWarning"] From e87dcafa6e9d244a3c1e7222f2fefcaffdefafd1 Mon Sep 17 00:00:00 2001 From: Emily Soth Date: Thu, 28 Mar 2024 15:53:02 -0700 Subject: [PATCH 04/11] typo --- .github/workflows/build-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index a8617da384..fb7412562e 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -93,7 +93,7 @@ jobs: uses: ./.github/actions/setup_env with: python-version: ${{ matrix.python-version }} - requirements-files: requirements.txt requirements-dev.txt constraints-test.txt + requirements-files: requirements.txt requirements-dev.txt constraints_tests.txt requirements: ${{ env.CONDA_DEFAULT_DEPENDENCIES }} - name: Download previous conda environment.yml From f9051f5a686e3db6cfa72f89147e3e74b63fd9e0 Mon Sep 17 00:00:00 2001 From: Emily Soth Date: Mon, 1 Apr 2024 10:16:43 -0700 Subject: [PATCH 05/11] hra: should rasterize with ALL_TOUCHED=TRUE --- src/natcap/invest/hra.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/natcap/invest/hra.py b/src/natcap/invest/hra.py index 5d5c0d477c..0a7398c847 100644 --- a/src/natcap/invest/hra.py +++ b/src/natcap/invest/hra.py @@ -633,10 +633,6 @@ def execute(args): # If the input is a vector, reproject to the AOI SRS and simplify. # Rasterization happens in the alignment step. elif gis_type == pygeoprocessing.VECTOR_TYPE: - # Habitats and stressors are rasterized with ALL_TOUCHED=TRUE - if name in habitats_info or name in stressors_info: - habitat_stressor_vectors.add(source_filepath) - # Using Shapefile here because its driver appears to not raise a # warning if a MultiPolygon geometry is inserted into a Polygon # layer, which was happening on a real-world sample dataset while @@ -680,6 +676,10 @@ def execute(args): dependent_task_list=[reprojected_vector_task] )) + # Habitats and stressors are rasterized with ALL_TOUCHED=TRUE + if name in habitats_info or name in stressors_info: + habitat_stressor_vectors.add(target_simplified_vector) + # Later operations make use of the habitats rasters or the stressors # rasters, so it's useful to collect those here now. if name in habitats_info: @@ -1648,7 +1648,6 @@ def _simplify(source_vector_path, tolerance, target_vector_path, for source_feature in source_layer: target_feature = ogr.Feature(target_layer_defn) source_geom = source_feature.GetGeometryRef() - simplified_geom = source_geom.SimplifyPreserveTopology(tolerance) if simplified_geom is not None: target_geom = simplified_geom From a289f98b89a6aafa2470792a88daed6b381e2036 Mon Sep 17 00:00:00 2001 From: Emily Soth Date: Mon, 1 Apr 2024 12:38:47 -0700 Subject: [PATCH 06/11] ndr: binary flow dir intermediate raster should be uint8, not int8 --- src/natcap/invest/ndr/ndr_core.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/natcap/invest/ndr/ndr_core.pyx b/src/natcap/invest/ndr/ndr_core.pyx index b56ba40d12..f39d35b035 100644 --- a/src/natcap/invest/ndr/ndr_core.pyx +++ b/src/natcap/invest/ndr/ndr_core.pyx @@ -415,9 +415,9 @@ def ndr_eff_calculation( # create direction raster in bytes def _mfd_to_flow_dir_op(mfd_array): - result = numpy.zeros(mfd_array.shape, dtype=numpy.int8) + result = numpy.zeros(mfd_array.shape, dtype=numpy.uint8) for i in range(8): - result[:] |= (((mfd_array >> (i*4)) & 0xF) > 0) << i + result[:] |= ((((mfd_array >> (i*4)) & 0xF) > 0) << i).astype(numpy.uint8) return result pygeoprocessing.raster_calculator( From fa213e54ba9cd03ea1cbb2c154e74257d929ce4f Mon Sep 17 00:00:00 2001 From: Emily Soth Date: Mon, 1 Apr 2024 15:25:17 -0700 Subject: [PATCH 07/11] fix bug in validation test revealed by gdal>=3.6.0 --- tests/test_validation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_validation.py b/tests/test_validation.py index 9d6d0d45d9..d756bd2fa8 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -2002,6 +2002,7 @@ def test_spatial_overlap_error(self): layer = vector.CreateLayer('layer', vector_srs, ogr.wkbPoint) new_feature = ogr.Feature(layer.GetLayerDefn()) new_feature.SetGeometry(ogr.CreateGeometryFromWkt('POINT (1 1)')) + layer.CreateFeature(new_feature) new_feature = None layer = None From 7a148dd46866f6d2bc4234b11163ffe82beea685 Mon Sep 17 00:00:00 2001 From: Emily Soth Date: Mon, 1 Apr 2024 16:06:52 -0700 Subject: [PATCH 08/11] remove gdal pin for readthedocs --- .readthedocs_environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs_environment.yml b/.readthedocs_environment.yml index 1d1681aa5f..ef61a90b8e 100644 --- a/.readthedocs_environment.yml +++ b/.readthedocs_environment.yml @@ -11,7 +11,7 @@ channels: - nodefaults dependencies: - python=3.11 -- gdal>=3.4.2,<3.6.0 +- gdal>=3.4.2 - pip - pip: - -r requirements.txt From 8d6164b2973331ff6e9b741bf18e5777e01bb3a7 Mon Sep 17 00:00:00 2001 From: Emily Soth Date: Mon, 1 Apr 2024 16:19:29 -0700 Subject: [PATCH 09/11] add history note --- HISTORY.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 439ca6d11c..65cc76bd85 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -68,6 +68,9 @@ Unreleased Changes raster. ``nan`` pixels will now be propertly ignored before calculating mean depths along fetch rays. https://github.com/natcap/invest/issues/1528 +* HRA + * Fixed a bug where habitat and stressor vectors were not being rasterized + with the `ALL_TOUCHED=TRUE` setting. * Urban Nature Access * Fixed a ``NameError`` that occurred when running the model using search radii defined per population group with an exponential search From 1c2476c9f498a64ff5ad1605798422ed38632ce1 Mon Sep 17 00:00:00 2001 From: Emily Soth Date: Mon, 1 Apr 2024 16:20:48 -0700 Subject: [PATCH 10/11] history note --- HISTORY.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 65cc76bd85..3db8ad4fe7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -59,7 +59,9 @@ Unreleased Changes versions of InVEST would skip these parameters' type-specific validation. Now, these parameters will be validated with their type-specific validation checks. - + * Add support for latest GDAL versions; remove test-specific constraint on + GDAL versions from invest requirements. + https://github.com/natcap/invest/issues/916 * Annual Water Yield * Added the results_suffix to a few intermediate files where it was missing. https://github.com/natcap/invest/issues/1517 From 4c35b1a8625ddc18a86c18080cc435e1f93927c2 Mon Sep 17 00:00:00 2001 From: Emily Soth Date: Wed, 3 Apr 2024 10:48:10 -0700 Subject: [PATCH 11/11] add note to constraints file about format --- constraints_tests.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/constraints_tests.txt b/constraints_tests.txt index 7599c49196..362d96b589 100644 --- a/constraints_tests.txt +++ b/constraints_tests.txt @@ -1,4 +1,6 @@ -# This file contains package constraints needed to run the invest test suite +# This file contains package constraints needed to run the invest test suite. +# It follows the pip constraints file format: +# https://pip.pypa.io/en/stable/user_guide/#constraints-files # A gdal bug caused our test suite to fail, but this issue is unlikely to # occur with regular use of invest. https://github.com/OSGeo/gdal/issues/8497