From e39420927e305b58b7119abe5aec802e4a089e99 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 26 Sep 2024 12:06:11 -0700 Subject: [PATCH 1/5] Using the known nodata value and float32 dtype. RE:#1635 --- HISTORY.rst | 8 ++++++++ src/natcap/invest/pollination.py | 21 ++++++++++++++++----- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index dfa49b3025..348fd1442a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -37,6 +37,14 @@ Unreleased Changes ------------------ +* Pollination + * Fixed an issue with nodata handling that was causing some outputs to be + filled either with the float32 value for positive infinity, or else with + a value very close to it. https://github.com/natcap/invest/issues/1635 + * While working on https://github.com/natcap/invest/issues/1635, we also + updated the stated dtype of most pollination model outputs to be float32 + instead of the float64 dtype that was being assumed previously. This + will result in smaller output filesizes with minimal loss of precision. * Workbench * Several small updates to the model input form UI to improve usability and visual consistency (https://github.com/natcap/invest/issues/912) diff --git a/src/natcap/invest/pollination.py b/src/natcap/invest/pollination.py index 60def5ec3e..e845f2180a 100644 --- a/src/natcap/invest/pollination.py +++ b/src/natcap/invest/pollination.py @@ -855,6 +855,7 @@ def execute(args): floral_resources_index_path_map[species], convolve_ps_path], target_path=pollinator_abundance_path, + target_dtype=numpy.float32, target_nodata=_INDEX_NODATA), dependent_task_list=[ foraged_flowers_index_task_map[(species, season)], @@ -935,7 +936,9 @@ def execute(args): rasters=[ half_saturation_raster_path, total_pollinator_abundance_index_path], - target_path=farm_pollinator_season_path), + target_path=farm_pollinator_season_path, + target_dtype=numpy.float32, + target_nodata=_INDEX_NODATA), dependent_task_list=[ half_saturation_task, total_pollinator_abundance_task[season]], target_path_list=[farm_pollinator_season_path])) @@ -976,7 +979,9 @@ def execute(args): kwargs=dict( op=pyt_op, rasters=[managed_pollinator_path, farm_pollinator_path], - target_path=total_pollinator_yield_path), + target_path=total_pollinator_yield_path, + target_dtype=numpy.float32, + target_nodata=_INDEX_NODATA), dependent_task_list=[farm_pollinator_task, managed_pollinator_task], target_path_list=[total_pollinator_yield_path]) @@ -984,12 +989,14 @@ def execute(args): wild_pollinator_yield_path = os.path.join( output_dir, _WILD_POLLINATOR_YIELD_FILE_PATTERN % file_suffix) wild_pollinator_task = task_graph.add_task( - task_name='calcualte_wild_pollinators', + task_name='calculate_wild_pollinators', func=pygeoprocessing.raster_map, kwargs=dict( op=pyw_op, rasters=[managed_pollinator_path, total_pollinator_yield_path], - target_path=wild_pollinator_yield_path), + target_path=wild_pollinator_yield_path, + target_dtype=numpy.float32, + target_nodata=_INDEX_NODATA), dependent_task_list=[pyt_task, managed_pollinator_task], target_path_list=[wild_pollinator_yield_path]) @@ -1392,7 +1399,8 @@ def _sum_arrays(*array_list): result = numpy.empty_like(array_list[0]) result[:] = 0 for array in array_list: - local_valid_mask = ~pygeoprocessing.array_equals_nodata(array, _INDEX_NODATA) + local_valid_mask = ~pygeoprocessing.array_equals_nodata( + array, _INDEX_NODATA) result[local_valid_mask] += array[local_valid_mask] valid_mask |= local_valid_mask result[~valid_mask] = _INDEX_NODATA @@ -1423,6 +1431,7 @@ def max_op(*substrate_index_arrays): pygeoprocessing.raster_map( op=max_op, rasters=substrate_path_list, + target_dtype=numpy.float32, target_path=target_habitat_nesting_index_path) @@ -1432,6 +1441,7 @@ def _multiply_by_scalar(raster_path, scalar, target_path): op=lambda array: array * scalar, rasters=[raster_path], target_path=target_path, + target_dtype=numpy.float32, target_nodata=_INDEX_NODATA, ) @@ -1455,6 +1465,7 @@ def _calculate_pollinator_supply_index( op=lambda f_r, h_n: species_abundance * f_r * h_n, rasters=[habitat_nesting_suitability_path, floral_resources_path], target_path=target_path, + target_dtype=numpy.float32, target_nodata=_INDEX_NODATA ) From e4cee566afda5c7b504f9d7208ba46d7488bef0d Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 26 Sep 2024 13:46:42 -0700 Subject: [PATCH 2/5] Seeing if env marker handling corrects pip installation issues. RE:#1635 --- scripts/convert-requirements-to-conda-yml.py | 52 +++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/scripts/convert-requirements-to-conda-yml.py b/scripts/convert-requirements-to-conda-yml.py index e5e2bc2d57..594a821a6a 100644 --- a/scripts/convert-requirements-to-conda-yml.py +++ b/scripts/convert-requirements-to-conda-yml.py @@ -2,6 +2,7 @@ """convert-requirements-to-conda-yml.py""" import argparse +import os import platform import sys @@ -14,6 +15,46 @@ """ +# Environment marker handling is taken straight from +# https://peps.python.org/pep-0508/#environment-markers +def _get_implementation_version(): + def format_full_version(info): + version = '{0.major}.{0.minor}.{0.micro}'.format(info) + kind = info.releaselevel + if kind != 'final': + version += kind[0] + str(info.serial) + return version + + if hasattr(sys, 'implementation'): + implementation_version = format_full_version( + sys.implementation.version) + else: + implementation_version = "0" + return implementation_version + + +# Environment marker handling is taken straight from +# https://peps.python.org/pep-0508/#environment-markers +ENV_MARKERS = { + "os_name": os.name(), + "sys_platform": sys.platform, + "platform_machine": platform.machine(), + "platform_python_impl": platform.python_implementation(), + "platform_release": platform.release(), + "platform_system": platform.system(), + "platform_version": platform.version(), + + # Deliberately not supporting python_version marker in order to avoid + # possible issues with the python version not yet being known when this + # script is run. + # "python_version": '.'.join(platform.python_version_tuple()[:2]), + + "python_full_version": platform.python_version(), + "implementation_name": sys.implementation.name, + "implementation_version": _get_implementation_version(), +} + + def build_environment_from_requirements(cli_args): """Build a conda environment.yml from requirements.txt files. @@ -62,7 +103,16 @@ def build_environment_from_requirements(cli_args): # requirement if we're using pip. conda_requirements.add('pip') - pip_requirements.add(line) + # Handle environment specifiers and see if the requirement + # should exist in this environment. + # InVEST pretty much just uses this for checking operating + # systems. + if ";" in line: + package, marker = line.split(';') + if eval(marker, ENV_MARKERS): + pip_requirements.add(package) + else: + pip_requirements.add(line) # If an scm needs to be installed for pip to clone to a # revision, add it to the conda package list. From 9d4646735f2d346ed21f60ecc669dc64321d467d Mon Sep 17 00:00:00 2001 From: James Douglass Date: Thu, 26 Sep 2024 13:49:41 -0700 Subject: [PATCH 3/5] Correcting an environment marker that is not a function. RE:#1635 --- scripts/convert-requirements-to-conda-yml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/convert-requirements-to-conda-yml.py b/scripts/convert-requirements-to-conda-yml.py index 594a821a6a..f3c6e51da2 100644 --- a/scripts/convert-requirements-to-conda-yml.py +++ b/scripts/convert-requirements-to-conda-yml.py @@ -36,7 +36,7 @@ def format_full_version(info): # Environment marker handling is taken straight from # https://peps.python.org/pep-0508/#environment-markers ENV_MARKERS = { - "os_name": os.name(), + "os_name": os.name, "sys_platform": sys.platform, "platform_machine": platform.machine(), "platform_python_impl": platform.python_implementation(), From 4634f745535eb16574c2315e76a101ba71815573 Mon Sep 17 00:00:00 2001 From: James Douglass Date: Tue, 8 Oct 2024 13:23:31 -0700 Subject: [PATCH 4/5] Reverting requirements conversion back to main. The build issues I was experiencing had nothing to do with the contents of this file. RE:#1635 --- scripts/convert-requirements-to-conda-yml.py | 52 +------------------- 1 file changed, 1 insertion(+), 51 deletions(-) diff --git a/scripts/convert-requirements-to-conda-yml.py b/scripts/convert-requirements-to-conda-yml.py index f3c6e51da2..e5e2bc2d57 100644 --- a/scripts/convert-requirements-to-conda-yml.py +++ b/scripts/convert-requirements-to-conda-yml.py @@ -2,7 +2,6 @@ """convert-requirements-to-conda-yml.py""" import argparse -import os import platform import sys @@ -15,46 +14,6 @@ """ -# Environment marker handling is taken straight from -# https://peps.python.org/pep-0508/#environment-markers -def _get_implementation_version(): - def format_full_version(info): - version = '{0.major}.{0.minor}.{0.micro}'.format(info) - kind = info.releaselevel - if kind != 'final': - version += kind[0] + str(info.serial) - return version - - if hasattr(sys, 'implementation'): - implementation_version = format_full_version( - sys.implementation.version) - else: - implementation_version = "0" - return implementation_version - - -# Environment marker handling is taken straight from -# https://peps.python.org/pep-0508/#environment-markers -ENV_MARKERS = { - "os_name": os.name, - "sys_platform": sys.platform, - "platform_machine": platform.machine(), - "platform_python_impl": platform.python_implementation(), - "platform_release": platform.release(), - "platform_system": platform.system(), - "platform_version": platform.version(), - - # Deliberately not supporting python_version marker in order to avoid - # possible issues with the python version not yet being known when this - # script is run. - # "python_version": '.'.join(platform.python_version_tuple()[:2]), - - "python_full_version": platform.python_version(), - "implementation_name": sys.implementation.name, - "implementation_version": _get_implementation_version(), -} - - def build_environment_from_requirements(cli_args): """Build a conda environment.yml from requirements.txt files. @@ -103,16 +62,7 @@ def build_environment_from_requirements(cli_args): # requirement if we're using pip. conda_requirements.add('pip') - # Handle environment specifiers and see if the requirement - # should exist in this environment. - # InVEST pretty much just uses this for checking operating - # systems. - if ";" in line: - package, marker = line.split(';') - if eval(marker, ENV_MARKERS): - pip_requirements.add(package) - else: - pip_requirements.add(line) + pip_requirements.add(line) # If an scm needs to be installed for pip to clone to a # revision, add it to the conda package list. From 8165704ba2d3a178298694bcaad457c3097920bc Mon Sep 17 00:00:00 2001 From: James Douglass Date: Wed, 9 Oct 2024 14:31:58 -0700 Subject: [PATCH 5/5] Restoring a comment in UNA. RE:#1635 --- src/natcap/invest/urban_nature_access.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/natcap/invest/urban_nature_access.py b/src/natcap/invest/urban_nature_access.py index 4c8261cc5e..136575a956 100644 --- a/src/natcap/invest/urban_nature_access.py +++ b/src/natcap/invest/urban_nature_access.py @@ -2548,6 +2548,7 @@ def _warp_lulc(source_lulc_path, target_lulc_path, target_pixel_size, 'near', target_bb=target_bounding_box, target_projection_wkt=source_raster_info['projection_wkt']) + # if there is no defined nodata, set a default value if target_nodata is None: # Guarantee that our nodata cannot be represented by the datatype - # select a nodata value that's out of range.