From 384d8f0e7058af9b3f0a1bca95349be52904bdf2 Mon Sep 17 00:00:00 2001 From: Maarten Vermeyen Date: Fri, 20 Sep 2024 12:07:18 +0200 Subject: [PATCH 01/26] Add hatch test config - add hatch test config - add config for hatch dev environment - separate test dependencies from other dev dependencies --- pyproject.toml | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 56a919b..5819687 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,14 @@ Repository = "https://github.com/OnroerendErfgoed/brdr" Issues = "https://github.com/OnroerendErfgoed/brdr/issues" [project.optional-dependencies] +test = [ + "pytest-cov==5.0.0", + "pytest==8.1.1", + "responses==0.25.3", + "toml==0.10.2", +] dev = [ + "brdr[test]", "black==24.4.0", "flake8==7.0.0", "geopandas==0.14.3", @@ -43,10 +50,6 @@ dev = [ "matplotlib==3.8.4", "mypy==1.9.0", "pip-tools==7.4.1", - "pytest-cov==5.0.0", - "pytest==8.1.1", - "responses==0.25.3", - "toml==0.10.2", "types-requests==2.31.0.20240406", ] @@ -59,3 +62,18 @@ packages = ["brdr"] [tool.black] target-version = ['py39', 'py310', 'py311', 'py312'] + +# Config for hatch environments +[tool.hatch.envs.dev] +features = [ + 'dev', +] + +# Config for hatch test +[tool.hatch.envs.hatch-test] +features = [ + 'test', +] + +[[tool.hatch.envs.hatch-test.matrix]] +python = ["3.9", "3.10", '3.11', '3.12'] From e20caaea5436cf13e95cccb6a2df9880d787db2b Mon Sep 17 00:00:00 2001 From: dieuska Date: Fri, 20 Sep 2024 18:13:08 +0200 Subject: [PATCH 02/26] #96 bugfix - save version_date as string in properties --- brdr/aligner.py | 3 ++- brdr/loader.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/brdr/aligner.py b/brdr/aligner.py index 5ff3cfe..603f32c 100644 --- a/brdr/aligner.py +++ b/brdr/aligner.py @@ -662,7 +662,8 @@ def get_brdr_formula(self, geometry: BaseGeometry, with_geom=False): key_ref in self.dict_reference_properties and VERSION_DATE in self.dict_reference_properties[key_ref] ): - version_date = self.dict_reference_properties[key_ref][VERSION_DATE] + str_version_date = self.dict_reference_properties[key_ref][VERSION_DATE] + version_date = datetime.strptime(str_version_date,DATE_FORMAT) if last_version_date is None and version_date is not None: last_version_date = version_date if version_date is not None and version_date > last_version_date: diff --git a/brdr/loader.py b/brdr/loader.py index 5f67a88..ad1ba0f 100644 --- a/brdr/loader.py +++ b/brdr/loader.py @@ -24,18 +24,19 @@ def load_data(self): if self.versiondate_info is not None: for key in self.data_dict_properties.keys(): try: - self.data_dict_properties[key][VERSION_DATE] = datetime.strptime( + date = datetime.strptime( self.data_dict_properties[key][self.versiondate_info["name"]], self.versiondate_info["format"], ) except: # Catch, to try extracting only the date with default -date format if specific format does not work - self.data_dict_properties[key][VERSION_DATE] = datetime.strptime( + date = datetime.strptime( self.data_dict_properties[key][self.versiondate_info["name"]][ :10 ], DATE_FORMAT, ) + self.data_dict_properties[key][VERSION_DATE] = datetime.strftime(date,DATE_FORMAT) return self.data_dict, self.data_dict_properties, self.data_dict_source From f2220e441ad2de8b11978571003d0cf429bfdae6 Mon Sep 17 00:00:00 2001 From: dieuska Date: Fri, 20 Sep 2024 18:13:40 +0200 Subject: [PATCH 03/26] added type to example --- examples/example_131635.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/example_131635.py b/examples/example_131635.py index 7051514..4afe314 100644 --- a/examples/example_131635.py +++ b/examples/example_131635.py @@ -1,5 +1,5 @@ from brdr.aligner import Aligner -from brdr.enums import GRBType +from brdr.enums import GRBType, AlignerInputType from brdr.grb import GRBActualLoader from brdr.oe import OnroerendErfgoedLoader from examples import print_brdr_formula @@ -16,7 +16,7 @@ aligner.load_thematic_data(loader) loader = GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=aligner) aligner.load_reference_data(loader) - ref_geojson = aligner.get_input_as_geojson() + ref_geojson = aligner.get_input_as_geojson(inputtype=AlignerInputType.REFERENCE) # RESULTS rel_dist = 2 From 6f43ddc414c6d4ce3c8479dac6aa9caaf123cd8b Mon Sep 17 00:00:00 2001 From: dieuska Date: Mon, 23 Sep 2024 16:43:43 +0200 Subject: [PATCH 04/26] async Threadpoolexecutor --- brdr/aligner.py | 78 +++++++++++------------------------ examples/example_speedtest.py | 2 +- 2 files changed, 26 insertions(+), 54 deletions(-) diff --git a/brdr/aligner.py b/brdr/aligner.py index 603f32c..6f8aa1e 100644 --- a/brdr/aligner.py +++ b/brdr/aligner.py @@ -2,6 +2,7 @@ import logging import os from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor from datetime import datetime from math import pi from typing import Iterable @@ -179,6 +180,7 @@ def load_reference_data(self, loader: Loader): ##########PROCESSORS####################### ########################################### + def process_geometry( self, input_geometry: BaseGeometry, @@ -282,6 +284,7 @@ def process_geometry( return result_dict + def process( self, relevant_distances: Iterable[float] = None, @@ -320,29 +323,33 @@ def process( self.threshold_overlap_percentage = threshold_overlap_percentage self.logger.feedback_debug("Process series" + str(self.relevant_distances)) dict_series = {} + dict_series_queue = {} dict_thematic = self.dict_thematic if self.multi_as_single_modus: dict_thematic = multipolygons_to_singles(dict_thematic) - - for key, geometry in dict_thematic.items(): - self.logger.feedback_info( - f"thematic id {str(key)} processed with relevant distances (m) [{str(self.relevant_distances)}]" - ) - dict_series[key] = {} - for relevant_distance in self.relevant_distances: - try: + with ThreadPoolExecutor(max_workers=5) as executor:#max_workers=5 + for key, geometry in dict_thematic.items(): + self.logger.feedback_info( + f"thematic id {str(key)} processed with relevant distances (m) [{str(self.relevant_distances)}]" + ) + dict_series[key] = {} + dict_series_queue[key] = {} + for relevant_distance in self.relevant_distances: self.relevant_distance = relevant_distance - processed_result = self.process_geometry( - geometry, - self.relevant_distance, - od_strategy, - threshold_overlap_percentage, - ) - except ValueError as e: - self.logger.feedback_warning(str(e)) - - dict_series[key][self.relevant_distance] = processed_result + try: + dict_series_queue[key][self.relevant_distance] = executor.submit(self.process_geometry, geometry, + self.relevant_distance, + od_strategy, + threshold_overlap_percentage,) + except ValueError as e: + print("error for" + f"thematic id {str(key)} processed with relevant distances (m) [{str(self.relevant_distances)}]") + dict_series_queue[key][self.relevant_distance] = None + self.logger.feedback_warning(str(e)) + + for key in dict_series_queue.keys(): + for rd in dict_series_queue[key]: + dict_series[key][rd] = dict_series_queue[key][rd].result() if self.multi_as_single_modus: dict_series = merge_process_results(dict_series) @@ -354,41 +361,6 @@ def process( return self.dict_processresults - # def process( - # self, - # relevant_distance=1, - # od_strategy=OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE, - # threshold_overlap_percentage=50, - # ) -> dict[str, dict[float, ProcessResult]]: - # """ - # Aligns a thematic dictionary of geometries to the reference layer based on - # specified parameters. - method to align a thematic dictionary to the reference - # layer - # - # Args: - # relevant_distance (float, optional): The relevant distance (in meters) for - # processing. Defaults to 1. - # od_strategy (int, optional): The strategy for overlap detection. - # Defaults to 1. - # threshold_overlap_percentage (float, optional): The threshold percentage for - # considering full overlap. Defaults to 50. - # - # Returns: - # dict: A dict containing processed data for each thematic key: - # - result: Aligned thematic data. - # - result_diff: global differences between thematic data and reference - # data. - # - result_diff_plus: Positive differences. - # - result_diff_min: Negative differences. - # - relevant_intersection: relevant intersections. - # - relevant_diff: relevant differences. - # - # """ - # self.relevant_distance=relevant_distance - # self.dict_result = self.process(relevant_distances=[self.relevant_distance], - # od_strategy=od_strategy, - # threshold_overlap_percentage=threshold_overlap_percentage) - # return self.dict_result def predictor( self, diff --git a/examples/example_speedtest.py b/examples/example_speedtest.py index ff460f1..de63772 100644 --- a/examples/example_speedtest.py +++ b/examples/example_speedtest.py @@ -19,7 +19,7 @@ aligner.load_reference_data(loader) times = [] -for iter in range(1, 3): +for iter in range(1, 10): starttime = datetime.now() # Example how to use the Aligner From 7d710c6978cf490147ad9a0f9defd9662e5308e2 Mon Sep 17 00:00:00 2001 From: dieuska Date: Tue, 24 Sep 2024 10:15:42 +0200 Subject: [PATCH 05/26] #97 added ThreadPoolExecutor --- brdr/aligner.py | 80 ++++++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/brdr/aligner.py b/brdr/aligner.py index 6f8aa1e..d501f15 100644 --- a/brdr/aligner.py +++ b/brdr/aligner.py @@ -3,6 +3,7 @@ import os from collections import defaultdict from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import wait from datetime import datetime from math import pi from typing import Iterable @@ -105,7 +106,8 @@ def __init__( """ self.logger = Logger(feedback) - self.relevant_distance = relevant_distance + if relevant_distances is None and relevant_distance is not None: + relevant_distances = [relevant_distance] self.relevant_distances = relevant_distances self.od_strategy = od_strategy self.threshold_overlap_percentage = threshold_overlap_percentage @@ -158,9 +160,6 @@ def __init__( self.multi_as_single_modus = True self.logger.feedback_info("Aligner initialized") - def buffer_distance(self): - return self.relevant_distance / 2 - ##########LOADERS########################## ########################################### @@ -218,10 +217,9 @@ def process_geometry( raise ValueError(message) self.logger.feedback_debug("process geometry") - self.relevant_distance = relevant_distance self.od_strategy = od_strategy self.threshold_overlap_percentage = threshold_overlap_percentage - + buffer_distance = relevant_distance/2 # combine all parts of the input geometry to one polygon input_geometry = unary_union(get_parts(input_geometry)) @@ -231,7 +229,7 @@ def process_geometry( preresult, relevant_intersection_array, relevant_diff_array, - ) = self._calculate_intersection_between_geometry_and_od(input_geometry) + ) = self._calculate_intersection_between_geometry_and_od(input_geometry,relevant_distance) # get a list of all ref_ids that are intersecting the thematic geometry ref_intersections = self.reference_items.take( self.reference_tree.query(geometry) @@ -250,7 +248,7 @@ def process_geometry( geom_intersection, geom_reference, False, - self.relevant_distance / 2, + buffer_distance, self.threshold_overlap_percentage, ) self.logger.feedback_debug("intersection calculated") @@ -270,7 +268,7 @@ def process_geometry( relevant_diff = Polygon() # POSTPROCESSING - result_dict = self._postprocess_preresult(preresult, geometry) + result_dict = self._postprocess_preresult(preresult, geometry,relevant_distance) result_dict["result_relevant_intersection"] = relevant_intersection result_dict["result_relevant_diff"] = relevant_diff @@ -317,13 +315,13 @@ def process( """ if relevant_distances is None: relevant_distances = [relevant_distance] - self.relevant_distance = relevant_distance self.relevant_distances = relevant_distances self.od_strategy = od_strategy self.threshold_overlap_percentage = threshold_overlap_percentage self.logger.feedback_debug("Process series" + str(self.relevant_distances)) dict_series = {} dict_series_queue = {} + futures = [] dict_thematic = self.dict_thematic if self.multi_as_single_modus: @@ -336,20 +334,22 @@ def process( dict_series[key] = {} dict_series_queue[key] = {} for relevant_distance in self.relevant_distances: - self.relevant_distance = relevant_distance try: - dict_series_queue[key][self.relevant_distance] = executor.submit(self.process_geometry, geometry, - self.relevant_distance, + future = executor.submit(self.process_geometry, geometry, + relevant_distance, od_strategy, threshold_overlap_percentage,) + futures.append(future) + dict_series_queue[key][relevant_distance] = future except ValueError as e: print("error for" + f"thematic id {str(key)} processed with relevant distances (m) [{str(self.relevant_distances)}]") - dict_series_queue[key][self.relevant_distance] = None + dict_series_queue[key][relevant_distance] = None self.logger.feedback_warning(str(e)) - - for key in dict_series_queue.keys(): - for rd in dict_series_queue[key]: - dict_series[key][rd] = dict_series_queue[key][rd].result() + self.logger.feedback_debug ("waiting all started RD calculations") + wait(futures) + for id_theme,dict_dist in dict_series_queue.items(): + for reldist,future in dict_dist.items(): + dict_series[id_theme][reldist] = future.result() if self.multi_as_single_modus: dict_series = merge_process_results(dict_series) @@ -831,8 +831,9 @@ def _prepare_reference_data(self): self.reference_union = None return - def _calculate_intersection_between_geometry_and_od(self, geometry): + def _calculate_intersection_between_geometry_and_od(self, geometry,relevant_distance): # Calculate the intersection between thematic and Openbaar Domein + buffer_distance = relevant_distance/2 relevant_intersection_array = [] relevant_difference_array = [] geom_thematic_od = Polygon() @@ -859,11 +860,11 @@ def _calculate_intersection_between_geometry_and_od(self, geometry): # geom of OD geom_od = safe_difference(geometry, self._get_reference_union()) # only the relevant parts of OD - geom_od_neg_pos = buffer_neg_pos(geom_od, self.buffer_distance()) + geom_od_neg_pos = buffer_neg_pos(geom_od, buffer_distance) # geom_thematic_od = safe_intersection(geom_od_neg_pos,geom_od)# resulting # thematic OD geom_od_neg_pos_buffered = buffer_pos( - geom_od_neg_pos, self.buffer_distance() + geom_od_neg_pos, buffer_distance ) # include parts geom_thematic_od = safe_intersection( geom_od_neg_pos_buffered, geom_od @@ -877,7 +878,7 @@ def _calculate_intersection_between_geometry_and_od(self, geometry): geom_thematic_od, relevant_difference_array, relevant_intersection_array, - ) = self._od_snap_all_side(geometry) + ) = self._od_snap_all_side(geometry,relevant_distance) elif self.od_strategy == OpenbaarDomeinStrategy.SNAP_FULL_AREA_SINGLE_SIDE: # Strategy useful for bigger areas. # integrates the entire inner area of the input geometry, @@ -886,7 +887,7 @@ def _calculate_intersection_between_geometry_and_od(self, geometry): self.logger.feedback_debug( "OD-strategy Full-area-variant of OD-SNAP_SINGLE_SIDE" ) - geom_thematic_od = self._od_full_area(geometry) + geom_thematic_od = self._od_full_area(geometry,relevant_distance) elif self.od_strategy == OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE: # Strategy useful for bigger areas. # integrates the entire inner area of the input geometry, @@ -900,10 +901,10 @@ def _calculate_intersection_between_geometry_and_od(self, geometry): geom_thematic_od, relevant_difference_array, relevant_intersection_array, - ) = self._od_snap_all_side(geometry) + ) = self._od_snap_all_side(geometry,relevant_distance) # This part is a copy of SNAP_FULL_AREA_SINGLE_SIDE geom_theme_od_min_clipped_plus_buffered_clipped = self._od_full_area( - geometry + geometry,relevant_distance ) # UNION the calculation of OD-SNAP_ALL_SIDE with FULL AREA of # OD-SNAP_FULL_AREA_SINGLE_SIDE @@ -921,7 +922,7 @@ def _calculate_intersection_between_geometry_and_od(self, geometry): # geom of OD geom_od = safe_difference(geometry, self._get_reference_union()) # only the relevant parts of OD - geom_od_neg_pos = buffer_neg_pos(geom_od, self.buffer_distance()) + geom_od_neg_pos = buffer_neg_pos(geom_od, buffer_distance) # resulting thematic OD geom_thematic_od = safe_intersection(geom_od_neg_pos, geom_od) elif self.od_strategy == OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE_VARIANT_2: @@ -943,20 +944,21 @@ def _calculate_intersection_between_geometry_and_od(self, geometry): relevant_difference_array, ) - def _od_full_area(self, geometry): + def _od_full_area(self, geometry,relevant_distance): + buffer_distance=relevant_distance/2 geom_theme_od = safe_difference(geometry, self._get_reference_union()) geom_theme_min_buffered = buffer_neg( buffer_pos( - buffer_neg(geometry, self.relevant_distance), - self.buffer_distance(), + buffer_neg(geometry, relevant_distance), + buffer_distance, ), - self.buffer_distance(), + buffer_distance, ) geom_theme_od_clipped_min_buffered = safe_intersection( geom_theme_min_buffered, geom_theme_od ) geom_theme_od_min_clipped_plus_buffered = buffer_pos( - geom_theme_od_clipped_min_buffered, self.relevant_distance + geom_theme_od_clipped_min_buffered, relevant_distance ) geom_theme_od_min_clipped_plus_buffered_clipped = safe_intersection( geom_theme_od_min_clipped_plus_buffered, geom_theme_od @@ -964,11 +966,12 @@ def _od_full_area(self, geometry): geom_thematic_od = geom_theme_od_min_clipped_plus_buffered_clipped return geom_thematic_od - def _od_snap_all_side(self, geometry): + def _od_snap_all_side(self, geometry,relevant_distance): + buffer_distance = relevant_distance/2 relevant_difference_array = [] relevant_intersection_array = [] geom_thematic_buffered = make_valid( - buffer_pos(geometry, BUFFER_MULTIPLICATION_FACTOR * self.relevant_distance) + buffer_pos(geometry, BUFFER_MULTIPLICATION_FACTOR * relevant_distance) ) clip_ref_thematic_buffered = safe_intersection( self._get_reference_union(), geom_thematic_buffered @@ -989,7 +992,7 @@ def _od_snap_all_side(self, geometry): geom_intersection, geom_reference, True, - self.relevant_distance / 2, + buffer_distance, self.threshold_overlap_percentage, ) relevant_intersection_array = self._add_multi_polygons_from_geom_to_array( @@ -1007,7 +1010,7 @@ def _get_reference_union(self): ) return self.reference_union - def _postprocess_preresult(self, preresult, geom_thematic) -> ProcessResult: + def _postprocess_preresult(self, preresult, geom_thematic,relevant_distance) -> ProcessResult: """ Postprocess the preresult with the following actions to create the final result *Corrections for areas that differ more than the relevant distance @@ -1032,6 +1035,7 @@ def _postprocess_preresult(self, preresult, geom_thematic) -> ProcessResult: geometry """ # Process array + buffer_distance = relevant_distance/2 result = [] geom_preresult = make_valid(unary_union(preresult)) geom_thematic = make_valid(geom_thematic) @@ -1069,14 +1073,14 @@ def _postprocess_preresult(self, preresult, geom_thematic) -> ProcessResult: geom_thematic_dissolved, safe_intersection( geom_diff_delete, - buffer_neg_pos(geom_diff_delete, self.buffer_distance()), + buffer_neg_pos(geom_diff_delete, buffer_distance), ), ) geom_diff_removed_added = safe_union( geom_diff_removed, safe_intersection( geom_diff_add, - buffer_neg_pos(geom_diff_add, self.buffer_distance()), + buffer_neg_pos(geom_diff_add, buffer_distance), ), ) geom_thematic_preresult = buffer_pos( @@ -1089,7 +1093,7 @@ def _postprocess_preresult(self, preresult, geom_thematic) -> ProcessResult: # Correction for Inner holes(donuts) / multipolygons # fill and remove gaps geom_thematic_cleaned_holes = fill_and_remove_gaps( - geom_thematic_preresult, self.buffer_distance() + geom_thematic_preresult, buffer_distance ) geom_thematic_result = buffer_pos( buffer_neg( From 688ab3026c953d2275abe96b4083d4fd46231175 Mon Sep 17 00:00:00 2001 From: dieuska Date: Tue, 24 Sep 2024 11:07:42 +0200 Subject: [PATCH 06/26] updated example parcelchangedetector --- examples/example_parcel_change_detector.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/examples/example_parcel_change_detector.py b/examples/example_parcel_change_detector.py index b419396..462659f 100644 --- a/examples/example_parcel_change_detector.py +++ b/examples/example_parcel_change_detector.py @@ -1,4 +1,5 @@ import logging +from datetime import datetime from brdr.aligner import Aligner from brdr.constants import EVALUATION_FIELD_NAME, RELEVANT_DISTANCE_FIELD_NAME @@ -39,18 +40,19 @@ # Initiate an Aligner to create a themeset that is base-referenced on a specific # base_year base_aligner = Aligner() +print("start loading OE-objects") # Load the thematic data to evaluate loader = OnroerendErfgoedLoader(bbox=bbox, partition=0) base_aligner.load_thematic_data(loader) -logging.info( +print( "Number of OE-thematic features loaded into base-aligner: " + str(len(base_aligner.dict_thematic)) ) base_aligner.load_reference_data( - GRBFiscalParcelLoader(year=base_year, aligner=base_aligner) + GRBFiscalParcelLoader(year=base_year, aligner=base_aligner,partition=1000) ) - +print("Reference-data loaded") # Exclude objects bigger than specified area keys_to_exclude = [] nr_features = len(base_aligner.dict_thematic) @@ -58,13 +60,15 @@ if base_aligner.dict_thematic[key].area > excluded_area: keys_to_exclude.append(key) counter_excluded = counter_excluded + 1 - logging.info( + print( "geometrie excluded; bigger than " + str(excluded_area) + ": " + key ) for x in keys_to_exclude: del base_aligner.dict_thematic[x] # # Align the features to the base-GRB +print("Process base objects") +starttime = datetime.now() base_process_result = base_aligner.process(relevant_distance=base_correction) # get resulting aligned features on Adpfxxxx, with formula processresults = base_aligner.get_results_as_geojson(formula=True) @@ -74,6 +78,7 @@ featurecollection_base_result = processresults["result"] # Update Featurecollection to actual version +print("Actualise base objects") fcs = update_to_actual_grb( featurecollection_base_result, base_aligner.name_thematic_id, @@ -107,3 +112,6 @@ + "//Excluded: " + str(counter_excluded) ) +endtime = datetime.now() +seconds = (endtime - starttime).total_seconds() +print("duration: " + str(seconds)) From 41322129b0c6b7f393d74ae0c25ed33ba19204fa Mon Sep 17 00:00:00 2001 From: dieuska Date: Wed, 25 Sep 2024 13:14:09 +0200 Subject: [PATCH 07/26] cleaned example --- examples/example_evaluate.py | 8 +++--- examples/example_evaluate_ao.py | 6 +++-- examples/example_multi_to_single.py | 38 +++++++++++++---------------- 3 files changed, 24 insertions(+), 28 deletions(-) diff --git a/examples/example_evaluate.py b/examples/example_evaluate.py index 5b399c1..dedf090 100644 --- a/examples/example_evaluate.py +++ b/examples/example_evaluate.py @@ -76,8 +76,6 @@ print(fcs["result"]) for feature in fc["result"]["features"]: - print( - feature["properties"][actual_aligner.name_thematic_id] - + ": " - + feature["properties"][EVALUATION_FIELD_NAME] - ) + id = feature["properties"][actual_aligner.name_thematic_id] + evaluation = feature["properties"][EVALUATION_FIELD_NAME] + print(id + ": " + evaluation) diff --git a/examples/example_evaluate_ao.py b/examples/example_evaluate_ao.py index 6025736..70e7c22 100644 --- a/examples/example_evaluate_ao.py +++ b/examples/example_evaluate_ao.py @@ -15,7 +15,7 @@ from brdr.utils import get_series_geojson_dict base_aligner = Aligner() -loader = OnroerendErfgoedLoader([120288]) +loader = OnroerendErfgoedLoader([120288,120108]) base_aligner.load_thematic_data(loader) base_year = "2022" base_aligner.load_reference_data( @@ -64,4 +64,6 @@ series_prop_dict=prop_dictionary, ) for feature in fc["result"]["features"]: - print(feature["properties"][EVALUATION_FIELD_NAME]) + id = feature["properties"][actual_aligner.name_thematic_id] + evaluation = feature["properties"][EVALUATION_FIELD_NAME] + print(id + ": " + evaluation) diff --git a/examples/example_multi_to_single.py b/examples/example_multi_to_single.py index 456f2b8..e50ec95 100644 --- a/examples/example_multi_to_single.py +++ b/examples/example_multi_to_single.py @@ -1,47 +1,43 @@ from brdr.aligner import Aligner from brdr.enums import GRBType from brdr.grb import GRBActualLoader -from brdr.oe import OnroerendErfgoedLoader +from brdr.oe import OnroerendErfgoedLoader, OEType from examples import print_brdr_formula from examples import show_map -# EXAMPLE of "multi_as_single_modus" +#This example shows the usage of the setting 'multi_as_single_modus' +#True: (default): All polygons inside a MultiPolygon will be processed seperataly by the algorithm, and merged after processing. +#False: Multipolygon will be processed directly by the algorithm -# Initiate brdr +#Example (ErfgoedObject): https://inventaris.onroerenderfgoed.be/erfgoedobjecten/305858 +loader = OnroerendErfgoedLoader([305858],oetype=OEType.EO) +relevant_distance = 5 #rd is taken very high to show the difference +od_strategy =4 + +# EXAMPLE of "multi_as_single_modus"=FALSE +print ("EXAMPLE with 'multi_as_single_modus'=False") aligner = Aligner() -# WITHOUT MULTI_TO_SINGLE aligner.multi_as_single_modus = False -# Load thematic data & reference data -# Get a specific feature of OE that exists out of a Multipolygon -loader = OnroerendErfgoedLoader([110082]) aligner.load_thematic_data(loader) aligner.load_reference_data( GRBActualLoader(aligner=aligner, grb_type=GRBType.GBG, partition=1000) ) - -rel_dist = 20 -dict_results = aligner.process(relevant_distance=rel_dist, od_strategy=4) +dict_results = aligner.process(relevant_distance=relevant_distance, od_strategy=od_strategy) aligner.save_results("output/") -show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) - print_brdr_formula(dict_results, aligner) +show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) -# WITH MULTI_TO_SINGLE -# Initiate brdr +# WITH "multi_as_single_modus"=True +print ("EXAMPLE with 'multi_as_single_modus'=True") aligner = Aligner() aligner.multi_as_single_modus = True -# Load thematic data & reference data -# Get a specific feature of OE that exists out of a Multipolygon -loader = OnroerendErfgoedLoader([110082]) aligner.load_thematic_data(loader) aligner.load_reference_data( GRBActualLoader(aligner=aligner, grb_type=GRBType.GBG, partition=1000) ) - -rel_dist = 20 -dict_results = aligner.process(relevant_distance=rel_dist, od_strategy=4) +dict_results = aligner.process(relevant_distance=relevant_distance, od_strategy=od_strategy) aligner.save_results("output/") +print_brdr_formula(dict_results, aligner) show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) -print_brdr_formula(dict_results, aligner) From 5ec9e17921c9a439e55fbcac2c53625fc7dfccbf Mon Sep 17 00:00:00 2001 From: dieuska Date: Wed, 25 Sep 2024 13:20:56 +0200 Subject: [PATCH 08/26] #97 max_worker as parameter --- brdr/aligner.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/brdr/aligner.py b/brdr/aligner.py index d501f15..c21bacc 100644 --- a/brdr/aligner.py +++ b/brdr/aligner.py @@ -80,6 +80,7 @@ def __init__( od_strategy=OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE, crs=DEFAULT_CRS, area_limit=None, + max_workers=None, ): """ Initializes the Aligner object @@ -102,6 +103,7 @@ def __init__( crs (str, optional): Coordinate Reference System (CRS) of the data. (default EPSG:31370) area_limit (int, optional): Maximum area for processing. (default 100000) + max_workers (int, optional): Amount of workers that is used in ThreadPoolExecutor when processing objects for multiple relevant distances. (default None) """ @@ -112,6 +114,7 @@ def __init__( self.od_strategy = od_strategy self.threshold_overlap_percentage = threshold_overlap_percentage self.area_limit = area_limit + self.max_workers = max_workers # PROCESSING DEFAULTS # thematic @@ -326,7 +329,7 @@ def process( if self.multi_as_single_modus: dict_thematic = multipolygons_to_singles(dict_thematic) - with ThreadPoolExecutor(max_workers=5) as executor:#max_workers=5 + with ThreadPoolExecutor(max_workers=self.max_workers) as executor:#max_workers=5 for key, geometry in dict_thematic.items(): self.logger.feedback_info( f"thematic id {str(key)} processed with relevant distances (m) [{str(self.relevant_distances)}]" @@ -342,7 +345,7 @@ def process( futures.append(future) dict_series_queue[key][relevant_distance] = future except ValueError as e: - print("error for" + f"thematic id {str(key)} processed with relevant distances (m) [{str(self.relevant_distances)}]") + self.logger.feedback_warning("error for" + f"thematic id {str(key)} processed with relevant distances (m) [{str(self.relevant_distances)}]") dict_series_queue[key][relevant_distance] = None self.logger.feedback_warning(str(e)) self.logger.feedback_debug ("waiting all started RD calculations") From fef5c7210f2fe62b8eb49396890fd989ee64f750 Mon Sep 17 00:00:00 2001 From: dieuska Date: Wed, 25 Sep 2024 17:31:36 +0200 Subject: [PATCH 09/26] cleanups compare-function --- brdr/aligner.py | 7 +- brdr/grb.py | 2 +- examples/example_evaluate.py | 1 - examples/example_grbspecificloader.py | 19 +++++- examples/example_parcel_change_detector.py | 8 +++ .../example_predictor_double_prediction.py | 42 ------------ examples/example_refactor_dict_series.py | 26 ------- examples/example_speedtest.py | 68 +++++++++++-------- 8 files changed, 70 insertions(+), 103 deletions(-) delete mode 100644 examples/example_predictor_double_prediction.py delete mode 100644 examples/example_refactor_dict_series.py diff --git a/brdr/aligner.py b/brdr/aligner.py index c21bacc..c36ea7d 100644 --- a/brdr/aligner.py +++ b/brdr/aligner.py @@ -96,7 +96,8 @@ def __init__( threshold_overlap_percentage (int, optional): Threshold (%) to determine from which overlapping-percentage a reference-polygon has to be included when there aren't relevant intersections or relevant differences - (default 50%) + (default 50%). + When setting this parameter to '-1' the original border for will be returned for cases where nor relevant intersections and relevant differences are found od_strategy (int, optional): Determines how the algorithm deals with parts of the geometry that are not on the reference (default 1: SNAP_SINGLE_SIDE) @@ -512,6 +513,7 @@ def compare( for dist in dict_series[theme_id].keys(): prop_dictionary[theme_id][dist] = {} for theme_id in dict_unchanged.keys(): + dict_evaluated_result[theme_id]={} prop_dictionary[theme_id] = {} for theme_id, dict_results in dict_predictions.items(): @@ -575,9 +577,10 @@ def compare( ] = Evaluation.TO_CHECK_4 for theme_id, geom in dict_unchanged.items(): + dict_evaluated_result[theme_id] = {0: {"result": geom}} prop_dictionary[theme_id] = { 0: { - "result": geom, + #"result": geom, EVALUATION_FIELD_NAME: Evaluation.NO_CHANGE_6, FORMULA_FIELD_NAME: json.dumps(self.get_brdr_formula(geom)), } diff --git a/brdr/grb.py b/brdr/grb.py index 1e846d1..5c1e484 100644 --- a/brdr/grb.py +++ b/brdr/grb.py @@ -497,7 +497,7 @@ def load_data(self): class GRBSpecificDateParcelLoader(GeoJsonLoader): def __init__(self, date, aligner, partition=1000): - logging.warning("experimental loader; use with care!!!") + logging.warning("Loader for GRB parcel-situation on specific date (experimental); Use it with care!!!") try: date = datetime.strptime(date, DATE_FORMAT).date() if date.year >= datetime.now().year: diff --git a/examples/example_evaluate.py b/examples/example_evaluate.py index dedf090..62649bb 100644 --- a/examples/example_evaluate.py +++ b/examples/example_evaluate.py @@ -59,7 +59,6 @@ ) actual_aligner.relevant_distances = np.arange(0, 200, 10, dtype=int) / 100 dict_evaluated, prop_dictionary = actual_aligner.compare( - # thematic_dict_formula=thematic_dict_formula, threshold_area=5, threshold_percentage=1, dict_unchanged=dict_unchanged, diff --git a/examples/example_grbspecificloader.py b/examples/example_grbspecificloader.py index 93d2c20..88c7564 100644 --- a/examples/example_grbspecificloader.py +++ b/examples/example_grbspecificloader.py @@ -1,8 +1,12 @@ +import os + from shapely import from_wkt from brdr.aligner import Aligner +from brdr.enums import AlignerInputType from brdr.grb import GRBSpecificDateParcelLoader from brdr.loader import DictLoader +from brdr.utils import write_geojson aligner = Aligner() thematic_dict = { @@ -10,7 +14,20 @@ "Polygon ((172283.76869662097305991 174272.85233648214489222, 172276.89871930953813717 174278.68436246179044247, 172274.71383684969623573 174280.57171753142029047, 172274.63047763772192411 174280.64478165470063686, 172272.45265833073062822 174282.52660570573061705, 172269.33533191855531186 174285.22093996312469244, 172265.55258252174826339 174288.49089696351438761, 172258.77032718938426115 174294.22654021997004747, 172258.63259260458289646 174294.342757155187428, 172254.93673790179309435 174288.79932878911495209, 172248.71360730109154247 174279.61860501393675804, 172248.96566232520854101 174279.43056782521307468, 172255.25363882273086347 174274.73737183399498463, 172257.08298882702365518 174273.37133203260600567, 172259.32325354730710387 174271.69890458136796951, 172261.65807284769834951 174269.9690355472266674, 172266.35596220899606124 174266.4871726930141449, 172273.34350050613284111 174261.30863015633076429, 172289.60360219911672175 174249.35944479051977396, 172293.30328181147342548 174246.59864199347794056, 172297.34760522318538278 174253.10583685990422964, 172289.53060952731175348 174259.6846851697191596, 172292.86485871637705714 174265.19099397677928209, 172283.76869662097305991 174272.85233648214489222))" ) } +#EXAMPLE to use a GRBSpecificLoader, that retrieves the parcels for a specific data, based on the 2 fiscal parcel-situations of the year before and after +#Based on the date, the referencelayer will be different +date = "2023-05-03" +date = "2023-08-03" loader = DictLoader(thematic_dict) aligner.load_thematic_data(loader) -loader = GRBSpecificDateParcelLoader(date="2023-05-03", aligner=aligner) +loader = GRBSpecificDateParcelLoader(date=date, aligner=aligner) aligner.load_reference_data(loader) + +# aligner.process() +# aligner.save_results(path = "output/") + +fc = aligner.get_input_as_geojson( + inputtype=AlignerInputType.REFERENCE, +) +write_geojson(os.path.join("output/", "grb_adp_"+ date +".geojson"), fc) + diff --git a/examples/example_parcel_change_detector.py b/examples/example_parcel_change_detector.py index 462659f..7d3fe9c 100644 --- a/examples/example_parcel_change_detector.py +++ b/examples/example_parcel_change_detector.py @@ -89,14 +89,20 @@ counter_equality = 0 counter_equality_by_alignment = 0 counter_difference = 0 +counter_no_change = 0 +#TODO: counter_difference collects al the 'TO_CHECK's' but these are multiple proposals, so clean up the stats +#TODO: Move this as general output from the updater? for feature in fcs["result"]["features"]: if EVALUATION_FIELD_NAME in feature["properties"].keys(): ev = feature["properties"][EVALUATION_FIELD_NAME] + print(ev) rd = feature["properties"][RELEVANT_DISTANCE_FIELD_NAME] if ev.startswith("equal") and rd == 0: counter_equality = counter_equality + 1 elif ev.startswith("equal") and rd > 0: counter_equality_by_alignment = counter_equality_by_alignment + 1 + elif ev.startswith("no_change"): + counter_no_change= counter_no_change + 1 else: counter_difference = counter_difference + 1 @@ -107,6 +113,8 @@ + str(counter_equality) + "//Equality by alignment: " + str(counter_equality_by_alignment) + + "//No change: " + + str(counter_no_change) + "//Difference: " + str(counter_difference) + "//Excluded: " diff --git a/examples/example_predictor_double_prediction.py b/examples/example_predictor_double_prediction.py deleted file mode 100644 index 9b2ad5e..0000000 --- a/examples/example_predictor_double_prediction.py +++ /dev/null @@ -1,42 +0,0 @@ -import numpy as np -from shapely import from_wkt - -from brdr.aligner import Aligner -from brdr.enums import GRBType -from brdr.grb import GRBActualLoader -from brdr.loader import DictLoader - -# Press the green button in the gutter to run the script. -if __name__ == "__main__": - """ - example to use the predictor-function to automatically predict which resulting - geometries are interesting to look at (based on detection of breakpoints and - relevant distances of 'no-change') - """ - # Initiate an Aligner - aligner = Aligner() - # Load thematic data & reference data - loader = DictLoader( - { - "id1": from_wkt( - "MultiPolygon Z (((138430.4033999964594841 194082.86080000177025795 0, 138422.19659999758005142 194080.36510000005364418 0, 138419.01550000160932541 194079.34930000081658363 0, 138412.59849999845027924 194077.14139999821782112 0, 138403.65579999983310699 194074.06430000066757202 0, 138402.19910000264644623 194077.67480000108480453 0, 138401.83420000225305557 194078.57939999923110008 0, 138400.89329999685287476 194080.91140000149607658 0, 138400.31650000065565109 194080.67880000174045563 0, 138399.27300000190734863 194083.37680000066757202 0, 138405.93310000002384186 194085.95410000160336494 0, 138413.51049999892711639 194088.80620000138878822 0, 138427.25680000334978104 194094.29969999939203262 0, 138430.4033999964594841 194082.86080000177025795 0)))" - ) - } - ) - aligner.load_thematic_data(loader) - loader = GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=aligner) - aligner.load_reference_data(loader) - - series = np.arange(0, 800, 10, dtype=int) / 100 - # predict which relevant distances are interesting to propose as resulting geometry - dict_series, dict_predictions, diffs = aligner.predictor( - relevant_distances=series, od_strategy=4, threshold_overlap_percentage=50 - ) - fcs = aligner.get_results_as_geojson(formula=False) - print(fcs["result"]) - # for key in dict_predictions: - # show_map( - # {key:dict_predictions[key]}, - # {key: aligner.dict_thematic[key]}, - # aligner.dict_reference, - # ) diff --git a/examples/example_refactor_dict_series.py b/examples/example_refactor_dict_series.py deleted file mode 100644 index 511a6a8..0000000 --- a/examples/example_refactor_dict_series.py +++ /dev/null @@ -1,26 +0,0 @@ -from brdr.aligner import Aligner -from brdr.enums import GRBType -from brdr.grb import GRBActualLoader -from brdr.oe import OnroerendErfgoedLoader - - -# EXAMPLE to test the algorithm for erfgoedobject with relevant distance 0.2m and -# od_strategy SNAP_ALL_SIDE - -# Initiate brdr -aligner = Aligner() -# Load thematic data & reference data -aanduidingsobjecten = [117798, 116800, 117881] - -loader = OnroerendErfgoedLoader(aanduidingsobjecten) -aligner.load_thematic_data(loader) -loader = GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=aligner) -aligner.load_reference_data(loader) - -test = aligner.process() -test = aligner.process([1, 2, 3]) -test = aligner.predictor() -fcs = aligner.get_results_as_geojson(formula=True) -print(test) -print(fcs) -print(fcs["result"]) diff --git a/examples/example_speedtest.py b/examples/example_speedtest.py index de63772..a808329 100644 --- a/examples/example_speedtest.py +++ b/examples/example_speedtest.py @@ -4,38 +4,43 @@ from brdr.aligner import Aligner from brdr.loader import GeoJsonFileLoader -# Initiate brdr -aligner = Aligner(relevant_distance=2) -aligner.multi_as_single_modus = True -# Load local thematic data and reference data -# loader = GeoJsonFileLoader( -# "../tests/testdata/theme.geojson", "theme_identifier" -# ) -loader = GeoJsonFileLoader( - "../tests/testdata/themelayer_not_referenced.geojson", "theme_identifier" -) -aligner.load_thematic_data(loader) -loader = GeoJsonFileLoader("../tests/testdata/reference_leuven.geojson", "capakey") -aligner.load_reference_data(loader) +def main(): + # Initiate brdr + aligner = Aligner(relevant_distance=2,max_workers=None) + iterations = 10 + aligner.multi_as_single_modus = True + # Load local thematic data and reference data + # loader = GeoJsonFileLoader( + # "../tests/testdata/theme.geojson", "theme_identifier" + # ) + loader = GeoJsonFileLoader( + "../tests/testdata/themelayer_not_referenced.geojson", "theme_identifier" + ) + aligner.load_thematic_data(loader) + loader = GeoJsonFileLoader("../tests/testdata/reference_leuven.geojson", "capakey") + aligner.load_reference_data(loader) -times = [] -for iter in range(1, 10): - starttime = datetime.now() + times = [] + total_starttime = datetime.now() + for iter in range(1, iterations + 1): + starttime = datetime.now() - # Example how to use the Aligner - aligner.predictor() - fcs = aligner.get_results_as_geojson(formula=True) - endtime = datetime.now() - seconds = (endtime - starttime).total_seconds() - times.append(seconds) - print(seconds) -print("duration: " + str(times)) - -print("Min: " + str(min(times))) -print("Max: " + str(max(times))) -print("Mean: " + str(statistics.mean(times))) -print("Median: " + str(statistics.median(times))) -print("Stdv: " + str(statistics.stdev(times))) + # Example how to use the Aligner + aligner.predictor() + fcs = aligner.get_results_as_geojson(formula=True) + endtime = datetime.now() + seconds = (endtime - starttime).total_seconds() + times.append(seconds) + print(seconds) + total_endtime = datetime.now() + total_seconds = (total_endtime - total_starttime).total_seconds() + print("Total time: " + str(total_seconds)) + print("duration: " + str(times)) + print("Min: " + str(min(times))) + print("Max: " + str(max(times))) + print("Mean: " + str(statistics.mean(times))) + print("Median: " + str(statistics.median(times))) + print("Stdv: " + str(statistics.stdev(times))) # #BEFORE REFACTORING dict_series # duration: [25.652311, 27.894154, 19.641618, 19.929254, 44.754033, 25.218422, 23.167992, 18.649832, 22.899336, 52.108296] @@ -52,3 +57,6 @@ # Mean: 17.981391 # Median: 17.8996155 # Stdv: 1.504459449440969 + +if __name__ == '__main__': + main() From b11b254b7d816542f6aedb79793500d56b8bb774 Mon Sep 17 00:00:00 2001 From: dieuska Date: Fri, 27 Sep 2024 13:15:46 +0200 Subject: [PATCH 10/26] checnged structure for adding evaluation attributes --- brdr/aligner.py | 409 ++++++++++++++------- brdr/constants.py | 15 +- brdr/geometry_utils.py | 4 + brdr/grb.py | 62 ++-- examples/example_evaluate.py | 21 +- examples/example_evaluate_ao.py | 31 +- examples/example_multi_to_single.py | 27 +- examples/example_parcel_change_detector.py | 5 +- examples/example_speedtest.py | 1 + pyproject.toml | 4 +- tests/test_aligner.py | 45 ++- tests/test_examples.py | 2 +- tests/test_grb.py | 44 +-- tests/test_integration.py | 2 +- tests/test_utils.py | 1 - 15 files changed, 421 insertions(+), 252 deletions(-) diff --git a/brdr/aligner.py b/brdr/aligner.py index c36ea7d..00c96ce 100644 --- a/brdr/aligner.py +++ b/brdr/aligner.py @@ -16,7 +16,6 @@ from shapely import make_valid from shapely import remove_repeated_points from shapely import to_geojson -from shapely import unary_union from shapely.geometry.base import BaseGeometry from brdr import __version__ @@ -27,8 +26,8 @@ DATE_FORMAT, THRESHOLD_EXCLUSION_PERCENTAGE, THRESHOLD_EXCLUSION_AREA, - FORMULA_FIELD_NAME, - EVALUATION_FIELD_NAME, + FORMULA_FIELD_NAME, EVALUATION_FIELD_NAME, FULL_BASE_FIELD_NAME, FULL_ACTUAL_FIELD_NAME, DIFF_PERCENTAGE_FIELD_NAME, + DIFF_AREA_FIELD_NAME, OD_ALIKE_FIELD_NAME, EQUAL_REFERENCE_FEATURES_FIELD_NAME, ) from brdr.constants import CORR_DISTANCE from brdr.constants import DEFAULT_CRS @@ -39,7 +38,7 @@ AlignerResultType, AlignerInputType, ) -from brdr.geometry_utils import buffer_neg +from brdr.geometry_utils import buffer_neg, safe_unary_union from brdr.geometry_utils import buffer_neg_pos from brdr.geometry_utils import buffer_pos from brdr.geometry_utils import fill_and_remove_gaps @@ -225,7 +224,7 @@ def process_geometry( self.threshold_overlap_percentage = threshold_overlap_percentage buffer_distance = relevant_distance/2 # combine all parts of the input geometry to one polygon - input_geometry = unary_union(get_parts(input_geometry)) + input_geometry = safe_unary_union(get_parts(input_geometry)) # array with all relevant parts of a thematic geometry; initial empty Polygon ( @@ -264,10 +263,10 @@ def process_geometry( relevant_diff, relevant_diff_array ) # UNION INTERMEDIATE LAYERS - relevant_intersection = unary_union(relevant_intersection_array) + relevant_intersection = safe_unary_union(relevant_intersection_array) if relevant_intersection is None or relevant_intersection.is_empty: relevant_intersection = Polygon() - relevant_diff = unary_union(relevant_diff_array) + relevant_diff = safe_unary_union(relevant_diff_array) if relevant_diff is None or relevant_diff.is_empty: relevant_diff = Polygon() @@ -281,7 +280,7 @@ def process_geometry( for key in ProcessResult.__annotations__: geometry = result_dict.get(key, Polygon()) # noqa if not geometry.is_empty: - geometry = unary_union(geometry) + geometry = safe_unary_union(geometry) result_dict[key] = geometry # noqa return result_dict @@ -289,6 +288,7 @@ def process_geometry( def process( self, + dict_thematic =None, relevant_distances: Iterable[float] = None, relevant_distance=1, od_strategy=OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE, @@ -326,34 +326,56 @@ def process( dict_series = {} dict_series_queue = {} futures = [] - dict_thematic = self.dict_thematic + if dict_thematic is None: + dict_thematic = self.dict_thematic if self.multi_as_single_modus: dict_thematic = multipolygons_to_singles(dict_thematic) - with ThreadPoolExecutor(max_workers=self.max_workers) as executor:#max_workers=5 + if self.max_workers!=-1: + with ThreadPoolExecutor(max_workers=self.max_workers) as executor:#max_workers=5 + for key, geometry in dict_thematic.items(): + self.logger.feedback_info( + f"thematic id {str(key)} processed with relevant distances (m) [{str(self.relevant_distances)}]" + ) + dict_series[key] = {} + dict_series_queue[key] = {} + for relevant_distance in self.relevant_distances: + try: + future = executor.submit(self.process_geometry, geometry, + relevant_distance, + od_strategy, + threshold_overlap_percentage,) + futures.append(future) + dict_series_queue[key][relevant_distance] = future + except ValueError as e: + self.logger.feedback_warning("error for" + f"thematic id {str(key)} processed with relevant distances (m) [{str(self.relevant_distances)}]") + dict_series_queue[key][relevant_distance] = None + self.logger.feedback_warning(str(e)) + self.logger.feedback_debug ("waiting all started RD calculations") + wait(futures) + for id_theme,dict_dist in dict_series_queue.items(): + for reldist,future in dict_dist.items(): + dict_series[id_theme][reldist] = future.result() + else: for key, geometry in dict_thematic.items(): self.logger.feedback_info( f"thematic id {str(key)} processed with relevant distances (m) [{str(self.relevant_distances)}]" ) dict_series[key] = {} - dict_series_queue[key] = {} for relevant_distance in self.relevant_distances: try: - future = executor.submit(self.process_geometry, geometry, - relevant_distance, + self.relevant_distance = relevant_distance + processed_result = self.process_geometry( + geometry, + self.relevant_distance, od_strategy, - threshold_overlap_percentage,) - futures.append(future) - dict_series_queue[key][relevant_distance] = future + threshold_overlap_percentage, + ) except ValueError as e: - self.logger.feedback_warning("error for" + f"thematic id {str(key)} processed with relevant distances (m) [{str(self.relevant_distances)}]") - dict_series_queue[key][relevant_distance] = None self.logger.feedback_warning(str(e)) - self.logger.feedback_debug ("waiting all started RD calculations") - wait(futures) - for id_theme,dict_dist in dict_series_queue.items(): - for reldist,future in dict_dist.items(): - dict_series[id_theme][reldist] = future.result() + processed_result = None + + dict_series[key][self.relevant_distance] = processed_result if self.multi_as_single_modus: dict_series = merge_process_results(dict_series) @@ -368,6 +390,7 @@ def process( def predictor( self, + dict_thematic=None, relevant_distances=np.arange(0, 300, 10, dtype=int) / 100, od_strategy=OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE, threshold_overlap_percentage=50, @@ -435,13 +458,15 @@ def predictor( Logs: - Debug logs the thematic element key being processed. """ + if dict_thematic is None: + dict_thematic=self.dict_thematic dict_predictions = defaultdict(dict) dict_series = self.process( relevant_distances=relevant_distances, od_strategy=od_strategy, threshold_overlap_percentage=threshold_overlap_percentage, ) - dict_thematic = self.dict_thematic + diffs_dict = diffs_from_dict_series(dict_series, dict_thematic) @@ -490,102 +515,142 @@ def predictor( diffs_dict, ) + # def compare( + # self, + # threshold_area=5, + # threshold_percentage=1, + # dict_unchanged=None, + # ): + # """ + # Compares input-geometries (with formula) and evaluates these geometries: An attribute is added to evaluate and decide if new + # proposals can be used + # """ + # dict_series, dict_predictions, diffs = self.predictor(self.relevant_distances) + # if dict_unchanged is None: + # dict_unchanged = {} + # theme_ids = list(dict_series.keys()) + # dict_evaluated_result = {} + # prop_dictionary = {} + # # Fill the dictionary-structure with empty values + # for theme_id in theme_ids: + # dict_evaluated_result[theme_id] = {} + # prop_dictionary[theme_id] = {} + # for dist in dict_series[theme_id].keys(): + # prop_dictionary[theme_id][dist] = {} + # for theme_id in dict_unchanged.keys(): + # dict_evaluated_result[theme_id]={} + # prop_dictionary[theme_id] = {} + # + # for theme_id, dict_results in dict_predictions.items(): + # equality = False + # for dist in sorted(dict_results.keys()): + # if equality: + # break + # geomresult = dict_results[dist]["result"] + # actual_formula = self.get_brdr_formula(geomresult) + # prop_dictionary[theme_id][dist][FORMULA_FIELD_NAME] = json.dumps( + # actual_formula + # ) + # base_formula = None + # if ( + # theme_id in self.dict_thematic_properties + # and FORMULA_FIELD_NAME in self.dict_thematic_properties[theme_id] + # ): + # base_formula = self.dict_thematic_properties[theme_id][ + # FORMULA_FIELD_NAME + # ] + # equality, prop = _check_equality( + # base_formula, + # actual_formula, + # threshold_area, + # threshold_percentage, + # ) + # if equality: + # dict_evaluated_result[theme_id][dist] = dict_predictions[theme_id][ + # dist + # ] + # prop_dictionary[theme_id][dist][EVALUATION_FIELD_NAME] = prop + # break + # + # evaluated_theme_ids = [ + # theme_id for theme_id, value in dict_evaluated_result.items() if value != {} + # ] + # + # # fill where no equality is found/ The biggest predicted distance is returned as + # # proposal + # for theme_id in theme_ids: + # if theme_id not in evaluated_theme_ids: + # if len(dict_predictions[theme_id].keys()) == 0: + # result = dict_series[theme_id][0] + # dict_evaluated_result[theme_id][0] = result + # prop_dictionary[theme_id][0][FORMULA_FIELD_NAME] = json.dumps( + # self.get_brdr_formula(result["result"]) + # ) + # prop_dictionary[theme_id][0][ + # EVALUATION_FIELD_NAME + # ] = Evaluation.NO_PREDICTION_5 + # continue + # # Add all predicted features so they can be manually checked + # for dist in dict_predictions[theme_id].keys(): + # predicted_resultset = dict_predictions[theme_id][dist] + # dict_evaluated_result[theme_id][dist] = predicted_resultset + # prop_dictionary[theme_id][dist][FORMULA_FIELD_NAME] = json.dumps( + # self.get_brdr_formula(predicted_resultset["result"]) + # ) + # prop_dictionary[theme_id][dist][ + # EVALUATION_FIELD_NAME + # ] = Evaluation.TO_CHECK_4 + # + # for theme_id, geom in dict_unchanged.items(): + # dict_evaluated_result[theme_id] = {0: {"result": geom}} + # prop_dictionary[theme_id] = { + # 0: { + # #"result": geom, + # EVALUATION_FIELD_NAME: Evaluation.NO_CHANGE_6, + # FORMULA_FIELD_NAME: json.dumps(self.get_brdr_formula(geom)), + # } + # } + # return dict_evaluated_result, prop_dictionary + def compare( self, - threshold_area=5, - threshold_percentage=1, - dict_unchanged=None, + affected=None, ): """ - Compares input-geometries (with formula) and evaluates these geometries: An attribute is added to evaluate and decide if new + Compares and evaluate input-geometries (with formula).Attributes are added to evaluate and decide if new proposals can be used """ - dict_series, dict_predictions, diffs = self.predictor(self.relevant_distances) - if dict_unchanged is None: - dict_unchanged = {} - theme_ids = list(dict_series.keys()) - dict_evaluated_result = {} + if affected is None: + affected =list(self.dict_thematic.keys()) + dict_affected={} + dict_unaffected={} + for id_theme,geom in self.dict_thematic.items(): + if id_theme in affected: + dict_affected[id_theme] = geom + else: + dict_unaffected[id_theme] = geom + self.dict_thematic=dict_affected + #AFFECTED + dict_series, dict_affected_predictions, diffs = self.predictor(dict_thematic=dict_affected,relevant_distances=self.relevant_distances) + dict_predictions_evaluated = {} prop_dictionary = {} - # Fill the dictionary-structure with empty values - for theme_id in theme_ids: - dict_evaluated_result[theme_id] = {} + for theme_id, dict_predictions_results in dict_affected_predictions.items(): + dict_predictions_evaluated[theme_id] = {} prop_dictionary[theme_id] = {} - for dist in dict_series[theme_id].keys(): + for dist in sorted(dict_predictions_results.keys()): prop_dictionary[theme_id][dist] = {} - for theme_id in dict_unchanged.keys(): - dict_evaluated_result[theme_id]={} - prop_dictionary[theme_id] = {} - - for theme_id, dict_results in dict_predictions.items(): - equality = False - for dist in sorted(dict_results.keys()): - if equality: - break - geomresult = dict_results[dist]["result"] - actual_formula = self.get_brdr_formula(geomresult) - prop_dictionary[theme_id][dist][FORMULA_FIELD_NAME] = json.dumps( - actual_formula - ) - base_formula = None - if ( - theme_id in self.dict_thematic_properties - and FORMULA_FIELD_NAME in self.dict_thematic_properties[theme_id] - ): - base_formula = self.dict_thematic_properties[theme_id][ - FORMULA_FIELD_NAME - ] - equality, prop = _check_equality( - base_formula, - actual_formula, - threshold_area, - threshold_percentage, - ) - if equality: - dict_evaluated_result[theme_id][dist] = dict_predictions[theme_id][ - dist - ] - prop_dictionary[theme_id][dist][EVALUATION_FIELD_NAME] = prop - break - - evaluated_theme_ids = [ - theme_id for theme_id, value in dict_evaluated_result.items() if value != {} - ] - - # fill where no equality is found/ The biggest predicted distance is returned as - # proposal - for theme_id in theme_ids: - if theme_id not in evaluated_theme_ids: - if len(dict_predictions[theme_id].keys()) == 0: - result = dict_series[theme_id][0] - dict_evaluated_result[theme_id][0] = result - prop_dictionary[theme_id][0][FORMULA_FIELD_NAME] = json.dumps( - self.get_brdr_formula(result["result"]) - ) - prop_dictionary[theme_id][0][ - EVALUATION_FIELD_NAME - ] = Evaluation.NO_PREDICTION_5 - continue - # Add all predicted features so they can be manually checked - for dist in dict_predictions[theme_id].keys(): - predicted_resultset = dict_predictions[theme_id][dist] - dict_evaluated_result[theme_id][dist] = predicted_resultset - prop_dictionary[theme_id][dist][FORMULA_FIELD_NAME] = json.dumps( - self.get_brdr_formula(predicted_resultset["result"]) - ) - prop_dictionary[theme_id][dist][ - EVALUATION_FIELD_NAME - ] = Evaluation.TO_CHECK_4 - - for theme_id, geom in dict_unchanged.items(): - dict_evaluated_result[theme_id] = {0: {"result": geom}} - prop_dictionary[theme_id] = { - 0: { - #"result": geom, - EVALUATION_FIELD_NAME: Evaluation.NO_CHANGE_6, - FORMULA_FIELD_NAME: json.dumps(self.get_brdr_formula(geom)), - } - } - return dict_evaluated_result, prop_dictionary + props = _evaluate(self,id_theme=theme_id,geom_predicted=dict_predictions_results[dist]["result"]) + dict_predictions_evaluated[theme_id][dist] = dict_affected_predictions[theme_id][dist] + prop_dictionary[theme_id][dist] = props + #UNAFFECTED + dict_unaffected_series = self.process(dict_thematic=dict_unaffected,relevant_distances=[0]) + for theme_id, dict_unaffected_results in dict_unaffected_series.items(): + dict_predictions_evaluated[theme_id] = {} + prop_dictionary[theme_id] = {0:{}} + props = _evaluate(self,id_theme=theme_id,geom_predicted=dict_unaffected_results[0]["result"]) + dict_predictions_evaluated[theme_id][0] = dict_affected_predictions[theme_id][0] + prop_dictionary[theme_id][0] = props + return dict_predictions_evaluated, prop_dictionary def get_brdr_formula(self, geometry: BaseGeometry, with_geom=False): """ @@ -612,6 +677,7 @@ def get_brdr_formula(self, geometry: BaseGeometry, with_geom=False): "brdr_version": str(__version__), "reference_source": self.dict_reference_source, "full": True, + "area": round(geometry.area, 2), "reference_features": {}, "reference_od": None, } @@ -679,7 +745,7 @@ def get_brdr_formula(self, geometry: BaseGeometry, with_geom=False): dict_formula[LAST_VERSION_DATE] = last_version_date.strftime(DATE_FORMAT) geom_od = buffer_pos( buffer_neg( - safe_difference(geometry, make_valid(unary_union(intersected))), + safe_difference(geometry, safe_unary_union(intersected)), CORR_DISTANCE, ), CORR_DISTANCE, @@ -801,8 +867,7 @@ def save_results( def get_thematic_union(self): if self.thematic_union is None: - self.thematic_union = make_valid( - unary_union(list(self.dict_thematic.values())) + self.thematic_union = safe_unary_union(list(self.dict_thematic.values()) ) return self.thematic_union @@ -1011,8 +1076,7 @@ def _od_snap_all_side(self, geometry,relevant_distance): def _get_reference_union(self): if self.reference_union is None: - self.reference_union = make_valid( - unary_union(list(self.dict_reference.values())) + self.reference_union = safe_unary_union(list(self.dict_reference.values()) ) return self.reference_union @@ -1043,7 +1107,7 @@ def _postprocess_preresult(self, preresult, geom_thematic,relevant_distance) -> # Process array buffer_distance = relevant_distance/2 result = [] - geom_preresult = make_valid(unary_union(preresult)) + geom_preresult = safe_unary_union(preresult) geom_thematic = make_valid(geom_thematic) if not (geom_thematic is None or geom_thematic.is_empty): @@ -1115,14 +1179,15 @@ def _postprocess_preresult(self, preresult, geom_thematic,relevant_distance) -> self.logger.feedback_warning( "Empty result: -->resulting geometry = empty geometry" ) + # geom_thematic_result = geom_thematic - geom_thematic_result = Polygon() + geom_thematic_result = Polygon() #TODO : this results in disappearance of objects -> instead, return original geometry # group all initial multipolygons into a new resulting dictionary result.append(geom_thematic_result) # create all resulting geometries - geom_thematic_result = make_valid(unary_union(result)) + geom_thematic_result = safe_unary_union(result) # negative and positive buffer is added to the difference-calculations, to # remove 'very small' differences (smaller than the correction distance) @@ -1303,7 +1368,7 @@ def _get_relevant_polygons_from_geom(geometry: BaseGeometry, buffer_distance: fl # If the input geometry is empty or None, do nothing. return geometry else: - geometry = make_valid(unary_union(geometry)) + geometry = safe_unary_union(geometry) # Create a GeometryCollection from the input geometry. geometry_collection = GeometryCollection(geometry) array = [] @@ -1314,7 +1379,7 @@ def _get_relevant_polygons_from_geom(geometry: BaseGeometry, buffer_distance: fl relevant_geom = buffer_neg(g, buffer_distance) if relevant_geom is not None and not relevant_geom.is_empty: array.append(g) - return make_valid(unary_union(array)) + return safe_unary_union(array) def _equal_geom_in_array(geom, geom_array): @@ -1358,7 +1423,7 @@ def _check_equality( == actual_formula["reference_features"].keys() and od_alike ): - if base_formula["full"] and base_formula["full"]: + if base_formula["full"] and actual_formula["full"]: return True, Evaluation.EQUALITY_FORMULA_GEOM_1 equal_reference_features = True @@ -1390,6 +1455,104 @@ def _check_equality( equal_reference_features = False if equal_reference_features: return True, Evaluation.EQUALITY_FORMULA_2 - if base_formula["full"] and base_formula["full"] and od_alike: + if base_formula["full"] and actual_formula["full"] and od_alike: return True, Evaluation.EQUALITY_GEOM_3 return False, Evaluation.NO_PREDICTION_5 + +def _evaluate(self,id_theme,geom_predicted): + """ + function that evaluates a predicted geometry and returns a properties-dictionary + """ + threshold_od_percentage = 1 + threshold_area = 5 + threshold_percentage = 1 + properties = { + FORMULA_FIELD_NAME:"", + EVALUATION_FIELD_NAME: Evaluation.TO_CHECK_4, + FULL_BASE_FIELD_NAME: None, + FULL_ACTUAL_FIELD_NAME: None, + EQUAL_REFERENCE_FEATURES_FIELD_NAME: None, + DIFF_PERCENTAGE_FIELD_NAME: None, + DIFF_AREA_FIELD_NAME: None, + OD_ALIKE_FIELD_NAME: None, + } + actual_formula = self.get_brdr_formula(geom_predicted) + properties[FORMULA_FIELD_NAME] = json.dumps( + actual_formula + ) + base_formula = None + if ( + id_theme in self.dict_thematic_properties + and FORMULA_FIELD_NAME in self.dict_thematic_properties[id_theme] + ): + base_formula = self.dict_thematic_properties[id_theme][ + FORMULA_FIELD_NAME + ] + + if base_formula is None or actual_formula is None: + properties[EVALUATION_FIELD_NAME]= Evaluation.NO_PREDICTION_5 + return properties + properties[FULL_BASE_FIELD_NAME] = base_formula["full"] + properties[FULL_ACTUAL_FIELD_NAME] = actual_formula["full"] + od_alike = False + if base_formula["reference_od"] is None and actual_formula["reference_od"] is None: + od_alike = True + elif base_formula["reference_od"] is None or actual_formula["reference_od"] is None: + od_alike = False + elif ( + abs( + base_formula["reference_od"]["area"] + - actual_formula["reference_od"]["area"] + ) + * 100 + / base_formula["reference_od"]["area"] + ) < threshold_od_percentage: + od_alike = True + properties[OD_ALIKE_FIELD_NAME] = od_alike + + if ( + base_formula["reference_features"].keys() + == actual_formula["reference_features"].keys() + and od_alike + ): + equal_reference_features = True + + if base_formula["full"] and base_formula["full"]: + properties[EVALUATION_FIELD_NAME] = Evaluation.EQUALITY_FORMULA_GEOM_1 + + max_diff_area_reference_feature = 0 + max_diff_percentage_reference_feature = 0 + for key in base_formula["reference_features"].keys(): + + if ( + base_formula["reference_features"][key]["full"] + != actual_formula["reference_features"][key]["full"] + ): + equal_reference_features = False + + + diff_area_reference_feature = abs( + base_formula["reference_features"][key]["area"] + - actual_formula["reference_features"][key]["area"] + ) + diff_percentage_reference_feature =( + abs( + base_formula["reference_features"][key]["area"] + - actual_formula["reference_features"][key]["area"] + ) + * 100 + / base_formula["reference_features"][key]["area"] + ) + if diff_area_reference_feature>max_diff_area_reference_feature: + max_diff_area_reference_feature=diff_area_reference_feature + if diff_percentage_reference_feature>max_diff_percentage_reference_feature: + max_diff_percentage_reference_feature=diff_percentage_reference_feature + properties[EQUAL_REFERENCE_FEATURES_FIELD_NAME] = equal_reference_features + properties[DIFF_AREA_FIELD_NAME] = max_diff_area_reference_feature + properties[DIFF_PERCENTAGE_FIELD_NAME] = max_diff_percentage_reference_feature + if equal_reference_features: + properties[EVALUATION_FIELD_NAME]= Evaluation.EQUALITY_FORMULA_2 + if base_formula["full"] and actual_formula["full"] and od_alike: + properties[EVALUATION_FIELD_NAME]= Evaluation.EQUALITY_GEOM_3 + return properties + diff --git a/brdr/constants.py b/brdr/constants.py index f43e20f..7a37a1e 100644 --- a/brdr/constants.py +++ b/brdr/constants.py @@ -38,11 +38,18 @@ # MULTI_SINGLE_ID_SEPARATOR #separator to split multipolygon_ids to single polygons MULTI_SINGLE_ID_SEPARATOR = "*$*" +PREFIX_FIELDNAME = "brdr_" +FORMULA_FIELD_NAME = PREFIX_FIELDNAME + "formula" +EVALUATION_FIELD_NAME = PREFIX_FIELDNAME + "evaluation" +DIFF_PERCENTAGE_FIELD_NAME = PREFIX_FIELDNAME + "diff_percentage" +DIFF_AREA_FIELD_NAME = PREFIX_FIELDNAME + "diff_area" +FULL_BASE_FIELD_NAME = PREFIX_FIELDNAME + "full_base" +FULL_ACTUAL_FIELD_NAME = PREFIX_FIELDNAME + "full_actual" +EQUAL_REFERENCE_FEATURES_FIELD_NAME = PREFIX_FIELDNAME + "equal_reference_features" +OD_ALIKE_FIELD_NAME = PREFIX_FIELDNAME + "od_alike" -FORMULA_FIELD_NAME = "brdr_formula" -EVALUATION_FIELD_NAME = "brdr_evaluation" -NR_CALCULATION_FIELD_NAME = "brdr_nr_calculations" -RELEVANT_DISTANCE_FIELD_NAME = "brdr_relevant_distance" +NR_CALCULATION_FIELD_NAME = PREFIX_FIELDNAME + "nr_calculations" +RELEVANT_DISTANCE_FIELD_NAME = PREFIX_FIELDNAME + "relevant_distance" LAST_VERSION_DATE = "last_version_date" VERSION_DATE = "version_date" diff --git a/brdr/geometry_utils.py b/brdr/geometry_utils.py index bca3996..aea7f53 100644 --- a/brdr/geometry_utils.py +++ b/brdr/geometry_utils.py @@ -13,9 +13,11 @@ from shapely import get_parts from shapely import intersection from shapely import is_empty +from shapely import make_valid from shapely import polygons from shapely import symmetric_difference from shapely import to_wkt +from shapely import unary_union from shapely import union from shapely.geometry.base import BaseGeometry from shapely.prepared import prep @@ -507,6 +509,8 @@ def fill_and_remove_gaps(input_geometry, buffer_value): ix_part = ix_part + 1 return cleaned_geometry +def safe_unary_union(geometries): + return make_valid(unary_union(geometries)) def get_bbox(geometry): """ diff --git a/brdr/grb.py b/brdr/grb.py index 5c1e484..85e6201 100644 --- a/brdr/grb.py +++ b/brdr/grb.py @@ -7,7 +7,6 @@ import numpy as np from shapely import intersects, Polygon from shapely.geometry import shape -from shapely.geometry.base import BaseGeometry from brdr.aligner import Aligner from brdr.constants import ( @@ -26,7 +25,7 @@ from brdr.constants import GRB_VERSION_DATE from brdr.constants import MAX_REFERENCE_BUFFER from brdr.enums import GRBType -from brdr.geometry_utils import buffer_pos, safe_intersection +from brdr.geometry_utils import buffer_pos, safe_intersection, safe_unary_union from brdr.geometry_utils import create_donut from brdr.geometry_utils import features_by_geometric_operation from brdr.geometry_utils import get_bbox @@ -78,20 +77,22 @@ def is_grb_changed( return False -def get_geoms_affected_by_grb_change( - aligner, +def get_affected_by_grb_change( + dict_thematic, grb_type=GRBType.ADP, date_start=date.today(), date_end=date.today(), one_by_one=False, border_distance=0, + geometry_thematic_union=None, + crs=DEFAULT_CRS, ): """ - Get a dictionary of thematic geometries that are affected bij GRB-changes in a + Get a list of affected and unaffected IDs by GRB-changes in a specific timespan Args: - aligner: Aligner instance + dict_thematic: dictionary if thematicID & Geometry grb_type: Type of GRB: parcels, buildings,... date_start: start-date to check changes in GRB date_end: end-date to check changes in GRB @@ -106,12 +107,9 @@ def get_geoms_affected_by_grb_change( dictionary of affected geometries """ - dict_thematic = aligner.dict_thematic - # if aligner.multi_as_single_modus: - # dict_thematic = merge_dict(dict_thematic) - crs = aligner.CRS - affected_dict: dict[str, BaseGeometry] = {} - unchanged_dict: dict[str, BaseGeometry] = {} + + affected = [] + unaffected = [] if border_distance > 0: for key in dict_thematic.keys(): dict_thematic[key] = create_donut(dict_thematic[key], border_distance) @@ -119,15 +117,16 @@ def get_geoms_affected_by_grb_change( for key in dict_thematic: geom = dict_thematic[key] if is_grb_changed(geom, grb_type, date_start, date_end): - affected_dict[key] = geom + affected.append(key) else: - unchanged_dict[key] = geom - return affected_dict, unchanged_dict + unaffected.append(key) + return affected, unaffected else: # Temporal filter on VERDATUM - geometry = aligner.get_thematic_union() + if geometry_thematic_union is None: + geometry_thematic_union = safe_unary_union(list(dict_thematic.values())) coll_changed_grb, name_reference_id = get_collection_grb_actual( - geometry, + geometry_thematic_union, grb_type=grb_type, partition=1000, date_start=date_start, @@ -140,7 +139,7 @@ def get_geoms_affected_by_grb_change( if len(dict_changed_grb) == 0: logging.info("No detected changes") - return affected_dict, dict_thematic # empty affected dict + return affected, list(dict_thematic.keys()) # empty affected dict logging.info("Changed parcels in timespan: " + str(len(dict_changed_grb))) thematic_intersections = features_by_geometric_operation( list(dict_thematic.values()), @@ -150,11 +149,8 @@ def get_geoms_affected_by_grb_change( ) logging.info("Number of filtered features: " + str(len(thematic_intersections))) for key, geom in dict_thematic.items(): - if key in thematic_intersections: - affected_dict[key] = geom - else: - unchanged_dict[key] = geom - return affected_dict, unchanged_dict + affected.append(key) if key in thematic_intersections else unaffected.append(key) + return affected, unaffected def get_last_version_date( @@ -399,18 +395,20 @@ def update_to_actual_grb( base_aligner_result.load_thematic_data(DictLoader(dict_thematic)) base_aligner_result.name_thematic_id = id_theme_fieldname - dict_affected, dict_unchanged = get_geoms_affected_by_grb_change( - base_aligner_result, + affected, unaffected = get_affected_by_grb_change( + dict_thematic=base_aligner_result.dict_thematic, grb_type=GRBType.ADP, date_start=datetime_start, date_end=datetime_end, one_by_one=False, - ) + geometry_thematic_union=base_aligner_result.get_thematic_union(), + crs=base_aligner_result.CRS, + ) logger.feedback_info( "Number of possible affected OE-thematic during timespan: " - + str(len(dict_affected)) + + str(len(affected)) ) - if len(dict_affected) == 0: + if len(affected) == 0: logger.feedback_info( "No change detected in referencelayer during timespan. Script is finished" ) @@ -419,9 +417,9 @@ def update_to_actual_grb( logger.feedback_debug(str(formula_field)) # Initiate a Aligner to reference thematic features to the actual borders - actual_aligner = Aligner(feedback=feedback) + actual_aligner = Aligner(feedback=feedback,max_workers=None) actual_aligner.load_thematic_data( - DictLoader(data_dict=dict_affected, data_dict_properties=dict_thematic_props) + DictLoader(data_dict=dict_thematic, data_dict_properties=dict_thematic_props) ) actual_aligner.load_reference_data( GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner) @@ -430,9 +428,7 @@ def update_to_actual_grb( actual_aligner.relevant_distances = ( np.arange(0, max_distance_for_actualisation * 100, 10, dtype=int) / 100 ) - dict_evaluated, prop_dictionary = actual_aligner.compare( - threshold_area=5, threshold_percentage=1, dict_unchanged=dict_unchanged - ) + dict_evaluated, prop_dictionary = actual_aligner.compare(affected = affected) return get_series_geojson_dict( dict_evaluated, diff --git a/examples/example_evaluate.py b/examples/example_evaluate.py index 62649bb..f9a7471 100644 --- a/examples/example_evaluate.py +++ b/examples/example_evaluate.py @@ -8,7 +8,7 @@ from brdr.enums import GRBType from brdr.grb import GRBActualLoader from brdr.grb import GRBFiscalParcelLoader -from brdr.grb import get_geoms_affected_by_grb_change +from brdr.grb import get_affected_by_grb_change from brdr.loader import DictLoader, GeoJsonFileLoader from brdr.utils import get_series_geojson_dict @@ -37,32 +37,26 @@ print(key + ": " + str(thematic_dict_formula[key])) base_aligner_result = Aligner() base_aligner_result.load_thematic_data(DictLoader(thematic_dict_result)) -dict_affected, dict_unchanged = get_geoms_affected_by_grb_change( - base_aligner_result, +affected, unaffected = get_affected_by_grb_change( + dict_thematic = thematic_dict_result, grb_type=GRBType.ADP, date_start=date(2022, 1, 1), date_end=date.today(), one_by_one=False, ) -if dict_affected == {}: +if len(affected)==0: print("No affected dicts") exit() -for key, value in dict_affected.items(): - print(key + ": " + value.wkt) +print("Affected_IDs: " + str(affected)) actual_aligner = Aligner() -loader = DictLoader(dict_affected) actual_aligner.load_thematic_data( - DictLoader(data_dict=dict_affected, data_dict_properties=thematic_dict_formula) + DictLoader(data_dict=thematic_dict_result, data_dict_properties=thematic_dict_formula) ) actual_aligner.load_reference_data( GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner) ) actual_aligner.relevant_distances = np.arange(0, 200, 10, dtype=int) / 100 -dict_evaluated, prop_dictionary = actual_aligner.compare( - threshold_area=5, - threshold_percentage=1, - dict_unchanged=dict_unchanged, -) +dict_evaluated, prop_dictionary = actual_aligner.compare(affected=affected) fc = get_series_geojson_dict( dict_evaluated, @@ -72,7 +66,6 @@ ) print(fc["result"]) fcs = actual_aligner.get_results_as_geojson(formula=True) -print(fcs["result"]) for feature in fc["result"]["features"]: id = feature["properties"][actual_aligner.name_thematic_id] diff --git a/examples/example_evaluate_ao.py b/examples/example_evaluate_ao.py index 70e7c22..ca1d209 100644 --- a/examples/example_evaluate_ao.py +++ b/examples/example_evaluate_ao.py @@ -6,7 +6,7 @@ from brdr.constants import EVALUATION_FIELD_NAME, FORMULA_FIELD_NAME from brdr.enums import GRBType from brdr.grb import ( - get_geoms_affected_by_grb_change, + get_affected_by_grb_change, GRBFiscalParcelLoader, GRBActualLoader, ) @@ -30,32 +30,30 @@ thematic_dict_formula[key] = { FORMULA_FIELD_NAME: base_aligner.get_brdr_formula(thematic_dict_result[key]) } + print(key + ": " + thematic_dict_result[key].wkt) + print(key + ": " + str(thematic_dict_formula[key])) base_aligner_result = Aligner() base_aligner_result.load_thematic_data(DictLoader(thematic_dict_result)) -dict_affected, dict_unchanged = get_geoms_affected_by_grb_change( - base_aligner_result, +affected, unaffected = get_affected_by_grb_change( + dict_thematic = thematic_dict_result, grb_type=GRBType.ADP, date_start=date(2022, 1, 1), date_end=date.today(), one_by_one=False, ) -if dict_affected == {}: +if len(affected)==0: print("No affected dicts") exit() - +print("Affected_IDs: " + str(affected)) actual_aligner = Aligner() actual_aligner.load_thematic_data( - DictLoader(data_dict=dict_affected, data_dict_properties=thematic_dict_formula) + DictLoader(data_dict=thematic_dict_result, data_dict_properties=thematic_dict_formula) ) -loader = GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner) -actual_aligner.load_reference_data(loader) -series = np.arange(0, 300, 10, dtype=int) / 100 - -dict_evaluated, prop_dictionary = actual_aligner.compare( - threshold_area=5, - threshold_percentage=1, - dict_unchanged=dict_unchanged, +actual_aligner.load_reference_data( + GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner) ) +actual_aligner.relevant_distances = np.arange(0, 200, 10, dtype=int) / 100 +dict_evaluated, prop_dictionary = actual_aligner.compare(affected=affected) fc = get_series_geojson_dict( dict_evaluated, @@ -63,7 +61,10 @@ id_field=actual_aligner.name_thematic_id, series_prop_dict=prop_dictionary, ) +print(fc["result"]) +fcs = actual_aligner.get_results_as_geojson(formula=True) + for feature in fc["result"]["features"]: id = feature["properties"][actual_aligner.name_thematic_id] evaluation = feature["properties"][EVALUATION_FIELD_NAME] - print(id + ": " + evaluation) + print(id + ": " + evaluation) \ No newline at end of file diff --git a/examples/example_multi_to_single.py b/examples/example_multi_to_single.py index e50ec95..3a14748 100644 --- a/examples/example_multi_to_single.py +++ b/examples/example_multi_to_single.py @@ -5,39 +5,42 @@ from examples import print_brdr_formula from examples import show_map -#This example shows the usage of the setting 'multi_as_single_modus' -#True: (default): All polygons inside a MultiPolygon will be processed seperataly by the algorithm, and merged after processing. -#False: Multipolygon will be processed directly by the algorithm +# This example shows the usage of the setting 'multi_as_single_modus' +# True: (default): All polygons inside a MultiPolygon will be processed seperataly by the algorithm, and merged after processing. +# False: Multipolygon will be processed directly by the algorithm -#Example (ErfgoedObject): https://inventaris.onroerenderfgoed.be/erfgoedobjecten/305858 -loader = OnroerendErfgoedLoader([305858],oetype=OEType.EO) -relevant_distance = 5 #rd is taken very high to show the difference -od_strategy =4 +# Example (ErfgoedObject): https://inventaris.onroerenderfgoed.be/erfgoedobjecten/305858 +loader = OnroerendErfgoedLoader([305858], oetype=OEType.EO) +relevant_distance = 5 # rd is taken very high to show the difference +od_strategy = 4 # EXAMPLE of "multi_as_single_modus"=FALSE -print ("EXAMPLE with 'multi_as_single_modus'=False") +print("EXAMPLE with 'multi_as_single_modus'=False") aligner = Aligner() aligner.multi_as_single_modus = False aligner.load_thematic_data(loader) aligner.load_reference_data( GRBActualLoader(aligner=aligner, grb_type=GRBType.GBG, partition=1000) ) -dict_results = aligner.process(relevant_distance=relevant_distance, od_strategy=od_strategy) +dict_results = aligner.process( + relevant_distance=relevant_distance, od_strategy=od_strategy +) aligner.save_results("output/") print_brdr_formula(dict_results, aligner) show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) # WITH "multi_as_single_modus"=True -print ("EXAMPLE with 'multi_as_single_modus'=True") +print("EXAMPLE with 'multi_as_single_modus'=True") aligner = Aligner() aligner.multi_as_single_modus = True aligner.load_thematic_data(loader) aligner.load_reference_data( GRBActualLoader(aligner=aligner, grb_type=GRBType.GBG, partition=1000) ) -dict_results = aligner.process(relevant_distance=relevant_distance, od_strategy=od_strategy) +dict_results = aligner.process( + relevant_distance=relevant_distance, od_strategy=od_strategy +) aligner.save_results("output/") print_brdr_formula(dict_results, aligner) show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) - diff --git a/examples/example_parcel_change_detector.py b/examples/example_parcel_change_detector.py index 7d3fe9c..2e9c33e 100644 --- a/examples/example_parcel_change_detector.py +++ b/examples/example_parcel_change_detector.py @@ -1,4 +1,4 @@ -import logging +import os from datetime import datetime from brdr.aligner import Aligner @@ -6,6 +6,7 @@ from brdr.grb import GRBFiscalParcelLoader from brdr.grb import update_to_actual_grb from brdr.oe import OnroerendErfgoedLoader +from brdr.utils import write_geojson # This code shows an example how the aligner can be used inside a flow of # parcel change detection: @@ -85,6 +86,8 @@ max_distance_for_actualisation=max_distance_for_actualisation, ) +write_geojson(os.path.join("output/", "parcel_change_detector_with.geojson"), fcs["result"]) + counter_equality = 0 counter_equality_by_alignment = 0 diff --git a/examples/example_speedtest.py b/examples/example_speedtest.py index a808329..e2cf994 100644 --- a/examples/example_speedtest.py +++ b/examples/example_speedtest.py @@ -4,6 +4,7 @@ from brdr.aligner import Aligner from brdr.loader import GeoJsonFileLoader + def main(): # Initiate brdr aligner = Aligner(relevant_distance=2,max_workers=None) diff --git a/pyproject.toml b/pyproject.toml index 5819687..057bcc2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,13 +66,13 @@ target-version = ['py39', 'py310', 'py311', 'py312'] # Config for hatch environments [tool.hatch.envs.dev] features = [ - 'dev', + 'dev', ] # Config for hatch test [tool.hatch.envs.hatch-test] features = [ - 'test', + 'test', ] [[tool.hatch.envs.hatch-test.matrix]] diff --git a/tests/test_aligner.py b/tests/test_aligner.py index 2dc2f7d..3c971db 100644 --- a/tests/test_aligner.py +++ b/tests/test_aligner.py @@ -17,7 +17,7 @@ from brdr.grb import ( GRBActualLoader, GRBFiscalParcelLoader, - get_geoms_affected_by_grb_change, + get_affected_by_grb_change, ) from brdr.loader import GeoJsonLoader, DictLoader from brdr.typings import FeatureCollection, ProcessResult @@ -374,46 +374,45 @@ def test_evaluate(self): thematic_dict_formula = {} thematic_dict_result = {} for key in base_process_result: - thematic_dict_result[key] = base_process_result[key][relevant_distance][ - "result" - ] + thematic_dict_result[key] = base_process_result[key][relevant_distance]["result"] thematic_dict_formula[key] = { - FORMULA_FIELD_NAME: base_aligner.get_brdr_formula( - thematic_dict_result[key] - ) + FORMULA_FIELD_NAME: base_aligner.get_brdr_formula(thematic_dict_result[key]) } - aligner_result = Aligner() - aligner_result.load_thematic_data(DictLoader(thematic_dict_result)) - dict_affected, dict_unchanged = get_geoms_affected_by_grb_change( - aligner=aligner_result, + print(key + ": " + thematic_dict_result[key].wkt) + print(key + ": " + str(thematic_dict_formula[key])) + base_aligner_result = Aligner() + base_aligner_result.load_thematic_data(DictLoader(thematic_dict_result)) + affected, unaffected = get_affected_by_grb_change( + dict_thematic=thematic_dict_result, grb_type=GRBType.ADP, date_start=date(2022, 1, 1), date_end=date.today(), one_by_one=False, ) - + if len(affected) == 0: + print("No affected dicts") + exit() + print("Affected_IDs: " + str(affected)) actual_aligner = Aligner() actual_aligner.load_thematic_data( - DictLoader( - data_dict=dict_affected, data_dict_properties=thematic_dict_formula - ) + DictLoader(data_dict=thematic_dict_result, data_dict_properties=thematic_dict_formula) ) - loader = GRBActualLoader( - grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner + actual_aligner.load_reference_data( + GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner) ) - actual_aligner.load_reference_data(loader) - series = np.arange(0, 200, 10, dtype=int) / 100 + actual_aligner.relevant_distances = np.arange(0, 200, 10, dtype=int) / 100 + dict_evaluated, prop_dictionary = actual_aligner.compare(affected=affected) - dict_evaluated, prop_dictionary = actual_aligner.compare( - threshold_area=5, - threshold_percentage=1, - ) fc = get_series_geojson_dict( dict_evaluated, crs=actual_aligner.CRS, id_field=actual_aligner.name_thematic_id, series_prop_dict=prop_dictionary, ) + print(fc["result"]) + fcs = actual_aligner.get_results_as_geojson(formula=True) + + def test_fully_aligned_geojson_output(self): aligned_shape = from_wkt( diff --git a/tests/test_examples.py b/tests/test_examples.py index 3bd1f02..ea63c84 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -191,7 +191,7 @@ def test_example_wanted_changes(self): # Example how to use a series (for histogram) series = np.arange(0, 300, 10, dtype=int) / 100 - dict_series = aligner.process(series, 4, 50) + dict_series = aligner.process(relevant_distances=series, od_strategy=4, threshold_overlap_percentage=50) resulting_areas = diffs_from_dict_series(dict_series, aligner.dict_thematic) for key in resulting_areas: if len(resulting_areas[key]) == len(series): diff --git a/tests/test_grb.py b/tests/test_grb.py index adc156f..29fd016 100644 --- a/tests/test_grb.py +++ b/tests/test_grb.py @@ -9,7 +9,7 @@ from brdr.grb import ( get_last_version_date, is_grb_changed, - get_geoms_affected_by_grb_change, + get_affected_by_grb_change, GRBSpecificDateParcelLoader, update_to_actual_grb, ) @@ -68,25 +68,25 @@ def test_get_geoms_affected_by_grb_change_outerborder(self): } aligner = Aligner() aligner.load_thematic_data(DictLoader(thematic_dict)) - dict_affected, dict_unchanged = get_geoms_affected_by_grb_change( - aligner=aligner, + affected, unaffected = get_affected_by_grb_change( + dict_thematic=thematic_dict, grb_type=GRBType.ADP, date_start=date.today() - timedelta(days=30), date_end=date.today(), one_by_one=False, border_distance=0, ) - assert len(dict_affected.keys()) > 0 + assert len(affected) > 0 - dict_affected, dict_unchanged = get_geoms_affected_by_grb_change( - aligner=aligner, + affected, unaffected = get_affected_by_grb_change( + dict_thematic=thematic_dict, grb_type=GRBType.ADP, date_start=date.today() - timedelta(days=30), date_end=date.today(), one_by_one=False, border_distance=10, ) - assert len(dict_affected.keys()) == 0 + assert len(affected) == 0 def test_get_geoms_affected_by_grb_change(self): thematic_dict = { @@ -101,23 +101,23 @@ def test_get_geoms_affected_by_grb_change(self): } aligner = Aligner() aligner.load_thematic_data(DictLoader(thematic_dict)) - dict_affected, dict_unchanged = get_geoms_affected_by_grb_change( - aligner=aligner, + affected,unaffected = get_affected_by_grb_change( + dict_thematic=thematic_dict, grb_type=GRBType.ADP, date_start=date.today() - timedelta(days=1), date_end=date.today(), one_by_one=True, ) - assert len(dict_affected.keys()) == 0 + assert len(affected) == 0 - dict_affected, dict_unchanged = get_geoms_affected_by_grb_change( - aligner=aligner, + affected,unaffected = get_affected_by_grb_change( + dict_thematic=thematic_dict, grb_type=GRBType.ADP, date_start=date.today() - timedelta(days=1000), date_end=date.today(), one_by_one=True, ) - assert len(dict_affected.keys()) == 0 + assert len(affected) == 0 thematic_dict2 = { "theme_id_2": from_wkt( "MultiPolygon (((174180.20077791667426936 171966.14649116666987538, " @@ -130,14 +130,14 @@ def test_get_geoms_affected_by_grb_change(self): } aligner2 = Aligner() aligner2.load_thematic_data(DictLoader(thematic_dict2)) - dict_affected, dict_unchanged = get_geoms_affected_by_grb_change( - aligner=aligner2, + affected,unaffected = get_affected_by_grb_change( + dict_thematic=thematic_dict2, grb_type=GRBType.ADP, date_start=date.today() - timedelta(days=1000), date_end=date.today(), one_by_one=True, ) - assert len(dict_affected.keys()) > 0 + assert len(affected) > 0 def test_get_geoms_affected_by_grb_change_bulk(self): thematic_dict = { @@ -152,23 +152,23 @@ def test_get_geoms_affected_by_grb_change_bulk(self): } aligner = Aligner() aligner.load_thematic_data(DictLoader(thematic_dict)) - dict_affected, dict_unchanged = get_geoms_affected_by_grb_change( - aligner=aligner, + affected,unaffected = get_affected_by_grb_change( + dict_thematic=thematic_dict, grb_type=GRBType.ADP, date_start=date.today() - timedelta(days=1), date_end=date.today(), one_by_one=False, ) - assert len(dict_affected.keys()) == 0 + assert len(affected) == 0 - dict_affected, dict_unchanged = get_geoms_affected_by_grb_change( - aligner=aligner, + affected,unaffected = get_affected_by_grb_change( + dict_thematic=thematic_dict, grb_type=GRBType.ADP, date_start=date.today() - timedelta(days=1000), date_end=date.today(), one_by_one=False, ) - assert len(dict_affected.keys()) > 0 + assert len(affected) > 0 def test_grbspecificdateparcelloader(self): thematic_dict = { diff --git a/tests/test_integration.py b/tests/test_integration.py index 0a77a56..1da1a90 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -55,7 +55,7 @@ def test_webservice_brdr(self): series = np.arange(0, 61, 1, dtype=float) / 10 - dict_series = aligner.process(series, openbaardomein_strategy, 50) + dict_series = aligner.process(relevant_distances=series, od_strategy=openbaardomein_strategy, threshold_overlap_percentage=50) dict_diffs = diffs_from_dict_series( dict_series, aligner.dict_thematic, DiffMetric.CHANGES_AREA ) diff --git a/tests/test_utils.py b/tests/test_utils.py index 59a06da..8c56abb 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -8,7 +8,6 @@ from brdr.oe import get_oe_dict_by_ids from brdr.typings import ProcessResult from brdr.utils import diffs_from_dict_series - # from brdr.utils import filter_dict_by_key from brdr.utils import get_breakpoints_zerostreak from brdr.utils import get_collection From 93f627840f552f1e5dd1fae49081228d10dd4284 Mon Sep 17 00:00:00 2001 From: dieuska Date: Fri, 27 Sep 2024 17:21:11 +0200 Subject: [PATCH 11/26] #99 added evaluation_fields --- brdr/aligner.py | 241 +++++++++++++++++--------------- brdr/enums.py | 19 +-- brdr/grb.py | 2 +- examples/example_evaluate.py | 13 +- examples/example_evaluate_ao.py | 13 +- tests/test_aligner.py | 2 +- 6 files changed, 147 insertions(+), 143 deletions(-) diff --git a/brdr/aligner.py b/brdr/aligner.py index 00c96ce..c28c7ad 100644 --- a/brdr/aligner.py +++ b/brdr/aligner.py @@ -150,6 +150,10 @@ def __init__( self.dict_processresults: dict[str, dict[float, ProcessResult]] = {} # dictionary with the 'predicted' results, grouped by theme_id and relevant_distance self.dict_predictions: dict[str, dict[float, ProcessResult]] = {} + # dictionary with the 'evaluated predicted' results, grouped by theme_id and relevant_distance + self.dict_evaluated_predictions: dict[str, dict[float, ProcessResult]] = {} + # dictionary with the 'evaluated predicted' properties, grouped by theme_id and relevant_distance + self.dict_evaluated_predictions_properties: dict[str, dict[float, {}]] = {} # Coordinate reference system # thematic geometries and reference geometries are assumed to be in the same CRS @@ -614,18 +618,19 @@ def predictor( def compare( self, - affected=None, + ids_to_compare=None, ): """ Compares and evaluate input-geometries (with formula).Attributes are added to evaluate and decide if new proposals can be used + affected: list with all IDs to evaluate. all other IDs will be unchanged. If None (default), all self.dict_thematic will be evaluated. """ - if affected is None: - affected =list(self.dict_thematic.keys()) + if ids_to_compare is None: + ids_to_compare =list(self.dict_thematic.keys()) dict_affected={} dict_unaffected={} for id_theme,geom in self.dict_thematic.items(): - if id_theme in affected: + if id_theme in ids_to_compare: dict_affected[id_theme] = geom else: dict_unaffected[id_theme] = geom @@ -639,17 +644,22 @@ def compare( prop_dictionary[theme_id] = {} for dist in sorted(dict_predictions_results.keys()): prop_dictionary[theme_id][dist] = {} - props = _evaluate(self,id_theme=theme_id,geom_predicted=dict_predictions_results[dist]["result"]) + props = self._evaluate(id_theme=theme_id,geom_predicted=dict_predictions_results[dist]["result"]) dict_predictions_evaluated[theme_id][dist] = dict_affected_predictions[theme_id][dist] prop_dictionary[theme_id][dist] = props #UNAFFECTED - dict_unaffected_series = self.process(dict_thematic=dict_unaffected,relevant_distances=[0]) - for theme_id, dict_unaffected_results in dict_unaffected_series.items(): + relevant_distance=0 + #dict_unaffected_series = self.process(dict_thematic=dict_unaffected,relevant_distances=[relevant_distance]) + #for theme_id, dict_unaffected_results in dict_unaffected_series.items(): + for theme_id, geom in dict_unaffected.items(): dict_predictions_evaluated[theme_id] = {} - prop_dictionary[theme_id] = {0:{}} - props = _evaluate(self,id_theme=theme_id,geom_predicted=dict_unaffected_results[0]["result"]) - dict_predictions_evaluated[theme_id][0] = dict_affected_predictions[theme_id][0] - prop_dictionary[theme_id][0] = props + prop_dictionary[theme_id] = {relevant_distance:{}} + props = self._evaluate(id_theme=theme_id,geom_predicted=geom) + props[EVALUATION_FIELD_NAME]=Evaluation.NO_CHANGE_6 + dict_predictions_evaluated[theme_id][relevant_distance] = {"result":geom} + prop_dictionary[theme_id][relevant_distance] = props + self.dict_evaluated_predictions = dict_predictions_evaluated + self.dict_evaluated_predictions_properties = prop_dictionary return dict_predictions_evaluated, prop_dictionary def get_brdr_formula(self, geometry: BaseGeometry, with_geom=False): @@ -769,11 +779,14 @@ def get_results_as_geojson( get a geojson of a dictionary containing the resulting geometries for all 'serial' relevant distances. If no dict_series is given, the dict_result returned. Optional: The descriptive formula is added as an attribute to the result""" - + prop_dictionary = None if resulttype == AlignerResultType.PROCESSRESULTS: dict_series = self.dict_processresults elif resulttype == AlignerResultType.PREDICTIONS: dict_series = self.dict_predictions + elif resulttype == AlignerResultType.EVALUATED_PREDICTIONS: + dict_series = self.dict_evaluated_predictions + prop_dictionary = self.dict_evaluated_predictions_properties else: raise (ValueError, "AlignerResultType unknown") if dict_series is None or dict_series == {}: @@ -781,8 +794,8 @@ def get_results_as_geojson( "Empty results: No calculated results to export." ) return {} - - prop_dictionary = defaultdict(dict) + if prop_dictionary is None: + prop_dictionary = defaultdict(dict) for theme_id, results_dict in dict_series.items(): for relevant_distance, process_results in results_dict.items(): @@ -791,7 +804,7 @@ def get_results_as_geojson( formula = self.get_brdr_formula(result) prop_dictionary[theme_id][relevant_distance] = { FORMULA_FIELD_NAME: json.dumps(formula) - } + }#TODO check if formula all available in properties return get_series_geojson_dict( dict_series, @@ -1222,6 +1235,105 @@ def _postprocess_preresult(self, preresult, geom_thematic,relevant_distance) -> "result_diff_min": geom_result_diff_min, } + def _evaluate(self, id_theme, geom_predicted): + """ + function that evaluates a predicted geometry and returns a properties-dictionary + """ + threshold_od_percentage = 1 + threshold_area = 5 + threshold_percentage = 1 + properties = { + FORMULA_FIELD_NAME: "", + EVALUATION_FIELD_NAME: Evaluation.TO_CHECK_4, + FULL_BASE_FIELD_NAME: None, + FULL_ACTUAL_FIELD_NAME: None, + OD_ALIKE_FIELD_NAME: None, + EQUAL_REFERENCE_FEATURES_FIELD_NAME: None, + DIFF_PERCENTAGE_FIELD_NAME: None, + DIFF_AREA_FIELD_NAME: None, + + } + actual_formula = self.get_brdr_formula(geom_predicted) + properties[FORMULA_FIELD_NAME] = json.dumps( + actual_formula + ) + base_formula = None + if ( + id_theme in self.dict_thematic_properties + and FORMULA_FIELD_NAME in self.dict_thematic_properties[id_theme] + ): + base_formula = self.dict_thematic_properties[id_theme][ + FORMULA_FIELD_NAME + ] + + if base_formula is None or actual_formula is None: + properties[EVALUATION_FIELD_NAME] = Evaluation.NO_PREDICTION_5 + return properties + properties[FULL_BASE_FIELD_NAME] = base_formula["full"] + properties[FULL_ACTUAL_FIELD_NAME] = actual_formula["full"] + od_alike = False + if base_formula["reference_od"] is None and actual_formula["reference_od"] is None: + od_alike = True + elif base_formula["reference_od"] is None or actual_formula["reference_od"] is None: + od_alike = False + elif ( + abs( + base_formula["reference_od"]["area"] + - actual_formula["reference_od"]["area"] + ) + * 100 + / base_formula["reference_od"]["area"] + ) < threshold_od_percentage: + od_alike = True + properties[OD_ALIKE_FIELD_NAME] = od_alike + + equal_reference_features = True + if ( + base_formula["reference_features"].keys() + == actual_formula["reference_features"].keys() + ): + equal_reference_features = True + max_diff_area_reference_feature = 0 + max_diff_percentage_reference_feature = 0 + for key in base_formula["reference_features"].keys(): + if ( + base_formula["reference_features"][key]["full"] + != actual_formula["reference_features"][key]["full"] + ): + equal_reference_features = False + + diff_area_reference_feature = abs( + base_formula["reference_features"][key]["area"] + - actual_formula["reference_features"][key]["area"] + ) + diff_percentage_reference_feature = ( + abs( + base_formula["reference_features"][key]["area"] + - actual_formula["reference_features"][key]["area"] + ) + * 100 + / base_formula["reference_features"][key]["area"] + ) + if diff_area_reference_feature > max_diff_area_reference_feature: + max_diff_area_reference_feature = diff_area_reference_feature + if diff_percentage_reference_feature > max_diff_percentage_reference_feature: + max_diff_percentage_reference_feature = diff_percentage_reference_feature + properties[EQUAL_REFERENCE_FEATURES_FIELD_NAME] = equal_reference_features + properties[DIFF_AREA_FIELD_NAME] = max_diff_area_reference_feature + properties[DIFF_PERCENTAGE_FIELD_NAME] = max_diff_percentage_reference_feature + #EVALUATION + if equal_reference_features and od_alike and base_formula["full"] and actual_formula["full"]: + properties[EVALUATION_FIELD_NAME] = Evaluation.EQUALITY_EQUAL_FORMULA_FULL_1 + elif equal_reference_features and od_alike and base_formula["full"] == actual_formula["full"]: + properties[EVALUATION_FIELD_NAME] = Evaluation.EQUALITY_EQUAL_FORMULA_2 + elif base_formula["full"] and actual_formula["full"] and od_alike: + properties[EVALUATION_FIELD_NAME] = Evaluation.EQUALITY_FULL_3 + #elif base_formula["full"] == actual_formula["full"] and od_alike:#TODO evaluate when not-full-parcels? + # properties[EVALUATION_FIELD_NAME] = Evaluation.EQUALITY_NON_FULL + else: + properties[EVALUATION_FIELD_NAME] = Evaluation.TO_CHECK_4 + return properties + @staticmethod def _add_multi_polygons_from_geom_to_array(geom: BaseGeometry, array): """ @@ -1424,7 +1536,7 @@ def _check_equality( and od_alike ): if base_formula["full"] and actual_formula["full"]: - return True, Evaluation.EQUALITY_FORMULA_GEOM_1 + return True, Evaluation.EQUALITY_EQUAL_FORMULA_FULL_1 equal_reference_features = True for key in base_formula["reference_features"].keys(): @@ -1456,103 +1568,8 @@ def _check_equality( if equal_reference_features: return True, Evaluation.EQUALITY_FORMULA_2 if base_formula["full"] and actual_formula["full"] and od_alike: - return True, Evaluation.EQUALITY_GEOM_3 + return True, Evaluation.EQUALITY_FULL_3 return False, Evaluation.NO_PREDICTION_5 -def _evaluate(self,id_theme,geom_predicted): - """ - function that evaluates a predicted geometry and returns a properties-dictionary - """ - threshold_od_percentage = 1 - threshold_area = 5 - threshold_percentage = 1 - properties = { - FORMULA_FIELD_NAME:"", - EVALUATION_FIELD_NAME: Evaluation.TO_CHECK_4, - FULL_BASE_FIELD_NAME: None, - FULL_ACTUAL_FIELD_NAME: None, - EQUAL_REFERENCE_FEATURES_FIELD_NAME: None, - DIFF_PERCENTAGE_FIELD_NAME: None, - DIFF_AREA_FIELD_NAME: None, - OD_ALIKE_FIELD_NAME: None, - } - actual_formula = self.get_brdr_formula(geom_predicted) - properties[FORMULA_FIELD_NAME] = json.dumps( - actual_formula - ) - base_formula = None - if ( - id_theme in self.dict_thematic_properties - and FORMULA_FIELD_NAME in self.dict_thematic_properties[id_theme] - ): - base_formula = self.dict_thematic_properties[id_theme][ - FORMULA_FIELD_NAME - ] - - if base_formula is None or actual_formula is None: - properties[EVALUATION_FIELD_NAME]= Evaluation.NO_PREDICTION_5 - return properties - properties[FULL_BASE_FIELD_NAME] = base_formula["full"] - properties[FULL_ACTUAL_FIELD_NAME] = actual_formula["full"] - od_alike = False - if base_formula["reference_od"] is None and actual_formula["reference_od"] is None: - od_alike = True - elif base_formula["reference_od"] is None or actual_formula["reference_od"] is None: - od_alike = False - elif ( - abs( - base_formula["reference_od"]["area"] - - actual_formula["reference_od"]["area"] - ) - * 100 - / base_formula["reference_od"]["area"] - ) < threshold_od_percentage: - od_alike = True - properties[OD_ALIKE_FIELD_NAME] = od_alike - - if ( - base_formula["reference_features"].keys() - == actual_formula["reference_features"].keys() - and od_alike - ): - equal_reference_features = True - - if base_formula["full"] and base_formula["full"]: - properties[EVALUATION_FIELD_NAME] = Evaluation.EQUALITY_FORMULA_GEOM_1 - - max_diff_area_reference_feature = 0 - max_diff_percentage_reference_feature = 0 - for key in base_formula["reference_features"].keys(): - - if ( - base_formula["reference_features"][key]["full"] - != actual_formula["reference_features"][key]["full"] - ): - equal_reference_features = False - - diff_area_reference_feature = abs( - base_formula["reference_features"][key]["area"] - - actual_formula["reference_features"][key]["area"] - ) - diff_percentage_reference_feature =( - abs( - base_formula["reference_features"][key]["area"] - - actual_formula["reference_features"][key]["area"] - ) - * 100 - / base_formula["reference_features"][key]["area"] - ) - if diff_area_reference_feature>max_diff_area_reference_feature: - max_diff_area_reference_feature=diff_area_reference_feature - if diff_percentage_reference_feature>max_diff_percentage_reference_feature: - max_diff_percentage_reference_feature=diff_percentage_reference_feature - properties[EQUAL_REFERENCE_FEATURES_FIELD_NAME] = equal_reference_features - properties[DIFF_AREA_FIELD_NAME] = max_diff_area_reference_feature - properties[DIFF_PERCENTAGE_FIELD_NAME] = max_diff_percentage_reference_feature - if equal_reference_features: - properties[EVALUATION_FIELD_NAME]= Evaluation.EQUALITY_FORMULA_2 - if base_formula["full"] and actual_formula["full"] and od_alike: - properties[EVALUATION_FIELD_NAME]= Evaluation.EQUALITY_GEOM_3 - return properties diff --git a/brdr/enums.py b/brdr/enums.py index 030a622..d5802b4 100644 --- a/brdr/enums.py +++ b/brdr/enums.py @@ -45,6 +45,7 @@ class AlignerResultType(str, Enum): """ PREDICTIONS = "predictions" + EVALUATED_PREDICTIONS = "evaluated_predictions" PROCESSRESULTS = "processresults" @@ -95,17 +96,17 @@ class Evaluation(str, Enum): """ Enum to evaluate an automatically updated geometry: - * EQUALITY_FORMULA_GEOM_1 = "equality_formula_geom_1" - * EQUALITY_FORMULA_2 = "equality_formula_2" - * EQUALITY_GEOM_3 = "equality_geom_3" - * TO_CHECK_4 = "to_check_4" - * NO_PREDICTION_5 = "no_prediction_5" - * NO_CHANGE_6 = "no_change_6" + EQUALITY_EQUAL_FORMULA_FULL_1 = "equality_equal_formula_full_1" + EQUALITY_EQUAL_FORMULA_2 = "equality_equal_formula_2" + EQUALITY_FULL_3 = "equality_full_3" + TO_CHECK_4 = "to_check_4" + NO_PREDICTION_5 = "no_prediction_5" + NO_CHANGE_6 = "no_change_6" """ - EQUALITY_FORMULA_GEOM_1 = "equality_formula_geom_1" - EQUALITY_FORMULA_2 = "equality_formula_2" - EQUALITY_GEOM_3 = "equality_geom_3" + EQUALITY_EQUAL_FORMULA_FULL_1 = "equality_equal_formula_full_1" + EQUALITY_EQUAL_FORMULA_2 = "equality_equal_formula_2" + EQUALITY_FULL_3 = "equality_full_3" TO_CHECK_4 = "to_check_4" NO_PREDICTION_5 = "no_prediction_5" NO_CHANGE_6 = "no_change_6" diff --git a/brdr/grb.py b/brdr/grb.py index 85e6201..a1c4c83 100644 --- a/brdr/grb.py +++ b/brdr/grb.py @@ -428,7 +428,7 @@ def update_to_actual_grb( actual_aligner.relevant_distances = ( np.arange(0, max_distance_for_actualisation * 100, 10, dtype=int) / 100 ) - dict_evaluated, prop_dictionary = actual_aligner.compare(affected = affected) + dict_evaluated, prop_dictionary = actual_aligner.compare(ids_to_compare= affected) return get_series_geojson_dict( dict_evaluated, diff --git a/examples/example_evaluate.py b/examples/example_evaluate.py index f9a7471..046592d 100644 --- a/examples/example_evaluate.py +++ b/examples/example_evaluate.py @@ -5,12 +5,11 @@ from brdr.aligner import Aligner from brdr.constants import FORMULA_FIELD_NAME, EVALUATION_FIELD_NAME -from brdr.enums import GRBType +from brdr.enums import GRBType, AlignerResultType from brdr.grb import GRBActualLoader from brdr.grb import GRBFiscalParcelLoader from brdr.grb import get_affected_by_grb_change from brdr.loader import DictLoader, GeoJsonFileLoader -from brdr.utils import get_series_geojson_dict thematic_dict = { "theme_id_1": from_wkt( @@ -56,16 +55,10 @@ GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner) ) actual_aligner.relevant_distances = np.arange(0, 200, 10, dtype=int) / 100 -dict_evaluated, prop_dictionary = actual_aligner.compare(affected=affected) +dict_evaluated, prop_dictionary = actual_aligner.compare(ids_to_compare=affected) -fc = get_series_geojson_dict( - dict_evaluated, - crs=actual_aligner.CRS, - id_field=actual_aligner.name_thematic_id, - series_prop_dict=prop_dictionary, -) +fc = actual_aligner.get_results_as_geojson(resulttype=AlignerResultType.EVALUATED_PREDICTIONS) print(fc["result"]) -fcs = actual_aligner.get_results_as_geojson(formula=True) for feature in fc["result"]["features"]: id = feature["properties"][actual_aligner.name_thematic_id] diff --git a/examples/example_evaluate_ao.py b/examples/example_evaluate_ao.py index ca1d209..21cb9de 100644 --- a/examples/example_evaluate_ao.py +++ b/examples/example_evaluate_ao.py @@ -4,7 +4,7 @@ from brdr.aligner import Aligner from brdr.constants import EVALUATION_FIELD_NAME, FORMULA_FIELD_NAME -from brdr.enums import GRBType +from brdr.enums import GRBType, AlignerResultType from brdr.grb import ( get_affected_by_grb_change, GRBFiscalParcelLoader, @@ -12,7 +12,6 @@ ) from brdr.loader import DictLoader from brdr.oe import OnroerendErfgoedLoader -from brdr.utils import get_series_geojson_dict base_aligner = Aligner() loader = OnroerendErfgoedLoader([120288,120108]) @@ -53,16 +52,10 @@ GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner) ) actual_aligner.relevant_distances = np.arange(0, 200, 10, dtype=int) / 100 -dict_evaluated, prop_dictionary = actual_aligner.compare(affected=affected) +dict_evaluated, prop_dictionary = actual_aligner.compare(ids_to_compare=affected) -fc = get_series_geojson_dict( - dict_evaluated, - crs=actual_aligner.CRS, - id_field=actual_aligner.name_thematic_id, - series_prop_dict=prop_dictionary, -) +fc = actual_aligner.get_results_as_geojson(resulttype=AlignerResultType.EVALUATED_PREDICTIONS) print(fc["result"]) -fcs = actual_aligner.get_results_as_geojson(formula=True) for feature in fc["result"]["features"]: id = feature["properties"][actual_aligner.name_thematic_id] diff --git a/tests/test_aligner.py b/tests/test_aligner.py index 3c971db..ca0bdbf 100644 --- a/tests/test_aligner.py +++ b/tests/test_aligner.py @@ -401,7 +401,7 @@ def test_evaluate(self): GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner) ) actual_aligner.relevant_distances = np.arange(0, 200, 10, dtype=int) / 100 - dict_evaluated, prop_dictionary = actual_aligner.compare(affected=affected) + dict_evaluated, prop_dictionary = actual_aligner.compare(ids_to_compare=affected) fc = get_series_geojson_dict( dict_evaluated, From 3db5746b0c76b001b98c39b56686f99ea2a8cced Mon Sep 17 00:00:00 2001 From: dieuska Date: Fri, 27 Sep 2024 17:38:46 +0200 Subject: [PATCH 12/26] #105 fixed that originalgeometries are returned when it would dissolve, to prevent it disappears --- brdr/aligner.py | 108 +++--------------------------------------------- 1 file changed, 5 insertions(+), 103 deletions(-) diff --git a/brdr/aligner.py b/brdr/aligner.py index c28c7ad..56e559d 100644 --- a/brdr/aligner.py +++ b/brdr/aligner.py @@ -519,102 +519,6 @@ def predictor( diffs_dict, ) - # def compare( - # self, - # threshold_area=5, - # threshold_percentage=1, - # dict_unchanged=None, - # ): - # """ - # Compares input-geometries (with formula) and evaluates these geometries: An attribute is added to evaluate and decide if new - # proposals can be used - # """ - # dict_series, dict_predictions, diffs = self.predictor(self.relevant_distances) - # if dict_unchanged is None: - # dict_unchanged = {} - # theme_ids = list(dict_series.keys()) - # dict_evaluated_result = {} - # prop_dictionary = {} - # # Fill the dictionary-structure with empty values - # for theme_id in theme_ids: - # dict_evaluated_result[theme_id] = {} - # prop_dictionary[theme_id] = {} - # for dist in dict_series[theme_id].keys(): - # prop_dictionary[theme_id][dist] = {} - # for theme_id in dict_unchanged.keys(): - # dict_evaluated_result[theme_id]={} - # prop_dictionary[theme_id] = {} - # - # for theme_id, dict_results in dict_predictions.items(): - # equality = False - # for dist in sorted(dict_results.keys()): - # if equality: - # break - # geomresult = dict_results[dist]["result"] - # actual_formula = self.get_brdr_formula(geomresult) - # prop_dictionary[theme_id][dist][FORMULA_FIELD_NAME] = json.dumps( - # actual_formula - # ) - # base_formula = None - # if ( - # theme_id in self.dict_thematic_properties - # and FORMULA_FIELD_NAME in self.dict_thematic_properties[theme_id] - # ): - # base_formula = self.dict_thematic_properties[theme_id][ - # FORMULA_FIELD_NAME - # ] - # equality, prop = _check_equality( - # base_formula, - # actual_formula, - # threshold_area, - # threshold_percentage, - # ) - # if equality: - # dict_evaluated_result[theme_id][dist] = dict_predictions[theme_id][ - # dist - # ] - # prop_dictionary[theme_id][dist][EVALUATION_FIELD_NAME] = prop - # break - # - # evaluated_theme_ids = [ - # theme_id for theme_id, value in dict_evaluated_result.items() if value != {} - # ] - # - # # fill where no equality is found/ The biggest predicted distance is returned as - # # proposal - # for theme_id in theme_ids: - # if theme_id not in evaluated_theme_ids: - # if len(dict_predictions[theme_id].keys()) == 0: - # result = dict_series[theme_id][0] - # dict_evaluated_result[theme_id][0] = result - # prop_dictionary[theme_id][0][FORMULA_FIELD_NAME] = json.dumps( - # self.get_brdr_formula(result["result"]) - # ) - # prop_dictionary[theme_id][0][ - # EVALUATION_FIELD_NAME - # ] = Evaluation.NO_PREDICTION_5 - # continue - # # Add all predicted features so they can be manually checked - # for dist in dict_predictions[theme_id].keys(): - # predicted_resultset = dict_predictions[theme_id][dist] - # dict_evaluated_result[theme_id][dist] = predicted_resultset - # prop_dictionary[theme_id][dist][FORMULA_FIELD_NAME] = json.dumps( - # self.get_brdr_formula(predicted_resultset["result"]) - # ) - # prop_dictionary[theme_id][dist][ - # EVALUATION_FIELD_NAME - # ] = Evaluation.TO_CHECK_4 - # - # for theme_id, geom in dict_unchanged.items(): - # dict_evaluated_result[theme_id] = {0: {"result": geom}} - # prop_dictionary[theme_id] = { - # 0: { - # #"result": geom, - # EVALUATION_FIELD_NAME: Evaluation.NO_CHANGE_6, - # FORMULA_FIELD_NAME: json.dumps(self.get_brdr_formula(geom)), - # } - # } - # return dict_evaluated_result, prop_dictionary def compare( self, @@ -799,13 +703,12 @@ def get_results_as_geojson( for theme_id, results_dict in dict_series.items(): for relevant_distance, process_results in results_dict.items(): - if formula: + if formula and not (theme_id in prop_dictionary and relevant_distance in prop_dictionary[theme_id] and FORMULA_FIELD_NAME in prop_dictionary[theme_id][relevant_distance]): result = process_results["result"] formula = self.get_brdr_formula(result) prop_dictionary[theme_id][relevant_distance] = { FORMULA_FIELD_NAME: json.dumps(formula) - }#TODO check if formula all available in properties - + } return get_series_geojson_dict( dict_series, crs=self.CRS, @@ -828,7 +731,6 @@ def get_input_as_geojson(self, inputtype=AlignerInputType.REFERENCE): property_id = self.name_reference_id else: raise (ValueError, "AlignerInputType unknown") - dict_properties if dict_to_geojson is None or dict_to_geojson == {}: self.logger.feedback_warning("Empty input: No input to export.") return {} @@ -1190,11 +1092,11 @@ def _postprocess_preresult(self, preresult, geom_thematic,relevant_distance) -> # Correction for empty preresults if geom_thematic_result.is_empty or geom_thematic_result is None: self.logger.feedback_warning( - "Empty result: -->resulting geometry = empty geometry" + "Empty result: -->resulting geometry = original geometry returned" ) - # geom_thematic_result = geom_thematic - geom_thematic_result = Polygon() #TODO : this results in disappearance of objects -> instead, return original geometry + geom_thematic_result = geom_thematic + # geom_thematic_result = Polygon() #If we return an empty geometry, the feature disappears, so we return the original geometry # group all initial multipolygons into a new resulting dictionary result.append(geom_thematic_result) From 7eb089175e58da3ae163c0f4766d792fb5c620bf Mon Sep 17 00:00:00 2001 From: dieuska Date: Mon, 30 Sep 2024 13:24:11 +0200 Subject: [PATCH 13/26] #83 constants to parameters --- brdr/aligner.py | 136 ++++++++++++++++++++-------------- brdr/constants.py | 40 +--------- brdr/geometry_utils.py | 25 +++---- brdr/grb.py | 8 +- examples/example_aligners.py | 4 +- examples/example_speedtest.py | 5 +- tests/test_aligner.py | 23 ++++-- 7 files changed, 122 insertions(+), 119 deletions(-) diff --git a/brdr/aligner.py b/brdr/aligner.py index 56e559d..b0ef17b 100644 --- a/brdr/aligner.py +++ b/brdr/aligner.py @@ -19,19 +19,14 @@ from shapely.geometry.base import BaseGeometry from brdr import __version__ +from brdr.constants import DEFAULT_CRS from brdr.constants import ( - BUFFER_MULTIPLICATION_FACTOR, LAST_VERSION_DATE, VERSION_DATE, DATE_FORMAT, - THRESHOLD_EXCLUSION_PERCENTAGE, - THRESHOLD_EXCLUSION_AREA, FORMULA_FIELD_NAME, EVALUATION_FIELD_NAME, FULL_BASE_FIELD_NAME, FULL_ACTUAL_FIELD_NAME, DIFF_PERCENTAGE_FIELD_NAME, DIFF_AREA_FIELD_NAME, OD_ALIKE_FIELD_NAME, EQUAL_REFERENCE_FEATURES_FIELD_NAME, ) -from brdr.constants import CORR_DISTANCE -from brdr.constants import DEFAULT_CRS -from brdr.constants import THRESHOLD_CIRCLE_RATIO from brdr.enums import ( OpenbaarDomeinStrategy, Evaluation, @@ -78,6 +73,12 @@ def __init__( threshold_overlap_percentage=50, od_strategy=OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE, crs=DEFAULT_CRS, + threshold_exclusion_area=0, + threshold_exclusion_percentage=0, + buffer_multiplication_factor=1.01, + threshold_circle_ratio=0.98, + correction_distance=0.01, + mitre_limit=10, area_limit=None, max_workers=None, ): @@ -113,8 +114,29 @@ def __init__( self.relevant_distances = relevant_distances self.od_strategy = od_strategy self.threshold_overlap_percentage = threshold_overlap_percentage + # Area in m² for excluding candidate reference when overlap(m²) is smaller than the + # threshold + self.threshold_exclusion_area = threshold_exclusion_area + # Percentage for excluding candidate reference when overlap(%) is smaller than the + # threshold + self.threshold_exclusion_percentage = threshold_exclusion_percentage self.area_limit = area_limit self.max_workers = max_workers + # Multiplication-factor used in OD-strategy 2 (SNAP-BOTH SIDED) when calculating + # OD-area to take into account + self.buffer_multiplication_factor=buffer_multiplication_factor + # Threshold-value to exclude circles getting processed (perfect circle = 1) based on + # POLSPY-POPPER algorithm + self.threshold_circle_ratio=threshold_circle_ratio + # Distance used in a pos_neg_buffer to remove slivers (technical correction) + self.correction_distance = correction_distance + # Buffer parameters: + # Distance to limit a buffered corner (MITER-join-style parameter) + # Explanation and examples: + # https://shapely.readthedocs.io/en/stable/reference/shapely.buffer.html + # https://postgis.net/docs/ST_Buffer.html + self.mitre_limit=mitre_limit + #quad_segments = 8 (by default in shapely) # PROCESSING DEFAULTS # thematic @@ -257,6 +279,8 @@ def process_geometry( False, buffer_distance, self.threshold_overlap_percentage, + self.threshold_exclusion_percentage, + self.threshold_exclusion_area,self.mitre_limit ) self.logger.feedback_debug("intersection calculated") preresult = self._add_multi_polygons_from_geom_to_array(geom, preresult) @@ -332,9 +356,9 @@ def process( futures = [] if dict_thematic is None: dict_thematic = self.dict_thematic - if self.multi_as_single_modus: dict_thematic = multipolygons_to_singles(dict_thematic) + if self.max_workers!=-1: with ThreadPoolExecutor(max_workers=self.max_workers) as executor:#max_workers=5 for key, geometry in dict_thematic.items(): @@ -502,7 +526,7 @@ def predictor( for rel_dist, processresults in dist_results.items(): predicted_geom = processresults["result"] if not _equal_geom_in_array( - predicted_geom, predicted_geoms_for_theme_id + predicted_geom, predicted_geoms_for_theme_id,self.correction_distance,self.mitre_limit ): dict_predictions_unique[theme_id][rel_dist] = processresults predicted_geoms_for_theme_id.append(processresults["result"]) @@ -660,9 +684,9 @@ def get_brdr_formula(self, geometry: BaseGeometry, with_geom=False): geom_od = buffer_pos( buffer_neg( safe_difference(geometry, safe_unary_union(intersected)), - CORR_DISTANCE, + self.correction_distance,mitre_limit=self.mitre_limit, ), - CORR_DISTANCE, + self.correction_distance,mitre_limit=self.mitre_limit, ) if geom_od is not None: area_od = round(geom_od.area, 2) @@ -846,11 +870,11 @@ def _calculate_intersection_between_geometry_and_od(self, geometry,relevant_dist # geom of OD geom_od = safe_difference(geometry, self._get_reference_union()) # only the relevant parts of OD - geom_od_neg_pos = buffer_neg_pos(geom_od, buffer_distance) + geom_od_neg_pos = buffer_neg_pos(geom_od, buffer_distance,mitre_limit=self.mitre_limit,) # geom_thematic_od = safe_intersection(geom_od_neg_pos,geom_od)# resulting # thematic OD geom_od_neg_pos_buffered = buffer_pos( - geom_od_neg_pos, buffer_distance + geom_od_neg_pos, buffer_distance,mitre_limit=self.mitre_limit, ) # include parts geom_thematic_od = safe_intersection( geom_od_neg_pos_buffered, geom_od @@ -908,7 +932,7 @@ def _calculate_intersection_between_geometry_and_od(self, geometry,relevant_dist # geom of OD geom_od = safe_difference(geometry, self._get_reference_union()) # only the relevant parts of OD - geom_od_neg_pos = buffer_neg_pos(geom_od, buffer_distance) + geom_od_neg_pos = buffer_neg_pos(geom_od, buffer_distance,mitre_limit=self.mitre_limit,) # resulting thematic OD geom_thematic_od = safe_intersection(geom_od_neg_pos, geom_od) elif self.od_strategy == OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE_VARIANT_2: @@ -935,16 +959,16 @@ def _od_full_area(self, geometry,relevant_distance): geom_theme_od = safe_difference(geometry, self._get_reference_union()) geom_theme_min_buffered = buffer_neg( buffer_pos( - buffer_neg(geometry, relevant_distance), - buffer_distance, + buffer_neg(geometry, relevant_distance,mitre_limit=self.mitre_limit,), + buffer_distance,mitre_limit=self.mitre_limit, ), - buffer_distance, + buffer_distance,mitre_limit=self.mitre_limit, ) geom_theme_od_clipped_min_buffered = safe_intersection( geom_theme_min_buffered, geom_theme_od ) geom_theme_od_min_clipped_plus_buffered = buffer_pos( - geom_theme_od_clipped_min_buffered, relevant_distance + geom_theme_od_clipped_min_buffered, relevant_distance,mitre_limit=self.mitre_limit, ) geom_theme_od_min_clipped_plus_buffered_clipped = safe_intersection( geom_theme_od_min_clipped_plus_buffered, geom_theme_od @@ -957,7 +981,7 @@ def _od_snap_all_side(self, geometry,relevant_distance): relevant_difference_array = [] relevant_intersection_array = [] geom_thematic_buffered = make_valid( - buffer_pos(geometry, BUFFER_MULTIPLICATION_FACTOR * relevant_distance) + buffer_pos(geometry, self.buffer_multiplication_factor * relevant_distance,mitre_limit=self.mitre_limit,) ) clip_ref_thematic_buffered = safe_intersection( self._get_reference_union(), geom_thematic_buffered @@ -980,6 +1004,9 @@ def _od_snap_all_side(self, geometry,relevant_distance): True, buffer_distance, self.threshold_overlap_percentage, + self.threshold_exclusion_percentage, + self.threshold_exclusion_area, + self.mitre_limit ) relevant_intersection_array = self._add_multi_polygons_from_geom_to_array( geom_relevant_intersection, [] @@ -1031,7 +1058,7 @@ def _postprocess_preresult(self, preresult, geom_thematic,relevant_distance) -> # if a circle: (Polsby-popper score) if ( 4 * pi * (geom_thematic.area / (geom_thematic.length**2)) - > THRESHOLD_CIRCLE_RATIO + > self.threshold_circle_ratio ): self.logger.feedback_debug( "Circle: -->resulting geometry = original geometry" @@ -1045,10 +1072,10 @@ def _postprocess_preresult(self, preresult, geom_thematic,relevant_distance) -> # Corrections for areas that differ more than the relevant distance geom_thematic_dissolved = buffer_pos( buffer_neg( - buffer_pos(geom_preresult, CORR_DISTANCE), - 2 * CORR_DISTANCE, + buffer_pos(geom_preresult, self.correction_distance,mitre_limit=self.mitre_limit,), + 2 * self.correction_distance,mitre_limit=self.mitre_limit, ), - CORR_DISTANCE, + self.correction_distance,mitre_limit=self.mitre_limit, ) # geom_symdiff = self._safe_symmetric_difference(geom_thematic, # geom_thematic_dissolved) @@ -1058,22 +1085,22 @@ def _postprocess_preresult(self, preresult, geom_thematic,relevant_distance) -> geom_thematic_dissolved, safe_intersection( geom_diff_delete, - buffer_neg_pos(geom_diff_delete, buffer_distance), + buffer_neg_pos(geom_diff_delete, buffer_distance,mitre_limit=self.mitre_limit,), ), ) geom_diff_removed_added = safe_union( geom_diff_removed, safe_intersection( geom_diff_add, - buffer_neg_pos(geom_diff_add, buffer_distance), + buffer_neg_pos(geom_diff_add, buffer_distance,mitre_limit=self.mitre_limit,), ), ) geom_thematic_preresult = buffer_pos( buffer_neg( - buffer_pos(geom_diff_removed_added, CORR_DISTANCE), - 2 * CORR_DISTANCE, + buffer_pos(geom_diff_removed_added, self.correction_distance,mitre_limit=self.mitre_limit,), + 2 * self.correction_distance,mitre_limit=self.mitre_limit, ), - CORR_DISTANCE, + self.correction_distance,mitre_limit=self.mitre_limit, ) # Correction for Inner holes(donuts) / multipolygons # fill and remove gaps @@ -1082,10 +1109,10 @@ def _postprocess_preresult(self, preresult, geom_thematic,relevant_distance) -> ) geom_thematic_result = buffer_pos( buffer_neg( - buffer_pos(geom_thematic_cleaned_holes, CORR_DISTANCE), - 2 * CORR_DISTANCE, + buffer_pos(geom_thematic_cleaned_holes, self.correction_distance,mitre_limit=self.mitre_limit,), + 2 * self.correction_distance,mitre_limit=self.mitre_limit, ), - CORR_DISTANCE, + self.correction_distance,mitre_limit=self.mitre_limit, ) geom_thematic_result = make_valid(remove_repeated_points(geom_thematic_result)) @@ -1108,24 +1135,24 @@ def _postprocess_preresult(self, preresult, geom_thematic,relevant_distance) -> # remove 'very small' differences (smaller than the correction distance) geom_result_diff = buffer_pos( buffer_neg( - safe_symmetric_difference(geom_thematic_result, geom_thematic), - CORR_DISTANCE, + safe_symmetric_difference(geom_thematic_result, geom_thematic,), + self.correction_distance,mitre_limit=self.mitre_limit, ), - CORR_DISTANCE, + self.correction_distance,mitre_limit=self.mitre_limit, ) geom_result_diff_plus = buffer_pos( buffer_neg( - safe_difference(geom_thematic_result, geom_thematic), - CORR_DISTANCE, + safe_difference(geom_thematic_result, geom_thematic,), + self.correction_distance,mitre_limit=self.mitre_limit, ), - CORR_DISTANCE, + self.correction_distance,mitre_limit=self.mitre_limit, ) geom_result_diff_min = buffer_pos( buffer_neg( - safe_difference(geom_thematic, geom_thematic_result), - CORR_DISTANCE, + safe_difference(geom_thematic, geom_thematic_result,), + self.correction_distance,mitre_limit=self.mitre_limit, ), - CORR_DISTANCE, + self.correction_distance,mitre_limit=self.mitre_limit, ) # geom_result_diff_plus = safe_difference(geom_thematic_result, geom_thematic) # geom_result_diff_min = safe_difference(geom_thematic, geom_thematic_result) @@ -1142,8 +1169,8 @@ def _evaluate(self, id_theme, geom_predicted): function that evaluates a predicted geometry and returns a properties-dictionary """ threshold_od_percentage = 1 - threshold_area = 5 - threshold_percentage = 1 + #threshold_area = 5 + #threshold_percentage = 1 properties = { FORMULA_FIELD_NAME: "", EVALUATION_FIELD_NAME: Evaluation.TO_CHECK_4, @@ -1271,8 +1298,9 @@ def _calculate_geom_by_intersection_and_reference( is_openbaar_domein, buffer_distance, threshold_overlap_percentage, - threshold_exclusion_percentage=THRESHOLD_EXCLUSION_PERCENTAGE, - threshold_exclusion_area=THRESHOLD_EXCLUSION_AREA, + threshold_exclusion_percentage, + threshold_exclusion_area, + mitre_limit, ): """ Calculates the geometry based on intersection and reference geometries. @@ -1322,8 +1350,8 @@ def _calculate_geom_by_intersection_and_reference( return Polygon(), Polygon(), Polygon() geom_difference = safe_difference(geom_reference, geom_intersection) - geom_relevant_intersection = buffer_neg(geom_intersection, buffer_distance) - geom_relevant_difference = buffer_neg(geom_difference, buffer_distance) + geom_relevant_intersection = buffer_neg(geom_intersection, buffer_distance,mitre_limit=mitre_limit,) + geom_relevant_difference = buffer_neg(geom_difference, buffer_distance,mitre_limit=mitre_limit,) if ( not geom_relevant_intersection.is_empty and not geom_relevant_difference.is_empty @@ -1335,15 +1363,15 @@ def _calculate_geom_by_intersection_and_reference( geom_reference, safe_intersection( geom_difference, - buffer_neg_pos(geom_difference, buffer_distance), + buffer_neg_pos(geom_difference, buffer_distance,mitre_limit=mitre_limit,), ), ), ) geom = safe_intersection( geom_x, buffer_pos( - buffer_neg_pos(geom_x, buffer_distance), - buffer_distance, + buffer_neg_pos(geom_x, buffer_distance,mitre_limit=mitre_limit,), + buffer_distance,mitre_limit=mitre_limit, ), ) # when calculating for OD, we create a 'virtual parcel'. When calculating this @@ -1352,7 +1380,7 @@ def _calculate_geom_by_intersection_and_reference( # in the result. The function below tries to exclude these non-logical parts. # see eo_id 206363 with relevant distance=0.2m and SNAP_ALL_SIDE if is_openbaar_domein: - geom = _get_relevant_polygons_from_geom(geom, buffer_distance) + geom = _get_relevant_polygons_from_geom(geom, buffer_distance,mitre_limit) elif not geom_relevant_intersection.is_empty and geom_relevant_difference.is_empty: geom = geom_reference elif geom_relevant_intersection.is_empty and not geom_relevant_difference.is_empty: @@ -1372,7 +1400,7 @@ def _calculate_geom_by_intersection_and_reference( return geom, geom_relevant_intersection, geom_relevant_difference -def _get_relevant_polygons_from_geom(geometry: BaseGeometry, buffer_distance: float): +def _get_relevant_polygons_from_geom(geometry: BaseGeometry, buffer_distance: float,mitre_limit): """ Get only the relevant parts (polygon) from a geometry. Points, Lines and Polygons smaller than relevant distance are excluded from the @@ -1390,13 +1418,13 @@ def _get_relevant_polygons_from_geom(geometry: BaseGeometry, buffer_distance: fl # Ensure each sub-geometry is valid. g = make_valid(g) if str(g.geom_type) in ["Polygon", "MultiPolygon"]: - relevant_geom = buffer_neg(g, buffer_distance) + relevant_geom = buffer_neg(g, buffer_distance,mitre_limit=mitre_limit,) if relevant_geom is not None and not relevant_geom.is_empty: array.append(g) return safe_unary_union(array) -def _equal_geom_in_array(geom, geom_array): +def _equal_geom_in_array(geom, geom_array,correction_distance,mitre_limit): """ Check if a predicted geometry is equal to other predicted geometries in a list. Equality is defined as there is the symmetrical difference is smaller than the CORRECTION DISTANCE @@ -1404,7 +1432,7 @@ def _equal_geom_in_array(geom, geom_array): """ for g in geom_array: # if safe_equals(geom,g): - if buffer_neg(safe_symmetric_difference(geom, g), CORR_DISTANCE).is_empty: + if buffer_neg(safe_symmetric_difference(geom, g), correction_distance,mitre_limit=mitre_limit,).is_empty: return True return False @@ -1468,7 +1496,7 @@ def _check_equality( ): equal_reference_features = False if equal_reference_features: - return True, Evaluation.EQUALITY_FORMULA_2 + return True, Evaluation.EQUALITY_EQUAL_FORMULA_2 if base_formula["full"] and actual_formula["full"] and od_alike: return True, Evaluation.EQUALITY_FULL_3 return False, Evaluation.NO_PREDICTION_5 diff --git a/brdr/constants.py b/brdr/constants.py index 7a37a1e..10f5b10 100644 --- a/brdr/constants.py +++ b/brdr/constants.py @@ -1,37 +1,6 @@ -# Thresholds -# Area in m² for excluding candidate reference when overlap(m²) is smaller than the -# threshold -THRESHOLD_EXCLUSION_AREA = 0 -# Percentage for excluding candidate reference when overlap(%) is smaller than the -# threshold -THRESHOLD_EXCLUSION_PERCENTAGE = 0 - -# Buffer parameters: -# Explanation and examples: -# https://shapely.readthedocs.io/en/stable/reference/shapely.buffer.html -# https://postgis.net/docs/ST_Buffer.html -# Distance to limit a buffered corner (MITER-join-style parameter) -MITRE_LIMIT = 10 -# Used in buffer-operations to define a quarter circle -QUAD_SEGMENTS = 5 - -# Correction-parameters (technical) - -# Multiplication-factor used in OD-strategy 2 (SNAP-BOTH SIDED) when calculating -# OD-area to take into account -BUFFER_MULTIPLICATION_FACTOR = 1.01 -# Threshold-value to exclude circles getting processed (perfect circle = 1) based on -# POLSPY-POPPER algorithm -THRESHOLD_CIRCLE_RATIO = 0.98 -# Distance used in a pos_neg_buffer to remove slivers (technical correction) -CORR_DISTANCE = 0.01 - # Download-settings: when extracting features by URL -# max buffer around thematic geometry to download reference parcels -MAX_REFERENCE_BUFFER = 10 # Limit used when extracting features by URL, using the feature API (f.e. from GRB) DOWNLOAD_LIMIT = 10000 - # default CRS: DEFAULT_CRS = "EPSG:31370" # BelgianLambert72 @@ -52,24 +21,21 @@ RELEVANT_DISTANCE_FIELD_NAME = PREFIX_FIELDNAME + "relevant_distance" LAST_VERSION_DATE = "last_version_date" VERSION_DATE = "version_date" - DATE_FORMAT = "%Y-%m-%d" + # GRB_CONSTANTS +# max buffer (m) around thematic geometry to download reference parcels +GRB_MAX_REFERENCE_BUFFER = 10 # URL of the OGC feature API of actual GRB to extract collections GRB_FEATURE_URL = "https://geo.api.vlaanderen.be/GRB/ogc/features/collections" - # URL of the OGC feature API of GRB fiscal parcels (situation of 1st of January) to # extract collections GRB_FISCAL_PARCELS_URL = "https://geo.api.vlaanderen.be/Adpf/ogc/features/collections" - # Property-name of version_date GRB_VERSION_DATE = "VERSDATUM" - # Property-name of id of GRB-parcels GRB_PARCEL_ID = "CAPAKEY" - # Property-name of id of GRB-parcels GRB_BUILDING_ID = "OIDN" - # Property-name of id of GRB-parcels GRB_KNW_ID = "OIDN" diff --git a/brdr/geometry_utils.py b/brdr/geometry_utils.py index aea7f53..2840522 100644 --- a/brdr/geometry_utils.py +++ b/brdr/geometry_utils.py @@ -22,11 +22,8 @@ from shapely.geometry.base import BaseGeometry from shapely.prepared import prep -from brdr.constants import MITRE_LIMIT -from brdr.constants import QUAD_SEGMENTS - -def buffer_neg_pos(geometry, buffer_value): +def buffer_neg_pos(geometry, buffer_value,mitre_limit=5): """ Computes two buffers accordingly: one with a negative buffer value and another with a positive buffer value. This function can be used the check where relevant areas @@ -59,18 +56,18 @@ def buffer_neg_pos(geometry, buffer_value): buffer( geometry, -buffer_value, - quad_segs=QUAD_SEGMENTS, + #quad_segs=QUAD_SEGMENTS, join_style="mitre", - mitre_limit=MITRE_LIMIT, + mitre_limit=mitre_limit, ), buffer_value, - quad_segs=QUAD_SEGMENTS, + #quad_segs=QUAD_SEGMENTS, join_style="mitre", - mitre_limit=MITRE_LIMIT, + mitre_limit=mitre_limit, ) -def buffer_neg(geometry, buffer_value): +def buffer_neg(geometry, buffer_value,mitre_limit=5): """ Computes the negative buffer of a given geometric object. @@ -97,13 +94,13 @@ def buffer_neg(geometry, buffer_value): return buffer( geometry, -buffer_value, - quad_segs=QUAD_SEGMENTS, + #quad_segs=QUAD_SEGMENTS, join_style="mitre", - mitre_limit=MITRE_LIMIT, + mitre_limit=mitre_limit, ) -def buffer_pos(geometry, buffer_value): +def buffer_pos(geometry, buffer_value,mitre_limit=5): """ Computes the positive buffer of a given geometric object. @@ -130,9 +127,9 @@ def buffer_pos(geometry, buffer_value): return buffer( geometry, buffer_value, - quad_segs=QUAD_SEGMENTS, + #quad_segs=QUAD_SEGMENTS, join_style="mitre", - mitre_limit=MITRE_LIMIT, + mitre_limit=mitre_limit, ) diff --git a/brdr/grb.py b/brdr/grb.py index a1c4c83..9278699 100644 --- a/brdr/grb.py +++ b/brdr/grb.py @@ -21,9 +21,9 @@ from brdr.constants import GRB_FEATURE_URL from brdr.constants import GRB_FISCAL_PARCELS_URL from brdr.constants import GRB_KNW_ID +from brdr.constants import GRB_MAX_REFERENCE_BUFFER from brdr.constants import GRB_PARCEL_ID from brdr.constants import GRB_VERSION_DATE -from brdr.constants import MAX_REFERENCE_BUFFER from brdr.enums import GRBType from brdr.geometry_utils import buffer_pos, safe_intersection, safe_unary_union from brdr.geometry_utils import create_donut @@ -450,7 +450,7 @@ def __init__(self, grb_type: GRBType, aligner, partition: int = 1000): def load_data(self): if not self.aligner.dict_thematic: raise ValueError("Thematic data not loaded") - geom_union = buffer_pos(self.aligner.get_thematic_union(), MAX_REFERENCE_BUFFER) + geom_union = buffer_pos(self.aligner.get_thematic_union(), GRB_MAX_REFERENCE_BUFFER) collection, id_property = get_collection_grb_actual( grb_type=self.grb_type, geometry=geom_union, @@ -479,7 +479,7 @@ def __init__(self, year: str, aligner, partition=1000): def load_data(self): if not self.aligner.dict_thematic: raise ValueError("Thematic data not loaded") - geom_union = buffer_pos(self.aligner.get_thematic_union(), MAX_REFERENCE_BUFFER) + geom_union = buffer_pos(self.aligner.get_thematic_union(), GRB_MAX_REFERENCE_BUFFER) collection = get_collection_grb_fiscal_parcels( year=self.year, geometry=geom_union, @@ -515,7 +515,7 @@ def __init__(self, date, aligner, partition=1000): def load_data(self): if not self.aligner.dict_thematic: raise ValueError("Thematic data not loaded") - geom_union = buffer_pos(self.aligner.get_thematic_union(), MAX_REFERENCE_BUFFER) + geom_union = buffer_pos(self.aligner.get_thematic_union(), GRB_MAX_REFERENCE_BUFFER) collection = get_collection_grb_parcels_by_date( date=self.date, geometry=geom_union, diff --git a/examples/example_aligners.py b/examples/example_aligners.py index a9523e5..e316008 100644 --- a/examples/example_aligners.py +++ b/examples/example_aligners.py @@ -40,7 +40,9 @@ # Example how to use a series (for histogram) series = [0.1, 0.2, 0.3, 0.4, 0.5, 1, 2] - dict_series = aligner.process(series, 2, 50) + dict_series = aligner.process( + relevant_distances=series, od_strategy=2, threshold_overlap_percentage=50 + ) resulting_areas = diffs_from_dict_series(dict_series, aligner.dict_thematic) plot_series(series, resulting_areas) diff --git a/examples/example_speedtest.py b/examples/example_speedtest.py index e2cf994..973d191 100644 --- a/examples/example_speedtest.py +++ b/examples/example_speedtest.py @@ -7,7 +7,7 @@ def main(): # Initiate brdr - aligner = Aligner(relevant_distance=2,max_workers=None) + aligner = Aligner(relevant_distance=2, max_workers=None) iterations = 10 aligner.multi_as_single_modus = True # Load local thematic data and reference data @@ -43,6 +43,7 @@ def main(): print("Median: " + str(statistics.median(times))) print("Stdv: " + str(statistics.stdev(times))) + # #BEFORE REFACTORING dict_series # duration: [25.652311, 27.894154, 19.641618, 19.929254, 44.754033, 25.218422, 23.167992, 18.649832, 22.899336, 52.108296] # Min: 18.649832 @@ -59,5 +60,5 @@ def main(): # Median: 17.8996155 # Stdv: 1.504459449440969 -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/tests/test_aligner.py b/tests/test_aligner.py index ca0bdbf..8b3360e 100644 --- a/tests/test_aligner.py +++ b/tests/test_aligner.py @@ -374,9 +374,13 @@ def test_evaluate(self): thematic_dict_formula = {} thematic_dict_result = {} for key in base_process_result: - thematic_dict_result[key] = base_process_result[key][relevant_distance]["result"] + thematic_dict_result[key] = base_process_result[key][relevant_distance][ + "result" + ] thematic_dict_formula[key] = { - FORMULA_FIELD_NAME: base_aligner.get_brdr_formula(thematic_dict_result[key]) + FORMULA_FIELD_NAME: base_aligner.get_brdr_formula( + thematic_dict_result[key] + ) } print(key + ": " + thematic_dict_result[key].wkt) print(key + ": " + str(thematic_dict_formula[key])) @@ -395,13 +399,20 @@ def test_evaluate(self): print("Affected_IDs: " + str(affected)) actual_aligner = Aligner() actual_aligner.load_thematic_data( - DictLoader(data_dict=thematic_dict_result, data_dict_properties=thematic_dict_formula) + DictLoader( + data_dict=thematic_dict_result, + data_dict_properties=thematic_dict_formula, + ) ) actual_aligner.load_reference_data( - GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner) + GRBActualLoader( + grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner + ) ) actual_aligner.relevant_distances = np.arange(0, 200, 10, dtype=int) / 100 - dict_evaluated, prop_dictionary = actual_aligner.compare(ids_to_compare=affected) + dict_evaluated, prop_dictionary = actual_aligner.compare( + ids_to_compare=affected + ) fc = get_series_geojson_dict( dict_evaluated, @@ -412,8 +423,6 @@ def test_evaluate(self): print(fc["result"]) fcs = actual_aligner.get_results_as_geojson(formula=True) - - def test_fully_aligned_geojson_output(self): aligned_shape = from_wkt( "MultiPolygon (((173463.11530961000244133 174423.83310307000647299, " From 695988c9ac370a960dd5b495da11c5288a5ea18b Mon Sep 17 00:00:00 2001 From: dieuska Date: Mon, 30 Sep 2024 16:31:51 +0200 Subject: [PATCH 14/26] #103 remarks added --- brdr/aligner.py | 30 ++++++++++++++++-------------- brdr/constants.py | 1 + brdr/typings.py | 1 + brdr/utils.py | 34 ++++++++++++++++++++++------------ tests/test_aligner.py | 2 +- tests/test_examples.py | 4 +++- 6 files changed, 44 insertions(+), 28 deletions(-) diff --git a/brdr/aligner.py b/brdr/aligner.py index b0ef17b..54255b2 100644 --- a/brdr/aligner.py +++ b/brdr/aligner.py @@ -307,7 +307,7 @@ def process_geometry( # make a unary union for each key value in the result dict for key in ProcessResult.__annotations__: geometry = result_dict.get(key, Polygon()) # noqa - if not geometry.is_empty: + if isinstance(geometry,BaseGeometry) and not geometry.is_empty: geometry = safe_unary_union(geometry) result_dict[key] = geometry # noqa @@ -382,8 +382,8 @@ def process( self.logger.feedback_debug ("waiting all started RD calculations") wait(futures) for id_theme,dict_dist in dict_series_queue.items(): - for reldist,future in dict_dist.items(): - dict_series[id_theme][reldist] = future.result() + for relevant_distance,future in dict_dist.items(): + dict_series[id_theme][relevant_distance] = future.result() else: for key, geometry in dict_thematic.items(): self.logger.feedback_info( @@ -392,10 +392,9 @@ def process( dict_series[key] = {} for relevant_distance in self.relevant_distances: try: - self.relevant_distance = relevant_distance processed_result = self.process_geometry( geometry, - self.relevant_distance, + relevant_distance, od_strategy, threshold_overlap_percentage, ) @@ -403,7 +402,7 @@ def process( self.logger.feedback_warning(str(e)) processed_result = None - dict_series[key][self.relevant_distance] = processed_result + dict_series[key][relevant_distance] = processed_result if self.multi_as_single_modus: dict_series = merge_process_results(dict_series) @@ -1047,6 +1046,7 @@ def _postprocess_preresult(self, preresult, geom_thematic,relevant_distance) -> geometry """ # Process array + remark = "" buffer_distance = relevant_distance/2 result = [] geom_preresult = safe_unary_union(preresult) @@ -1060,14 +1060,15 @@ def _postprocess_preresult(self, preresult, geom_thematic,relevant_distance) -> 4 * pi * (geom_thematic.area / (geom_thematic.length**2)) > self.threshold_circle_ratio ): - self.logger.feedback_debug( - "Circle: -->resulting geometry = original geometry" - ) - return {"result": geom_thematic} + remark = "Circle detected: -->resulting geometry = original geometry" + self.logger.feedback_debug(remark) + return {"result": geom_thematic,"remark": remark} # Correction for unchanged geometries if geom_preresult == geom_thematic: - return {"result": geom_thematic} + remark = "Unchanged geometry: -->resulting geometry = original geometry" + self.logger.feedback_debug(remark) + return {"result": geom_thematic,"remark": remark} # Corrections for areas that differ more than the relevant distance geom_thematic_dissolved = buffer_pos( @@ -1118,9 +1119,8 @@ def _postprocess_preresult(self, preresult, geom_thematic,relevant_distance) -> # Correction for empty preresults if geom_thematic_result.is_empty or geom_thematic_result is None: - self.logger.feedback_warning( - "Empty result: -->resulting geometry = original geometry returned" - ) + remark = "Calculated empty result: -->original geometry returned" + self.logger.feedback_warning(remark) geom_thematic_result = geom_thematic # geom_thematic_result = Polygon() #If we return an empty geometry, the feature disappears, so we return the original geometry @@ -1157,11 +1157,13 @@ def _postprocess_preresult(self, preresult, geom_thematic,relevant_distance) -> # geom_result_diff_plus = safe_difference(geom_thematic_result, geom_thematic) # geom_result_diff_min = safe_difference(geom_thematic, geom_thematic_result) + return { "result": geom_thematic_result, "result_diff": geom_result_diff, "result_diff_plus": geom_result_diff_plus, "result_diff_min": geom_result_diff_min, + "remark": remark } def _evaluate(self, id_theme, geom_predicted): diff --git a/brdr/constants.py b/brdr/constants.py index 10f5b10..820f437 100644 --- a/brdr/constants.py +++ b/brdr/constants.py @@ -19,6 +19,7 @@ NR_CALCULATION_FIELD_NAME = PREFIX_FIELDNAME + "nr_calculations" RELEVANT_DISTANCE_FIELD_NAME = PREFIX_FIELDNAME + "relevant_distance" +REMARK_FIELD_NAME = PREFIX_FIELDNAME + "remark" LAST_VERSION_DATE = "last_version_date" VERSION_DATE = "version_date" DATE_FORMAT = "%Y-%m-%d" diff --git a/brdr/typings.py b/brdr/typings.py index 22556f0..5ad2c05 100644 --- a/brdr/typings.py +++ b/brdr/typings.py @@ -37,3 +37,4 @@ class ProcessResult(TypedDict, total=False): result_diff_min: BaseGeometry result_relevant_intersection: BaseGeometry result_relevant_diff: BaseGeometry + remark: str diff --git a/brdr/utils.py b/brdr/utils.py index 05b47a7..505e1d6 100644 --- a/brdr/utils.py +++ b/brdr/utils.py @@ -19,7 +19,7 @@ DEFAULT_CRS, DOWNLOAD_LIMIT, RELEVANT_DISTANCE_FIELD_NAME, - NR_CALCULATION_FIELD_NAME, + NR_CALCULATION_FIELD_NAME, REMARK_FIELD_NAME, ) from brdr.enums import DiffMetric from brdr.geometry_utils import get_partitions, get_bbox @@ -46,8 +46,12 @@ def get_series_geojson_dict( properties[id_field] = theme_id properties[NR_CALCULATION_FIELD_NAME] = nr_calculations properties[RELEVANT_DISTANCE_FIELD_NAME] = relative_distance + if "remark" in process_result: + properties[REMARK_FIELD_NAME] = process_result["remark"] for results_type, geom in process_result.items(): + if not isinstance(geom,BaseGeometry): + continue if results_type not in features_list_dict: features_list_dict[results_type] = [] @@ -493,13 +497,13 @@ def merge_process_results( result_dict: dict[str, dict[float, ProcessResult]] ) -> dict[str, dict[float, ProcessResult]]: """ - Merges geometries in a dictionary from multiple themes into a single theme. + Merges processresults in a dictionary from multiple themeIDs into a single themeID. Args: result_dict (dict): A dictionary where keys are theme IDs and values are process results - Returns: dict: A new dictionary with merged geometries, where keys are global - theme IDs and values are merged geometries. + Returns: dict: A new dictionary with merged geometries and remarks (processresults, where keys are global + theme IDs and values are merged geometries and remarks. """ grouped_results: dict[str, dict[float, ProcessResult]] = {} @@ -511,13 +515,19 @@ def merge_process_results( else: for rel_dist, process_result in dict_results.items(): for key in process_result: - geom: BaseGeometry = process_result[key] # noqa - if geom.is_empty or geom is None: + value = process_result[key] # noqa + if isinstance(value,str): + existing_remark: str = grouped_results[id_theme_global][rel_dist][ key] # noqa + grouped_results[id_theme_global][rel_dist][key] = existing_remark + '|' + str(value) continue - existing: BaseGeometry = grouped_results[id_theme_global][rel_dist][ - key - ] # noqa - grouped_results[id_theme_global][rel_dist][key] = unary_union( - [existing, geom] - ) # noqa + elif isinstance(value,BaseGeometry): + geom = value + if geom.is_empty or geom is None: + continue + existing: BaseGeometry = grouped_results[id_theme_global][rel_dist][ + key + ] # noqa + grouped_results[id_theme_global][rel_dist][key] = unary_union( + [existing, geom] + ) # noqa return grouped_results diff --git a/tests/test_aligner.py b/tests/test_aligner.py index 8b3360e..ca518e8 100644 --- a/tests/test_aligner.py +++ b/tests/test_aligner.py @@ -240,7 +240,7 @@ def test_all_od_strategies(self): od_strategy=od_strategy, threshold_overlap_percentage=50, ) - self.assertEqual(len(process_result["theme_id_1"][relevant_distance]), 6) + self.assertEqual(len(process_result["theme_id_1"][relevant_distance]), 7) def test_process_interior_ring(self): thematic_dict = { diff --git a/tests/test_examples.py b/tests/test_examples.py index ea63c84..1289651 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -191,7 +191,9 @@ def test_example_wanted_changes(self): # Example how to use a series (for histogram) series = np.arange(0, 300, 10, dtype=int) / 100 - dict_series = aligner.process(relevant_distances=series, od_strategy=4, threshold_overlap_percentage=50) + dict_series = aligner.process( + relevant_distances=series, od_strategy=4, threshold_overlap_percentage=50 + ) resulting_areas = diffs_from_dict_series(dict_series, aligner.dict_thematic) for key in resulting_areas: if len(resulting_areas[key]) == len(series): From 7b7a3e15ac61a7743d677ee9f05b44b35071151a Mon Sep 17 00:00:00 2001 From: dieuska Date: Tue, 1 Oct 2024 14:42:22 +0200 Subject: [PATCH 15/26] #110 adaptions so code can handle all type of IDs (integers, strings) --- brdr/aligner.py | 57 ++++++++++++---------- brdr/constants.py | 3 +- brdr/grb.py | 44 +++++++---------- brdr/utils.py | 12 +++-- examples/example_evaluate.py | 7 +-- examples/example_evaluate_ao.py | 7 +-- examples/example_integer_id.py | 39 +++++++++++++++ examples/example_multipolygon.py | 2 +- examples/example_parcel_change_detector.py | 4 +- examples/example_parcel_vs_building.py | 4 +- examples/example_update_to_actual_grb.py | 7 +-- tests/test_aligner.py | 4 +- tests/test_grb.py | 4 +- tests/test_utils.py | 11 +++-- 14 files changed, 127 insertions(+), 78 deletions(-) create mode 100644 examples/example_integer_id.py diff --git a/brdr/aligner.py b/brdr/aligner.py index 54255b2..000496d 100644 --- a/brdr/aligner.py +++ b/brdr/aligner.py @@ -19,12 +19,13 @@ from shapely.geometry.base import BaseGeometry from brdr import __version__ -from brdr.constants import DEFAULT_CRS +from brdr.constants import DEFAULT_CRS, BASE_FORMULA_FIELD_NAME from brdr.constants import ( LAST_VERSION_DATE, VERSION_DATE, DATE_FORMAT, - FORMULA_FIELD_NAME, EVALUATION_FIELD_NAME, FULL_BASE_FIELD_NAME, FULL_ACTUAL_FIELD_NAME, DIFF_PERCENTAGE_FIELD_NAME, + NEW_FORMULA_FIELD_NAME, EVALUATION_FIELD_NAME, FULL_BASE_FIELD_NAME, FULL_ACTUAL_FIELD_NAME, + DIFF_PERCENTAGE_FIELD_NAME, DIFF_AREA_FIELD_NAME, OD_ALIKE_FIELD_NAME, EQUAL_REFERENCE_FEATURES_FIELD_NAME, ) from brdr.enums import ( @@ -143,11 +144,11 @@ def __init__( # name of the identifier-field of the thematic data (id has to be unique) self.name_thematic_id = "theme_identifier" # dictionary to store all thematic geometries to handle - self.dict_thematic: dict[str, BaseGeometry] = {} + self.dict_thematic: dict[any, BaseGeometry] = {} # dictionary to store properties of the reference-features (optional) - self.dict_thematic_properties: dict[str, dict] = {} + self.dict_thematic_properties: dict[any, dict] = {} # Dict to store source-information of the thematic dictionary - self.dict_thematic_source: dict[str, str] = {} + self.dict_thematic_source: dict[any, str] = {} # dictionary to store all unioned thematic geometries self.thematic_union = None @@ -157,11 +158,11 @@ def __init__( # CAPAKEY for GRB-parcels) self.name_reference_id = "ref_identifier" # dictionary to store all reference geometries - self.dict_reference: dict[str, BaseGeometry] = {} + self.dict_reference: dict[any, BaseGeometry] = {} # dictionary to store properties of the reference-features (optional) - self.dict_reference_properties: dict[str, dict] = {} + self.dict_reference_properties: dict[any, dict] = {} # Dict to store source-information of the reference dictionary - self.dict_reference_source: dict[str, str] = {} + self.dict_reference_source: dict[any, str] = {} # to save a unioned geometry of all reference polygons; needed for calculation # in most OD-strategies self.reference_union = None @@ -169,13 +170,13 @@ def __init__( # results # output-dictionaries (all results of process()), grouped by theme_id and relevant_distance - self.dict_processresults: dict[str, dict[float, ProcessResult]] = {} + self.dict_processresults: dict[any, dict[float, ProcessResult]] = {} # dictionary with the 'predicted' results, grouped by theme_id and relevant_distance - self.dict_predictions: dict[str, dict[float, ProcessResult]] = {} + self.dict_predictions: dict[any, dict[float, ProcessResult]] = {} # dictionary with the 'evaluated predicted' results, grouped by theme_id and relevant_distance - self.dict_evaluated_predictions: dict[str, dict[float, ProcessResult]] = {} + self.dict_evaluated_predictions: dict[any, dict[float, ProcessResult]] = {} # dictionary with the 'evaluated predicted' properties, grouped by theme_id and relevant_distance - self.dict_evaluated_predictions_properties: dict[str, dict[float, {}]] = {} + self.dict_evaluated_predictions_properties: dict[any, dict[float, {}]] = {} # Coordinate reference system # thematic geometries and reference geometries are assumed to be in the same CRS @@ -321,7 +322,7 @@ def process( relevant_distance=1, od_strategy=OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE, threshold_overlap_percentage=50, - ) -> dict[str, dict[float, ProcessResult]]: + ) -> dict[any, dict[float, ProcessResult]]: """ Calculates the resulting dictionaries for thematic data based on a series of relevant distances. @@ -356,8 +357,9 @@ def process( futures = [] if dict_thematic is None: dict_thematic = self.dict_thematic + dict_multi_as_single = {} if self.multi_as_single_modus: - dict_thematic = multipolygons_to_singles(dict_thematic) + dict_thematic, dict_multi_as_single = multipolygons_to_singles(dict_thematic) if self.max_workers!=-1: with ThreadPoolExecutor(max_workers=self.max_workers) as executor:#max_workers=5 @@ -405,7 +407,7 @@ def process( dict_series[key][relevant_distance] = processed_result if self.multi_as_single_modus: - dict_series = merge_process_results(dict_series) + dict_series = merge_process_results(dict_series,dict_multi_as_single) self.logger.feedback_info( "End of processing series: " + str(self.relevant_distances) @@ -700,7 +702,7 @@ def get_brdr_formula(self, geometry: BaseGeometry, with_geom=False): ########################################### def get_results_as_geojson( - self, resulttype=AlignerResultType.PROCESSRESULTS, formula=False + self, resulttype=AlignerResultType.PROCESSRESULTS, formula=False, attributes=False ): """ get a geojson of a dictionary containing the resulting geometries for all @@ -726,12 +728,15 @@ def get_results_as_geojson( for theme_id, results_dict in dict_series.items(): for relevant_distance, process_results in results_dict.items(): - if formula and not (theme_id in prop_dictionary and relevant_distance in prop_dictionary[theme_id] and FORMULA_FIELD_NAME in prop_dictionary[theme_id][relevant_distance]): + if relevant_distance not in prop_dictionary[theme_id]: + prop_dictionary[theme_id][relevant_distance]={} + if attributes: + for attr,value in self.dict_thematic_properties[theme_id].items(): + prop_dictionary[theme_id][relevant_distance][attr] = value + if formula: #and not (theme_id in prop_dictionary and relevant_distance in prop_dictionary[theme_id] and NEW_FORMULA_FIELD_NAME in prop_dictionary[theme_id][relevant_distance]): result = process_results["result"] formula = self.get_brdr_formula(result) - prop_dictionary[theme_id][relevant_distance] = { - FORMULA_FIELD_NAME: json.dumps(formula) - } + prop_dictionary[theme_id][relevant_distance][NEW_FORMULA_FIELD_NAME] = json.dumps(formula) return get_series_geojson_dict( dict_series, crs=self.CRS, @@ -1174,7 +1179,7 @@ def _evaluate(self, id_theme, geom_predicted): #threshold_area = 5 #threshold_percentage = 1 properties = { - FORMULA_FIELD_NAME: "", + NEW_FORMULA_FIELD_NAME: "", EVALUATION_FIELD_NAME: Evaluation.TO_CHECK_4, FULL_BASE_FIELD_NAME: None, FULL_ACTUAL_FIELD_NAME: None, @@ -1185,17 +1190,17 @@ def _evaluate(self, id_theme, geom_predicted): } actual_formula = self.get_brdr_formula(geom_predicted) - properties[FORMULA_FIELD_NAME] = json.dumps( + properties[NEW_FORMULA_FIELD_NAME] = json.dumps( actual_formula ) base_formula = None if ( id_theme in self.dict_thematic_properties - and FORMULA_FIELD_NAME in self.dict_thematic_properties[id_theme] + and BASE_FORMULA_FIELD_NAME in self.dict_thematic_properties[id_theme] ): - base_formula = self.dict_thematic_properties[id_theme][ - FORMULA_FIELD_NAME - ] + base_formula = json.loads(self.dict_thematic_properties[id_theme][ + BASE_FORMULA_FIELD_NAME + ]) if base_formula is None or actual_formula is None: properties[EVALUATION_FIELD_NAME] = Evaluation.NO_PREDICTION_5 diff --git a/brdr/constants.py b/brdr/constants.py index 820f437..d49a1ad 100644 --- a/brdr/constants.py +++ b/brdr/constants.py @@ -8,7 +8,8 @@ MULTI_SINGLE_ID_SEPARATOR = "*$*" PREFIX_FIELDNAME = "brdr_" -FORMULA_FIELD_NAME = PREFIX_FIELDNAME + "formula" +BASE_FORMULA_FIELD_NAME = PREFIX_FIELDNAME + "base_formula" +NEW_FORMULA_FIELD_NAME = PREFIX_FIELDNAME + "new_formula" EVALUATION_FIELD_NAME = PREFIX_FIELDNAME + "evaluation" DIFF_PERCENTAGE_FIELD_NAME = PREFIX_FIELDNAME + "diff_percentage" DIFF_AREA_FIELD_NAME = PREFIX_FIELDNAME + "diff_area" diff --git a/brdr/grb.py b/brdr/grb.py index 9278699..0939e0d 100644 --- a/brdr/grb.py +++ b/brdr/grb.py @@ -5,7 +5,7 @@ from datetime import datetime import numpy as np -from shapely import intersects, Polygon +from shapely import intersects from shapely.geometry import shape from brdr.aligner import Aligner @@ -14,7 +14,7 @@ LAST_VERSION_DATE, DATE_FORMAT, VERSION_DATE, - FORMULA_FIELD_NAME, + BASE_FORMULA_FIELD_NAME, ) from brdr.constants import DOWNLOAD_LIMIT from brdr.constants import GRB_BUILDING_ID @@ -24,7 +24,7 @@ from brdr.constants import GRB_MAX_REFERENCE_BUFFER from brdr.constants import GRB_PARCEL_ID from brdr.constants import GRB_VERSION_DATE -from brdr.enums import GRBType +from brdr.enums import GRBType, AlignerResultType from brdr.geometry_utils import buffer_pos, safe_intersection, safe_unary_union from brdr.geometry_utils import create_donut from brdr.geometry_utils import features_by_geometric_operation @@ -34,7 +34,6 @@ from brdr.utils import geojson_to_dicts from brdr.utils import get_collection from brdr.utils import get_collection_by_partition -from brdr.utils import get_series_geojson_dict log = logging.getLogger(__name__) @@ -341,9 +340,10 @@ def get_collection_grb_parcels_by_date( def update_to_actual_grb( featurecollection, id_theme_fieldname, - formula_field=FORMULA_FIELD_NAME, + base_formula_field=BASE_FORMULA_FIELD_NAME, max_distance_for_actualisation=2, feedback=None, + attributes=True, ): """ Function to update a thematic featurecollection to the most actual version of GRB. @@ -357,30 +357,27 @@ def update_to_actual_grb( last_version_date = datetime.now().date() for feature in featurecollection["features"]: id_theme = feature["properties"][id_theme_fieldname] - try: - geom = shape(feature["geometry"]) - except Exception: - geom = Polygon() - logger.feedback_debug("id theme: " + id_theme) - logger.feedback_debug("geometry (wkt): " + geom.wkt) + geom = shape(feature["geometry"]) + #logger.feedback_debug("id theme: " + id_theme) + #logger.feedback_debug("geometry (wkt): " + geom.wkt) dict_thematic[id_theme] = geom + dict_thematic_props[id_theme] = feature["properties"] try: - dict_thematic_props[id_theme] = { - FORMULA_FIELD_NAME: json.loads(feature["properties"][formula_field]) - } - logger.feedback_debug("formula: " + str(dict_thematic_props[id_theme])) + dict_thematic_props[id_theme][BASE_FORMULA_FIELD_NAME] =feature["properties"][base_formula_field] + base_formula = json.loads(dict_thematic_props[id_theme][BASE_FORMULA_FIELD_NAME]) + logger.feedback_debug("formula: " + str(base_formula)) except Exception: raise Exception("Formula -attribute-field (json) cannot be loaded") try: logger.feedback_debug(str(dict_thematic_props[id_theme])) if ( - LAST_VERSION_DATE in dict_thematic_props[id_theme][FORMULA_FIELD_NAME] - and dict_thematic_props[id_theme][FORMULA_FIELD_NAME][LAST_VERSION_DATE] + LAST_VERSION_DATE in base_formula + and base_formula[LAST_VERSION_DATE] is not None - and dict_thematic_props[id_theme][FORMULA_FIELD_NAME][LAST_VERSION_DATE] + and base_formula[LAST_VERSION_DATE] != "" ): - str_lvd = dict_thematic_props[id_theme][FORMULA_FIELD_NAME][ + str_lvd = base_formula[ LAST_VERSION_DATE ] lvd = datetime.strptime(str_lvd, DATE_FORMAT).date() @@ -414,7 +411,7 @@ def update_to_actual_grb( ) return {} logger.feedback_debug(str(datetime_start)) - logger.feedback_debug(str(formula_field)) + logger.feedback_debug(str(base_formula_field)) # Initiate a Aligner to reference thematic features to the actual borders actual_aligner = Aligner(feedback=feedback,max_workers=None) @@ -430,12 +427,7 @@ def update_to_actual_grb( ) dict_evaluated, prop_dictionary = actual_aligner.compare(ids_to_compare= affected) - return get_series_geojson_dict( - dict_evaluated, - crs=actual_aligner.CRS, - id_field=actual_aligner.name_thematic_id, - series_prop_dict=prop_dictionary, - ) + return actual_aligner.get_results_as_geojson(resulttype=AlignerResultType.EVALUATED_PREDICTIONS,formula=True,attributes=attributes) class GRBActualLoader(GeoJsonLoader): diff --git a/brdr/utils.py b/brdr/utils.py index 505e1d6..c3a2e65 100644 --- a/brdr/utils.py +++ b/brdr/utils.py @@ -144,6 +144,7 @@ def multipolygons_to_singles(dict_geoms): message printed. """ resulting_dict_geoms = {} + dict_multi_as_single = {} for key in dict_geoms: geom = dict_geoms[key] if str(geom.geom_type) == "Polygon": @@ -156,11 +157,12 @@ def multipolygons_to_singles(dict_geoms): i = 0 for p in polygons: new_key = str(key) + MULTI_SINGLE_ID_SEPARATOR + str(i) + dict_multi_as_single[new_key] = key resulting_dict_geoms[new_key] = p i = i + 1 else: logging.debug("geom excluded: " + str(geom) + " for key: " + str(key)) - return resulting_dict_geoms + return resulting_dict_geoms,dict_multi_as_single def polygonize_reference_data(dict_ref): @@ -494,7 +496,8 @@ def _add_bbox_to_url(url, crs=DEFAULT_CRS, bbox=None): def merge_process_results( - result_dict: dict[str, dict[float, ProcessResult]] + result_dict: dict[str, dict[float, ProcessResult]], + dict_multi_as_single:dict ) -> dict[str, dict[float, ProcessResult]]: """ Merges processresults in a dictionary from multiple themeIDs into a single themeID. @@ -509,7 +512,10 @@ def merge_process_results( grouped_results: dict[str, dict[float, ProcessResult]] = {} for id_theme, dict_results in result_dict.items(): - id_theme_global = id_theme.split(MULTI_SINGLE_ID_SEPARATOR)[0] + if id_theme in dict_multi_as_single.keys(): + id_theme_global = dict_multi_as_single[id_theme] + else: + id_theme_global=id_theme if id_theme_global not in grouped_results: grouped_results[id_theme_global] = dict_results else: diff --git a/examples/example_evaluate.py b/examples/example_evaluate.py index 046592d..be929b9 100644 --- a/examples/example_evaluate.py +++ b/examples/example_evaluate.py @@ -1,10 +1,11 @@ +import json from datetime import date import numpy as np from shapely import from_wkt from brdr.aligner import Aligner -from brdr.constants import FORMULA_FIELD_NAME, EVALUATION_FIELD_NAME +from brdr.constants import EVALUATION_FIELD_NAME, BASE_FORMULA_FIELD_NAME from brdr.enums import GRBType, AlignerResultType from brdr.grb import GRBActualLoader from brdr.grb import GRBFiscalParcelLoader @@ -30,7 +31,7 @@ for key in base_process_result: thematic_dict_result[key] = base_process_result[key][relevant_distance]["result"] thematic_dict_formula[key] = { - FORMULA_FIELD_NAME: base_aligner.get_brdr_formula(thematic_dict_result[key]) + BASE_FORMULA_FIELD_NAME: json.dumps(base_aligner.get_brdr_formula(thematic_dict_result[key])) } print(key + ": " + thematic_dict_result[key].wkt) print(key + ": " + str(thematic_dict_formula[key])) @@ -57,7 +58,7 @@ actual_aligner.relevant_distances = np.arange(0, 200, 10, dtype=int) / 100 dict_evaluated, prop_dictionary = actual_aligner.compare(ids_to_compare=affected) -fc = actual_aligner.get_results_as_geojson(resulttype=AlignerResultType.EVALUATED_PREDICTIONS) +fc = actual_aligner.get_results_as_geojson(resulttype=AlignerResultType.EVALUATED_PREDICTIONS,formula=True, attributes=True) print(fc["result"]) for feature in fc["result"]["features"]: diff --git a/examples/example_evaluate_ao.py b/examples/example_evaluate_ao.py index 21cb9de..a0b5e00 100644 --- a/examples/example_evaluate_ao.py +++ b/examples/example_evaluate_ao.py @@ -1,9 +1,10 @@ +import json from datetime import date import numpy as np from brdr.aligner import Aligner -from brdr.constants import EVALUATION_FIELD_NAME, FORMULA_FIELD_NAME +from brdr.constants import EVALUATION_FIELD_NAME, BASE_FORMULA_FIELD_NAME from brdr.enums import GRBType, AlignerResultType from brdr.grb import ( get_affected_by_grb_change, @@ -27,7 +28,7 @@ for key in base_process_result: thematic_dict_result[key] = base_process_result[key][relevant_distance]["result"] thematic_dict_formula[key] = { - FORMULA_FIELD_NAME: base_aligner.get_brdr_formula(thematic_dict_result[key]) + BASE_FORMULA_FIELD_NAME: json.dumps(base_aligner.get_brdr_formula(thematic_dict_result[key])) } print(key + ": " + thematic_dict_result[key].wkt) print(key + ": " + str(thematic_dict_formula[key])) @@ -54,7 +55,7 @@ actual_aligner.relevant_distances = np.arange(0, 200, 10, dtype=int) / 100 dict_evaluated, prop_dictionary = actual_aligner.compare(ids_to_compare=affected) -fc = actual_aligner.get_results_as_geojson(resulttype=AlignerResultType.EVALUATED_PREDICTIONS) +fc = actual_aligner.get_results_as_geojson(resulttype=AlignerResultType.EVALUATED_PREDICTIONS,formula=True, attributes=True) print(fc["result"]) for feature in fc["result"]["features"]: diff --git a/examples/example_integer_id.py b/examples/example_integer_id.py new file mode 100644 index 0000000..e0355fb --- /dev/null +++ b/examples/example_integer_id.py @@ -0,0 +1,39 @@ +from brdr.aligner import Aligner +from brdr.enums import OpenbaarDomeinStrategy, AlignerResultType +from brdr.geometry_utils import geom_from_wkt +from brdr.loader import DictLoader + +# CREATE AN ALIGNER +aligner = Aligner( + crs="EPSG:31370", +) +aligner.multi_as_single_modus = True +# ADD A THEMATIC POLYGON TO THEMATIC DICTIONARY and LOAD into Aligner +id = 1 +thematic_dict = {id: geom_from_wkt("POLYGON ((0 0, 0 9, 5 10, 10 0, 0 0))")} +thematic_dict_properties = { + id: {"propA": 1, "propB": 1.1, "propC": "dit is tekst", "propD": None} +} +loader = DictLoader( + data_dict=thematic_dict, data_dict_properties=thematic_dict_properties +) +aligner.load_thematic_data(loader) +# ADD A REFERENCE POLYGON TO REFERENCE DICTIONARY and LOAD into Aligner +reference_dict = {100: geom_from_wkt("POLYGON ((0 1, 0 10,8 10,10 1,0 1))")} +loader = DictLoader(reference_dict) +aligner.load_reference_data(loader) +# EXECUTE THE ALIGNMENT +relevant_distance = 1 +process_result = aligner.process( + relevant_distance=relevant_distance, + od_strategy=OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE, + threshold_overlap_percentage=50, +) +# PRINT RESULTS IN WKT +print("result: " + process_result[id][relevant_distance]["result"].wkt) +print("added area: " + process_result[id][relevant_distance]["result_diff_plus"].wkt) +print("removed area: " + process_result[id][relevant_distance]["result_diff_min"].wkt) +fcs = aligner.get_results_as_geojson( + resulttype=AlignerResultType.PROCESSRESULTS, formula=True, attributes=True +) +print(fcs["result"]) diff --git a/examples/example_multipolygon.py b/examples/example_multipolygon.py index 0c3ed81..e189d51 100644 --- a/examples/example_multipolygon.py +++ b/examples/example_multipolygon.py @@ -13,7 +13,7 @@ aligner0.load_thematic_data( GeoJsonFileLoader("../tests/testdata/multipolygon.geojson", "theme_identifier") ) -aligner0.dict_thematic = multipolygons_to_singles(aligner0.dict_thematic) +aligner0.dict_thematic,dict_multi_as_single = multipolygons_to_singles(aligner0.dict_thematic) aligner0.load_thematic_data( DictLoader( aligner0.dict_thematic, diff --git a/examples/example_parcel_change_detector.py b/examples/example_parcel_change_detector.py index 2e9c33e..8cc7f56 100644 --- a/examples/example_parcel_change_detector.py +++ b/examples/example_parcel_change_detector.py @@ -2,7 +2,8 @@ from datetime import datetime from brdr.aligner import Aligner -from brdr.constants import EVALUATION_FIELD_NAME, RELEVANT_DISTANCE_FIELD_NAME +from brdr.constants import EVALUATION_FIELD_NAME, RELEVANT_DISTANCE_FIELD_NAME, BASE_FORMULA_FIELD_NAME, \ + NEW_FORMULA_FIELD_NAME from brdr.grb import GRBFiscalParcelLoader from brdr.grb import update_to_actual_grb from brdr.oe import OnroerendErfgoedLoader @@ -83,6 +84,7 @@ fcs = update_to_actual_grb( featurecollection_base_result, base_aligner.name_thematic_id, + base_formula_field=NEW_FORMULA_FIELD_NAME, max_distance_for_actualisation=max_distance_for_actualisation, ) diff --git a/examples/example_parcel_vs_building.py b/examples/example_parcel_vs_building.py index 7d3cba1..0d1fdc6 100644 --- a/examples/example_parcel_vs_building.py +++ b/examples/example_parcel_vs_building.py @@ -35,9 +35,9 @@ # Example how to use a series (for histogram) series = np.arange(0, 300, 10, dtype=int) / 100 - x_dict_series = aligner_x.process(series, 4, 50) + x_dict_series = aligner_x.process(relevant_distances=series, od_strategy=4, threshold_overlap_percentage=50) x_resulting_areas = diffs_from_dict_series(x_dict_series, aligner_x.dict_thematic) - y_dict_series = aligner_y.process(series, 4, 50) + y_dict_series = aligner_y.process(relevant_distances=series, od_strategy=4, threshold_overlap_percentage=50) y_resulting_areas = diffs_from_dict_series(y_dict_series, aligner_y.dict_thematic) # plot_diffs(series,x_resulting_areas) # plot_diffs(series,y_resulting_areas) diff --git a/examples/example_update_to_actual_grb.py b/examples/example_update_to_actual_grb.py index 9f97bf0..5dbc7e2 100644 --- a/examples/example_update_to_actual_grb.py +++ b/examples/example_update_to_actual_grb.py @@ -1,5 +1,5 @@ from brdr.aligner import Aligner -from brdr.constants import EVALUATION_FIELD_NAME +from brdr.constants import EVALUATION_FIELD_NAME, NEW_FORMULA_FIELD_NAME from brdr.grb import GRBFiscalParcelLoader from brdr.grb import update_to_actual_grb from brdr.loader import GeoJsonLoader @@ -17,6 +17,7 @@ { "type": "Feature", "properties": { + "testattribute": "test", "nr_calculations": 1, "ID": "206285", "relevant_distance": 2.0, @@ -58,12 +59,12 @@ GRBFiscalParcelLoader(year=base_year, aligner=base_aligner) ) base_process_result = base_aligner.process(relevant_distance=2) -fcs = base_aligner.get_results_as_geojson(formula=True) +fcs = base_aligner.get_results_as_geojson(formula=True,attributes=True) featurecollection_base_result = fcs["result"] print(featurecollection_base_result) # Update Featurecollection to actual version featurecollection = update_to_actual_grb( - featurecollection_base_result, base_aligner.name_thematic_id + featurecollection_base_result, base_aligner.name_thematic_id,base_formula_field=NEW_FORMULA_FIELD_NAME ) # Print results for feature in featurecollection["result"]["features"]: diff --git a/tests/test_aligner.py b/tests/test_aligner.py index ca518e8..f458015 100644 --- a/tests/test_aligner.py +++ b/tests/test_aligner.py @@ -9,7 +9,7 @@ from shapely.geometry import shape from brdr.aligner import Aligner -from brdr.constants import FORMULA_FIELD_NAME +from brdr.constants import NEW_FORMULA_FIELD_NAME from brdr.enums import GRBType, AlignerResultType from brdr.enums import OpenbaarDomeinStrategy from brdr.geometry_utils import _grid_bounds @@ -378,7 +378,7 @@ def test_evaluate(self): "result" ] thematic_dict_formula[key] = { - FORMULA_FIELD_NAME: base_aligner.get_brdr_formula( + NEW_FORMULA_FIELD_NAME: base_aligner.get_brdr_formula( thematic_dict_result[key] ) } diff --git a/tests/test_grb.py b/tests/test_grb.py index 29fd016..0998d48 100644 --- a/tests/test_grb.py +++ b/tests/test_grb.py @@ -218,7 +218,7 @@ def test_update_to_actual_grb(self): }, "properties": { "area": 503.67736346047064, - "brdr_formula": '{"alignment_date": "2024-09-19", "brdr_version": "0.2.1", "reference_source": {"source": "Adpf", "version_date": "2022-01-01"}, "full": true, "reference_features": {"12034A0181/00K000": {"full": true, "area": 503.68, "percentage": 100, "version_date": "2019-08-30"}}, "reference_od": null, "last_version_date": "2019-08-30"}', + "brdr_base_formula": '{"alignment_date": "2024-09-19", "brdr_version": "0.2.1", "reference_source": {"source": "Adpf", "version_date": "2022-01-01"}, "full": true, "reference_features": {"12034A0181/00K000": {"full": true, "area": 503.68, "percentage": 100, "version_date": "2019-08-30"}}, "reference_od": null, "last_version_date": "2019-08-30"}', "nr_calculations": 1, "perimeter": 125.74541473322422, "relevant_distance": 2, @@ -233,7 +233,7 @@ def test_update_to_actual_grb(self): # Update Featurecollection to actual version featurecollection = update_to_actual_grb( - featurecollection_base_result, name_thematic_id + featurecollection_base_result, name_thematic_id,base_formula_field="brdr_base_formula" ) # Print results for feature in featurecollection["result"]["features"]: diff --git a/tests/test_utils.py b/tests/test_utils.py index 8c56abb..bc185cc 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -41,26 +41,26 @@ def test_get_breakpoints_zerostreak_no_zerostreaks(self): def test_multipolygons_to_singles_empty_dict(self): data = {} - result = multipolygons_to_singles(data) + result,dict_multi_as_single = multipolygons_to_singles(data) self.assertEqual(result, {}) def test_multipolygons_to_singles_with_point(self): geometry1 = shapely.geometry.Polygon([(0, 0), (1, 0), (1, 1), (0, 1)]) geometry2 = shapely.geometry.Point(0, 0) data = {"test_id1": geometry1, "test_id2": geometry2} - result = multipolygons_to_singles(data) + result,dict_multi_as_single = multipolygons_to_singles(data) self.assertEqual(result, {"test_id1": geometry1}) def test_multipolygons_to_singles_single_polygon(self): geometry = shapely.geometry.Polygon([(0, 0), (1, 0), (1, 1), (0, 1)]) data = {"test_id": geometry} - result = multipolygons_to_singles(data) + result,dict_multi_as_single = multipolygons_to_singles(data) self.assertEqual(result, data) def test_multipolygons_to_singles_multipolygon_single_poly(self): geometry = shapely.geometry.Polygon([(0, 0), (1, 0), (1, 1), (0, 1)]) data = {"test_id": shapely.geometry.MultiPolygon([geometry])} - result = multipolygons_to_singles(data) + result,dict_multi_as_single = multipolygons_to_singles(data) self.assertEqual(result, {"test_id": geometry}) def test_polygonize_reference_data_no_overlap(self): @@ -168,6 +168,7 @@ def test_merge_process_results(self): key_1 = "key" + MULTI_SINGLE_ID_SEPARATOR + "1" key_2 = "key" + MULTI_SINGLE_ID_SEPARATOR + "2" key_3 = "key_3" + dict_multi_as_single = {key_1:"key",key_2:"key"} process_result_1 = ProcessResult() process_result_1["result"] = Polygon([(0, 0), (10, 0), (10, 10), (0, 10)]) process_result_2 = ProcessResult() @@ -179,5 +180,5 @@ def test_merge_process_results(self): key_2: {0: process_result_2}, key_3: {0: process_result_3}, } - merged_testdict = merge_process_results(testdict) + merged_testdict = merge_process_results(result_dict=testdict,dict_multi_as_single = dict_multi_as_single) assert len(merged_testdict.keys()) == 2 From e7f432d164cb4877146d5f27a46a49c8e798cbb7 Mon Sep 17 00:00:00 2001 From: dieuska Date: Tue, 1 Oct 2024 16:03:22 +0200 Subject: [PATCH 16/26] #107 --- brdr/aligner.py | 22 ++++++++++++++++++++-- tests/test_aligner.py | 26 ++++++++++++++++++++------ 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/brdr/aligner.py b/brdr/aligner.py index 54255b2..b135880 100644 --- a/brdr/aligner.py +++ b/brdr/aligner.py @@ -17,6 +17,7 @@ from shapely import remove_repeated_points from shapely import to_geojson from shapely.geometry.base import BaseGeometry +from shapely.geometry.multipolygon import MultiPolygon from brdr import __version__ from brdr.constants import DEFAULT_CRS @@ -406,6 +407,25 @@ def process( if self.multi_as_single_modus: dict_series = merge_process_results(dict_series) + # Check if geom changes from polygon to multipolygon or vice versa + for theme_id, dict_dist_results in dict_series.items(): + original_geometry = self.dict_thematic[theme_id] + original_geometry_length =-1 + if original_geometry.geom_type =="Polygon": + original_geometry_length=1 + elif original_geometry.geom_type =="MultiPolygon": + original_geometry_length = len(original_geometry.geoms) + for relevant_distance, process_result in dict_dist_results.items(): + resulting_geom = process_result["result"] + resulting_geometry_length =-1 + if resulting_geom.geom_type == "Polygon": + resulting_geometry_length = 1 + elif resulting_geom.geom_type == "MultiPolygon": + resulting_geometry_length = len(resulting_geom.geoms) + if original_geometry_length != resulting_geometry_length: + msg = "Difference in amount of polygons" + self.logger.feedback_debug(msg) + process_result["remark"] = process_result["remark"] + " / " + msg self.logger.feedback_info( "End of processing series: " + str(self.relevant_distances) @@ -1154,8 +1174,6 @@ def _postprocess_preresult(self, preresult, geom_thematic,relevant_distance) -> ), self.correction_distance,mitre_limit=self.mitre_limit, ) - # geom_result_diff_plus = safe_difference(geom_thematic_result, geom_thematic) - # geom_result_diff_min = safe_difference(geom_thematic, geom_thematic_result) return { diff --git a/tests/test_aligner.py b/tests/test_aligner.py index ca518e8..4abd36d 100644 --- a/tests/test_aligner.py +++ b/tests/test_aligner.py @@ -7,6 +7,8 @@ from shapely import from_wkt from shapely.geometry import Polygon from shapely.geometry import shape +from shapely.geometry.multipolygon import MultiPolygon +from shapely.predicates import equals from brdr.aligner import Aligner from brdr.constants import FORMULA_FIELD_NAME @@ -269,8 +271,8 @@ def test_process_interior_ring(self): self.assertEqual(len(result_dict), len(thematic_dict)) def test_process_circle(self): - # TODO geometry = Point(0, 0).buffer(3) + #geometry = MultiPolygon([geometry]) thematic_dict = {"key": geometry} self.sample_aligner.load_thematic_data(DictLoader(thematic_dict)) # LOAD REFERENCE DICTIONARY @@ -336,7 +338,7 @@ def test_get_reference_as_geojson(self): self.sample_aligner.get_input_as_geojson() def test_fully_aligned_input(self): - aligned_shape = from_wkt("POLYGON ((0 0, 0 9, 5 10, 10 0, 0 0))") + aligned_shape = from_wkt("MULTIPOLYGON (((0 0, 0 9, 5 10, 10 0, 0 0)))") loader = DictLoader({"theme_id_1": aligned_shape}) self.sample_aligner.load_thematic_data( DictLoader({"theme_id_1": aligned_shape}) @@ -344,13 +346,13 @@ def test_fully_aligned_input(self): self.sample_aligner.load_reference_data(DictLoader({"ref_id_1": aligned_shape})) relevant_distance = 1 result = self.sample_aligner.process(relevant_distance=relevant_distance) - assert result["theme_id_1"][relevant_distance].get("result") == aligned_shape - assert result["theme_id_1"][relevant_distance].get("result_diff") == Polygon() + assert equals(result["theme_id_1"][relevant_distance].get("result"),aligned_shape) + assert result["theme_id_1"][relevant_distance].get("result_diff").is_empty assert ( - result["theme_id_1"][relevant_distance].get("result_diff_min") == Polygon() + result["theme_id_1"][relevant_distance].get("result_diff_min").is_empty ) assert ( - result["theme_id_1"][relevant_distance].get("result_diff_plus") == Polygon() + result["theme_id_1"][relevant_distance].get("result_diff_plus").is_empty ) def test_evaluate(self): @@ -423,6 +425,18 @@ def test_evaluate(self): print(fc["result"]) fcs = actual_aligner.get_results_as_geojson(formula=True) + def test_remark_for_poly_multipoly(self): + shape = from_wkt( + "MultiPolygon(((48893.03662109375 214362.93756103515625, 48890.8258056640625 214368.482666015625, 48890.7159423828125 214368.44110107421875, 48887.6488037109375 214367.2845458984375, 48886.3800048828125 214368.68017578125, 48885.1068115234375 214370.08062744140625, 48884.3330078125 214369.782470703125, 48882.563720703125 214369.10064697265625, 48882.1116943359375 214370.1346435546875, 48878.5626220703125 214368.70196533203125, 48877.839111328125 214368.40997314453125, 48877.2352294921875 214369.79376220703125, 48876.7911376953125 214369.60687255859375, 48875.0850830078125 214373.62353515625, 48875.478759765625 214373.8182373046875, 48881.5286865234375 214376.81109619140625, 48885.10546875 214372.36151123046875, 48887.0050048828125 214370.08538818359375, 48888.4698486328125 214368.330078125, 48890.366943359375 214369.2685546875, 48901.0638427734375 214374.56024169921875, 48905.0159912109375 214369.61175537109375, 48904.472900390625 214367.53851318359375, 48893.03662109375 214362.93756103515625)))") + self.sample_aligner.load_thematic_data( + DictLoader({"theme_id_1": shape}) + ) + self.sample_aligner.load_reference_data(GRBActualLoader( grb_type=GRBType.ADP, partition=1000, aligner=self.sample_aligner + )) + self.sample_aligner.process(relevant_distances=[2]) + assert self.sample_aligner.dict_processresults["theme_id_1"][2]["remark"]!="" + + def test_fully_aligned_geojson_output(self): aligned_shape = from_wkt( "MultiPolygon (((173463.11530961000244133 174423.83310307000647299, " From 077d5f75fb08a106e6b1195ec4ee8ec6d35d7109 Mon Sep 17 00:00:00 2001 From: dieuska Date: Tue, 1 Oct 2024 16:10:07 +0200 Subject: [PATCH 17/26] #107 --- brdr/aligner.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/brdr/aligner.py b/brdr/aligner.py index 000496d..c176fe1 100644 --- a/brdr/aligner.py +++ b/brdr/aligner.py @@ -409,6 +409,26 @@ def process( if self.multi_as_single_modus: dict_series = merge_process_results(dict_series,dict_multi_as_single) + # Check if geom changes from polygon to multipolygon or vice versa + for theme_id, dict_dist_results in dict_series.items(): + original_geometry = self.dict_thematic[theme_id] + original_geometry_length =-1 + if original_geometry.geom_type =="Polygon": + original_geometry_length=1 + elif original_geometry.geom_type =="MultiPolygon": + original_geometry_length = len(original_geometry.geoms) + for relevant_distance, process_result in dict_dist_results.items(): + resulting_geom = process_result["result"] + resulting_geometry_length =-1 + if resulting_geom.geom_type == "Polygon": + resulting_geometry_length = 1 + elif resulting_geom.geom_type == "MultiPolygon": + resulting_geometry_length = len(resulting_geom.geoms) + if original_geometry_length != resulting_geometry_length: + msg = "Difference in amount of polygons" + self.logger.feedback_debug(msg) + process_result["remark"] = process_result["remark"] + " / " + msg + self.logger.feedback_info( "End of processing series: " + str(self.relevant_distances) ) @@ -1506,7 +1526,4 @@ def _check_equality( return True, Evaluation.EQUALITY_EQUAL_FORMULA_2 if base_formula["full"] and actual_formula["full"] and od_alike: return True, Evaluation.EQUALITY_FULL_3 - return False, Evaluation.NO_PREDICTION_5 - - - + return False, Evaluation.NO_PREDICTION_5 \ No newline at end of file From 691f3e28670a9a5a5d341227713830a9975cc3aa Mon Sep 17 00:00:00 2001 From: dieuska Date: Wed, 2 Oct 2024 10:47:45 +0200 Subject: [PATCH 18/26] cleanups --- brdr/aligner.py | 463 ++++++++++++++------- brdr/grb.py | 59 ++- brdr/utils.py | 4 +- examples/example_multipolygon.py | 4 +- examples/example_parcel_change_detector.py | 3 +- examples/example_update_to_actual_grb.py | 6 +- tests/test_aligner.py | 1 - 7 files changed, 351 insertions(+), 189 deletions(-) diff --git a/brdr/aligner.py b/brdr/aligner.py index c176fe1..69298aa 100644 --- a/brdr/aligner.py +++ b/brdr/aligner.py @@ -24,9 +24,14 @@ LAST_VERSION_DATE, VERSION_DATE, DATE_FORMAT, - NEW_FORMULA_FIELD_NAME, EVALUATION_FIELD_NAME, FULL_BASE_FIELD_NAME, FULL_ACTUAL_FIELD_NAME, + NEW_FORMULA_FIELD_NAME, + EVALUATION_FIELD_NAME, + FULL_BASE_FIELD_NAME, + FULL_ACTUAL_FIELD_NAME, DIFF_PERCENTAGE_FIELD_NAME, - DIFF_AREA_FIELD_NAME, OD_ALIKE_FIELD_NAME, EQUAL_REFERENCE_FEATURES_FIELD_NAME, + DIFF_AREA_FIELD_NAME, + OD_ALIKE_FIELD_NAME, + EQUAL_REFERENCE_FEATURES_FIELD_NAME, ) from brdr.enums import ( OpenbaarDomeinStrategy, @@ -74,6 +79,7 @@ def __init__( threshold_overlap_percentage=50, od_strategy=OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE, crs=DEFAULT_CRS, + multi_as_single_modus=True, threshold_exclusion_area=0, threshold_exclusion_percentage=0, buffer_multiplication_factor=1.01, @@ -125,10 +131,10 @@ def __init__( self.max_workers = max_workers # Multiplication-factor used in OD-strategy 2 (SNAP-BOTH SIDED) when calculating # OD-area to take into account - self.buffer_multiplication_factor=buffer_multiplication_factor + self.buffer_multiplication_factor = buffer_multiplication_factor # Threshold-value to exclude circles getting processed (perfect circle = 1) based on # POLSPY-POPPER algorithm - self.threshold_circle_ratio=threshold_circle_ratio + self.threshold_circle_ratio = threshold_circle_ratio # Distance used in a pos_neg_buffer to remove slivers (technical correction) self.correction_distance = correction_distance # Buffer parameters: @@ -136,8 +142,8 @@ def __init__( # Explanation and examples: # https://shapely.readthedocs.io/en/stable/reference/shapely.buffer.html # https://postgis.net/docs/ST_Buffer.html - self.mitre_limit=mitre_limit - #quad_segments = 8 (by default in shapely) + self.mitre_limit = mitre_limit + # quad_segments = 8 (by default in shapely) # PROCESSING DEFAULTS # thematic @@ -187,7 +193,7 @@ def __init__( self.CRS = crs # this parameter is used to treat multipolygon as single polygons. So polygons # with ID splitter are separately evaluated and merged on result. - self.multi_as_single_modus = True + self.multi_as_single_modus = multi_as_single_modus self.logger.feedback_info("Aligner initialized") ##########LOADERS########################## @@ -209,7 +215,6 @@ def load_reference_data(self, loader: Loader): ##########PROCESSORS####################### ########################################### - def process_geometry( self, input_geometry: BaseGeometry, @@ -249,7 +254,7 @@ def process_geometry( self.logger.feedback_debug("process geometry") self.od_strategy = od_strategy self.threshold_overlap_percentage = threshold_overlap_percentage - buffer_distance = relevant_distance/2 + buffer_distance = relevant_distance / 2 # combine all parts of the input geometry to one polygon input_geometry = safe_unary_union(get_parts(input_geometry)) @@ -259,7 +264,9 @@ def process_geometry( preresult, relevant_intersection_array, relevant_diff_array, - ) = self._calculate_intersection_between_geometry_and_od(input_geometry,relevant_distance) + ) = self._calculate_intersection_between_geometry_and_od( + input_geometry, relevant_distance + ) # get a list of all ref_ids that are intersecting the thematic geometry ref_intersections = self.reference_items.take( self.reference_tree.query(geometry) @@ -281,7 +288,8 @@ def process_geometry( buffer_distance, self.threshold_overlap_percentage, self.threshold_exclusion_percentage, - self.threshold_exclusion_area,self.mitre_limit + self.threshold_exclusion_area, + self.mitre_limit, ) self.logger.feedback_debug("intersection calculated") preresult = self._add_multi_polygons_from_geom_to_array(geom, preresult) @@ -300,7 +308,9 @@ def process_geometry( relevant_diff = Polygon() # POSTPROCESSING - result_dict = self._postprocess_preresult(preresult, geometry,relevant_distance) + result_dict = self._postprocess_preresult( + preresult, geometry, relevant_distance + ) result_dict["result_relevant_intersection"] = relevant_intersection result_dict["result_relevant_diff"] = relevant_diff @@ -308,16 +318,15 @@ def process_geometry( # make a unary union for each key value in the result dict for key in ProcessResult.__annotations__: geometry = result_dict.get(key, Polygon()) # noqa - if isinstance(geometry,BaseGeometry) and not geometry.is_empty: + if isinstance(geometry, BaseGeometry) and not geometry.is_empty: geometry = safe_unary_union(geometry) result_dict[key] = geometry # noqa return result_dict - def process( self, - dict_thematic =None, + dict_thematic=None, relevant_distances: Iterable[float] = None, relevant_distance=1, od_strategy=OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE, @@ -359,10 +368,14 @@ def process( dict_thematic = self.dict_thematic dict_multi_as_single = {} if self.multi_as_single_modus: - dict_thematic, dict_multi_as_single = multipolygons_to_singles(dict_thematic) + dict_thematic, dict_multi_as_single = multipolygons_to_singles( + dict_thematic + ) - if self.max_workers!=-1: - with ThreadPoolExecutor(max_workers=self.max_workers) as executor:#max_workers=5 + if self.max_workers != -1: + with ThreadPoolExecutor( + max_workers=self.max_workers + ) as executor: # max_workers=5 for key, geometry in dict_thematic.items(): self.logger.feedback_info( f"thematic id {str(key)} processed with relevant distances (m) [{str(self.relevant_distances)}]" @@ -371,20 +384,26 @@ def process( dict_series_queue[key] = {} for relevant_distance in self.relevant_distances: try: - future = executor.submit(self.process_geometry, geometry, + future = executor.submit( + self.process_geometry, + geometry, relevant_distance, od_strategy, - threshold_overlap_percentage,) + threshold_overlap_percentage, + ) futures.append(future) - dict_series_queue[key][relevant_distance] = future + dict_series_queue[key][relevant_distance] = future except ValueError as e: - self.logger.feedback_warning("error for" + f"thematic id {str(key)} processed with relevant distances (m) [{str(self.relevant_distances)}]") + self.logger.feedback_warning( + "error for" + + f"thematic id {str(key)} processed with relevant distances (m) [{str(self.relevant_distances)}]" + ) dict_series_queue[key][relevant_distance] = None self.logger.feedback_warning(str(e)) - self.logger.feedback_debug ("waiting all started RD calculations") + self.logger.feedback_debug("waiting all started RD calculations") wait(futures) - for id_theme,dict_dist in dict_series_queue.items(): - for relevant_distance,future in dict_dist.items(): + for id_theme, dict_dist in dict_series_queue.items(): + for relevant_distance, future in dict_dist.items(): dict_series[id_theme][relevant_distance] = future.result() else: for key, geometry in dict_thematic.items(): @@ -407,19 +426,19 @@ def process( dict_series[key][relevant_distance] = processed_result if self.multi_as_single_modus: - dict_series = merge_process_results(dict_series,dict_multi_as_single) + dict_series = merge_process_results(dict_series, dict_multi_as_single) # Check if geom changes from polygon to multipolygon or vice versa for theme_id, dict_dist_results in dict_series.items(): original_geometry = self.dict_thematic[theme_id] - original_geometry_length =-1 - if original_geometry.geom_type =="Polygon": - original_geometry_length=1 - elif original_geometry.geom_type =="MultiPolygon": + original_geometry_length = -1 + if original_geometry.geom_type == "Polygon": + original_geometry_length = 1 + elif original_geometry.geom_type == "MultiPolygon": original_geometry_length = len(original_geometry.geoms) for relevant_distance, process_result in dict_dist_results.items(): resulting_geom = process_result["result"] - resulting_geometry_length =-1 + resulting_geometry_length = -1 if resulting_geom.geom_type == "Polygon": resulting_geometry_length = 1 elif resulting_geom.geom_type == "MultiPolygon": @@ -427,7 +446,7 @@ def process( if original_geometry_length != resulting_geometry_length: msg = "Difference in amount of polygons" self.logger.feedback_debug(msg) - process_result["remark"] = process_result["remark"] + " / " + msg + process_result["remark"] = process_result["remark"] + " | " + msg self.logger.feedback_info( "End of processing series: " + str(self.relevant_distances) @@ -436,7 +455,6 @@ def process( return self.dict_processresults - def predictor( self, dict_thematic=None, @@ -508,7 +526,7 @@ def predictor( - Debug logs the thematic element key being processed. """ if dict_thematic is None: - dict_thematic=self.dict_thematic + dict_thematic = self.dict_thematic dict_predictions = defaultdict(dict) dict_series = self.process( relevant_distances=relevant_distances, @@ -516,7 +534,6 @@ def predictor( threshold_overlap_percentage=threshold_overlap_percentage, ) - diffs_dict = diffs_from_dict_series(dict_series, dict_thematic) for theme_id, diffs in diffs_dict.items(): @@ -547,7 +564,10 @@ def predictor( for rel_dist, processresults in dist_results.items(): predicted_geom = processresults["result"] if not _equal_geom_in_array( - predicted_geom, predicted_geoms_for_theme_id,self.correction_distance,self.mitre_limit + predicted_geom, + predicted_geoms_for_theme_id, + self.correction_distance, + self.mitre_limit, ): dict_predictions_unique[theme_id][rel_dist] = processresults predicted_geoms_for_theme_id.append(processresults["result"]) @@ -564,7 +584,6 @@ def predictor( diffs_dict, ) - def compare( self, ids_to_compare=None, @@ -575,17 +594,19 @@ def compare( affected: list with all IDs to evaluate. all other IDs will be unchanged. If None (default), all self.dict_thematic will be evaluated. """ if ids_to_compare is None: - ids_to_compare =list(self.dict_thematic.keys()) - dict_affected={} - dict_unaffected={} - for id_theme,geom in self.dict_thematic.items(): + ids_to_compare = list(self.dict_thematic.keys()) + dict_affected = {} + dict_unaffected = {} + for id_theme, geom in self.dict_thematic.items(): if id_theme in ids_to_compare: dict_affected[id_theme] = geom else: dict_unaffected[id_theme] = geom - self.dict_thematic=dict_affected - #AFFECTED - dict_series, dict_affected_predictions, diffs = self.predictor(dict_thematic=dict_affected,relevant_distances=self.relevant_distances) + self.dict_thematic = dict_affected + # AFFECTED + dict_series, dict_affected_predictions, diffs = self.predictor( + dict_thematic=dict_affected, relevant_distances=self.relevant_distances + ) dict_predictions_evaluated = {} prop_dictionary = {} for theme_id, dict_predictions_results in dict_affected_predictions.items(): @@ -593,19 +614,24 @@ def compare( prop_dictionary[theme_id] = {} for dist in sorted(dict_predictions_results.keys()): prop_dictionary[theme_id][dist] = {} - props = self._evaluate(id_theme=theme_id,geom_predicted=dict_predictions_results[dist]["result"]) - dict_predictions_evaluated[theme_id][dist] = dict_affected_predictions[theme_id][dist] + props = self._evaluate( + id_theme=theme_id, + geom_predicted=dict_predictions_results[dist]["result"], + ) + dict_predictions_evaluated[theme_id][dist] = dict_affected_predictions[ + theme_id + ][dist] prop_dictionary[theme_id][dist] = props - #UNAFFECTED - relevant_distance=0 - #dict_unaffected_series = self.process(dict_thematic=dict_unaffected,relevant_distances=[relevant_distance]) - #for theme_id, dict_unaffected_results in dict_unaffected_series.items(): + # UNAFFECTED + relevant_distance = 0 + # dict_unaffected_series = self.process(dict_thematic=dict_unaffected,relevant_distances=[relevant_distance]) + # for theme_id, dict_unaffected_results in dict_unaffected_series.items(): for theme_id, geom in dict_unaffected.items(): dict_predictions_evaluated[theme_id] = {} - prop_dictionary[theme_id] = {relevant_distance:{}} - props = self._evaluate(id_theme=theme_id,geom_predicted=geom) - props[EVALUATION_FIELD_NAME]=Evaluation.NO_CHANGE_6 - dict_predictions_evaluated[theme_id][relevant_distance] = {"result":geom} + prop_dictionary[theme_id] = {relevant_distance: {}} + props = self._evaluate(id_theme=theme_id, geom_predicted=geom) + props[EVALUATION_FIELD_NAME] = Evaluation.NO_CHANGE_6 + dict_predictions_evaluated[theme_id][relevant_distance] = {"result": geom} prop_dictionary[theme_id][relevant_distance] = props self.dict_evaluated_predictions = dict_predictions_evaluated self.dict_evaluated_predictions_properties = prop_dictionary @@ -666,7 +692,7 @@ def get_brdr_formula(self, geometry: BaseGeometry, with_geom=False): and VERSION_DATE in self.dict_reference_properties[key_ref] ): str_version_date = self.dict_reference_properties[key_ref][VERSION_DATE] - version_date = datetime.strptime(str_version_date,DATE_FORMAT) + version_date = datetime.strptime(str_version_date, DATE_FORMAT) if last_version_date is None and version_date is not None: last_version_date = version_date if version_date is not None and version_date > last_version_date: @@ -705,9 +731,11 @@ def get_brdr_formula(self, geometry: BaseGeometry, with_geom=False): geom_od = buffer_pos( buffer_neg( safe_difference(geometry, safe_unary_union(intersected)), - self.correction_distance,mitre_limit=self.mitre_limit, + self.correction_distance, + mitre_limit=self.mitre_limit, ), - self.correction_distance,mitre_limit=self.mitre_limit, + self.correction_distance, + mitre_limit=self.mitre_limit, ) if geom_od is not None: area_od = round(geom_od.area, 2) @@ -722,7 +750,10 @@ def get_brdr_formula(self, geometry: BaseGeometry, with_geom=False): ########################################### def get_results_as_geojson( - self, resulttype=AlignerResultType.PROCESSRESULTS, formula=False, attributes=False + self, + resulttype=AlignerResultType.PROCESSRESULTS, + formula=False, + attributes=False, ): """ get a geojson of a dictionary containing the resulting geometries for all @@ -749,14 +780,18 @@ def get_results_as_geojson( for theme_id, results_dict in dict_series.items(): for relevant_distance, process_results in results_dict.items(): if relevant_distance not in prop_dictionary[theme_id]: - prop_dictionary[theme_id][relevant_distance]={} + prop_dictionary[theme_id][relevant_distance] = {} if attributes: - for attr,value in self.dict_thematic_properties[theme_id].items(): + for attr, value in self.dict_thematic_properties[theme_id].items(): prop_dictionary[theme_id][relevant_distance][attr] = value - if formula: #and not (theme_id in prop_dictionary and relevant_distance in prop_dictionary[theme_id] and NEW_FORMULA_FIELD_NAME in prop_dictionary[theme_id][relevant_distance]): + if ( + formula + ): # and not (theme_id in prop_dictionary and relevant_distance in prop_dictionary[theme_id] and NEW_FORMULA_FIELD_NAME in prop_dictionary[theme_id][relevant_distance]): result = process_results["result"] formula = self.get_brdr_formula(result) - prop_dictionary[theme_id][relevant_distance][NEW_FORMULA_FIELD_NAME] = json.dumps(formula) + prop_dictionary[theme_id][relevant_distance][ + NEW_FORMULA_FIELD_NAME + ] = json.dumps(formula) return get_series_geojson_dict( dict_series, crs=self.CRS, @@ -830,8 +865,7 @@ def save_results( def get_thematic_union(self): if self.thematic_union is None: - self.thematic_union = safe_unary_union(list(self.dict_thematic.values()) - ) + self.thematic_union = safe_unary_union(list(self.dict_thematic.values())) return self.thematic_union def _prepare_reference_data(self): @@ -865,9 +899,11 @@ def _prepare_reference_data(self): self.reference_union = None return - def _calculate_intersection_between_geometry_and_od(self, geometry,relevant_distance): + def _calculate_intersection_between_geometry_and_od( + self, geometry, relevant_distance + ): # Calculate the intersection between thematic and Openbaar Domein - buffer_distance = relevant_distance/2 + buffer_distance = relevant_distance / 2 relevant_intersection_array = [] relevant_difference_array = [] geom_thematic_od = Polygon() @@ -894,11 +930,17 @@ def _calculate_intersection_between_geometry_and_od(self, geometry,relevant_dist # geom of OD geom_od = safe_difference(geometry, self._get_reference_union()) # only the relevant parts of OD - geom_od_neg_pos = buffer_neg_pos(geom_od, buffer_distance,mitre_limit=self.mitre_limit,) + geom_od_neg_pos = buffer_neg_pos( + geom_od, + buffer_distance, + mitre_limit=self.mitre_limit, + ) # geom_thematic_od = safe_intersection(geom_od_neg_pos,geom_od)# resulting # thematic OD geom_od_neg_pos_buffered = buffer_pos( - geom_od_neg_pos, buffer_distance,mitre_limit=self.mitre_limit, + geom_od_neg_pos, + buffer_distance, + mitre_limit=self.mitre_limit, ) # include parts geom_thematic_od = safe_intersection( geom_od_neg_pos_buffered, geom_od @@ -912,7 +954,7 @@ def _calculate_intersection_between_geometry_and_od(self, geometry,relevant_dist geom_thematic_od, relevant_difference_array, relevant_intersection_array, - ) = self._od_snap_all_side(geometry,relevant_distance) + ) = self._od_snap_all_side(geometry, relevant_distance) elif self.od_strategy == OpenbaarDomeinStrategy.SNAP_FULL_AREA_SINGLE_SIDE: # Strategy useful for bigger areas. # integrates the entire inner area of the input geometry, @@ -921,7 +963,7 @@ def _calculate_intersection_between_geometry_and_od(self, geometry,relevant_dist self.logger.feedback_debug( "OD-strategy Full-area-variant of OD-SNAP_SINGLE_SIDE" ) - geom_thematic_od = self._od_full_area(geometry,relevant_distance) + geom_thematic_od = self._od_full_area(geometry, relevant_distance) elif self.od_strategy == OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE: # Strategy useful for bigger areas. # integrates the entire inner area of the input geometry, @@ -935,10 +977,10 @@ def _calculate_intersection_between_geometry_and_od(self, geometry,relevant_dist geom_thematic_od, relevant_difference_array, relevant_intersection_array, - ) = self._od_snap_all_side(geometry,relevant_distance) + ) = self._od_snap_all_side(geometry, relevant_distance) # This part is a copy of SNAP_FULL_AREA_SINGLE_SIDE geom_theme_od_min_clipped_plus_buffered_clipped = self._od_full_area( - geometry,relevant_distance + geometry, relevant_distance ) # UNION the calculation of OD-SNAP_ALL_SIDE with FULL AREA of # OD-SNAP_FULL_AREA_SINGLE_SIDE @@ -956,7 +998,11 @@ def _calculate_intersection_between_geometry_and_od(self, geometry,relevant_dist # geom of OD geom_od = safe_difference(geometry, self._get_reference_union()) # only the relevant parts of OD - geom_od_neg_pos = buffer_neg_pos(geom_od, buffer_distance,mitre_limit=self.mitre_limit,) + geom_od_neg_pos = buffer_neg_pos( + geom_od, + buffer_distance, + mitre_limit=self.mitre_limit, + ) # resulting thematic OD geom_thematic_od = safe_intersection(geom_od_neg_pos, geom_od) elif self.od_strategy == OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE_VARIANT_2: @@ -978,21 +1024,29 @@ def _calculate_intersection_between_geometry_and_od(self, geometry,relevant_dist relevant_difference_array, ) - def _od_full_area(self, geometry,relevant_distance): - buffer_distance=relevant_distance/2 + def _od_full_area(self, geometry, relevant_distance): + buffer_distance = relevant_distance / 2 geom_theme_od = safe_difference(geometry, self._get_reference_union()) geom_theme_min_buffered = buffer_neg( buffer_pos( - buffer_neg(geometry, relevant_distance,mitre_limit=self.mitre_limit,), - buffer_distance,mitre_limit=self.mitre_limit, + buffer_neg( + geometry, + relevant_distance, + mitre_limit=self.mitre_limit, + ), + buffer_distance, + mitre_limit=self.mitre_limit, ), - buffer_distance,mitre_limit=self.mitre_limit, + buffer_distance, + mitre_limit=self.mitre_limit, ) geom_theme_od_clipped_min_buffered = safe_intersection( geom_theme_min_buffered, geom_theme_od ) geom_theme_od_min_clipped_plus_buffered = buffer_pos( - geom_theme_od_clipped_min_buffered, relevant_distance,mitre_limit=self.mitre_limit, + geom_theme_od_clipped_min_buffered, + relevant_distance, + mitre_limit=self.mitre_limit, ) geom_theme_od_min_clipped_plus_buffered_clipped = safe_intersection( geom_theme_od_min_clipped_plus_buffered, geom_theme_od @@ -1000,12 +1054,16 @@ def _od_full_area(self, geometry,relevant_distance): geom_thematic_od = geom_theme_od_min_clipped_plus_buffered_clipped return geom_thematic_od - def _od_snap_all_side(self, geometry,relevant_distance): - buffer_distance = relevant_distance/2 + def _od_snap_all_side(self, geometry, relevant_distance): + buffer_distance = relevant_distance / 2 relevant_difference_array = [] relevant_intersection_array = [] geom_thematic_buffered = make_valid( - buffer_pos(geometry, self.buffer_multiplication_factor * relevant_distance,mitre_limit=self.mitre_limit,) + buffer_pos( + geometry, + self.buffer_multiplication_factor * relevant_distance, + mitre_limit=self.mitre_limit, + ) ) clip_ref_thematic_buffered = safe_intersection( self._get_reference_union(), geom_thematic_buffered @@ -1030,7 +1088,7 @@ def _od_snap_all_side(self, geometry,relevant_distance): self.threshold_overlap_percentage, self.threshold_exclusion_percentage, self.threshold_exclusion_area, - self.mitre_limit + self.mitre_limit, ) relevant_intersection_array = self._add_multi_polygons_from_geom_to_array( geom_relevant_intersection, [] @@ -1042,11 +1100,12 @@ def _od_snap_all_side(self, geometry,relevant_distance): def _get_reference_union(self): if self.reference_union is None: - self.reference_union = safe_unary_union(list(self.dict_reference.values()) - ) + self.reference_union = safe_unary_union(list(self.dict_reference.values())) return self.reference_union - def _postprocess_preresult(self, preresult, geom_thematic,relevant_distance) -> ProcessResult: + def _postprocess_preresult( + self, preresult, geom_thematic, relevant_distance + ) -> ProcessResult: """ Postprocess the preresult with the following actions to create the final result *Corrections for areas that differ more than the relevant distance @@ -1072,7 +1131,7 @@ def _postprocess_preresult(self, preresult, geom_thematic,relevant_distance) -> """ # Process array remark = "" - buffer_distance = relevant_distance/2 + buffer_distance = relevant_distance / 2 result = [] geom_preresult = safe_unary_union(preresult) geom_thematic = make_valid(geom_thematic) @@ -1087,21 +1146,27 @@ def _postprocess_preresult(self, preresult, geom_thematic,relevant_distance) -> ): remark = "Circle detected: -->resulting geometry = original geometry" self.logger.feedback_debug(remark) - return {"result": geom_thematic,"remark": remark} + return {"result": geom_thematic, "remark": remark} # Correction for unchanged geometries if geom_preresult == geom_thematic: remark = "Unchanged geometry: -->resulting geometry = original geometry" self.logger.feedback_debug(remark) - return {"result": geom_thematic,"remark": remark} + return {"result": geom_thematic, "remark": remark} # Corrections for areas that differ more than the relevant distance geom_thematic_dissolved = buffer_pos( buffer_neg( - buffer_pos(geom_preresult, self.correction_distance,mitre_limit=self.mitre_limit,), - 2 * self.correction_distance,mitre_limit=self.mitre_limit, + buffer_pos( + geom_preresult, + self.correction_distance, + mitre_limit=self.mitre_limit, + ), + 2 * self.correction_distance, + mitre_limit=self.mitre_limit, ), - self.correction_distance,mitre_limit=self.mitre_limit, + self.correction_distance, + mitre_limit=self.mitre_limit, ) # geom_symdiff = self._safe_symmetric_difference(geom_thematic, # geom_thematic_dissolved) @@ -1111,22 +1176,36 @@ def _postprocess_preresult(self, preresult, geom_thematic,relevant_distance) -> geom_thematic_dissolved, safe_intersection( geom_diff_delete, - buffer_neg_pos(geom_diff_delete, buffer_distance,mitre_limit=self.mitre_limit,), + buffer_neg_pos( + geom_diff_delete, + buffer_distance, + mitre_limit=self.mitre_limit, + ), ), ) geom_diff_removed_added = safe_union( geom_diff_removed, safe_intersection( geom_diff_add, - buffer_neg_pos(geom_diff_add, buffer_distance,mitre_limit=self.mitre_limit,), + buffer_neg_pos( + geom_diff_add, + buffer_distance, + mitre_limit=self.mitre_limit, + ), ), ) geom_thematic_preresult = buffer_pos( buffer_neg( - buffer_pos(geom_diff_removed_added, self.correction_distance,mitre_limit=self.mitre_limit,), - 2 * self.correction_distance,mitre_limit=self.mitre_limit, + buffer_pos( + geom_diff_removed_added, + self.correction_distance, + mitre_limit=self.mitre_limit, + ), + 2 * self.correction_distance, + mitre_limit=self.mitre_limit, ), - self.correction_distance,mitre_limit=self.mitre_limit, + self.correction_distance, + mitre_limit=self.mitre_limit, ) # Correction for Inner holes(donuts) / multipolygons # fill and remove gaps @@ -1135,10 +1214,16 @@ def _postprocess_preresult(self, preresult, geom_thematic,relevant_distance) -> ) geom_thematic_result = buffer_pos( buffer_neg( - buffer_pos(geom_thematic_cleaned_holes, self.correction_distance,mitre_limit=self.mitre_limit,), - 2 * self.correction_distance,mitre_limit=self.mitre_limit, + buffer_pos( + geom_thematic_cleaned_holes, + self.correction_distance, + mitre_limit=self.mitre_limit, + ), + 2 * self.correction_distance, + mitre_limit=self.mitre_limit, ), - self.correction_distance,mitre_limit=self.mitre_limit, + self.correction_distance, + mitre_limit=self.mitre_limit, ) geom_thematic_result = make_valid(remove_repeated_points(geom_thematic_result)) @@ -1160,35 +1245,49 @@ def _postprocess_preresult(self, preresult, geom_thematic,relevant_distance) -> # remove 'very small' differences (smaller than the correction distance) geom_result_diff = buffer_pos( buffer_neg( - safe_symmetric_difference(geom_thematic_result, geom_thematic,), - self.correction_distance,mitre_limit=self.mitre_limit, + safe_symmetric_difference( + geom_thematic_result, + geom_thematic, + ), + self.correction_distance, + mitre_limit=self.mitre_limit, ), - self.correction_distance,mitre_limit=self.mitre_limit, + self.correction_distance, + mitre_limit=self.mitre_limit, ) geom_result_diff_plus = buffer_pos( buffer_neg( - safe_difference(geom_thematic_result, geom_thematic,), - self.correction_distance,mitre_limit=self.mitre_limit, + safe_difference( + geom_thematic_result, + geom_thematic, + ), + self.correction_distance, + mitre_limit=self.mitre_limit, ), - self.correction_distance,mitre_limit=self.mitre_limit, + self.correction_distance, + mitre_limit=self.mitre_limit, ) geom_result_diff_min = buffer_pos( buffer_neg( - safe_difference(geom_thematic, geom_thematic_result,), - self.correction_distance,mitre_limit=self.mitre_limit, + safe_difference( + geom_thematic, + geom_thematic_result, + ), + self.correction_distance, + mitre_limit=self.mitre_limit, ), - self.correction_distance,mitre_limit=self.mitre_limit, + self.correction_distance, + mitre_limit=self.mitre_limit, ) # geom_result_diff_plus = safe_difference(geom_thematic_result, geom_thematic) # geom_result_diff_min = safe_difference(geom_thematic, geom_thematic_result) - return { "result": geom_thematic_result, "result_diff": geom_result_diff, "result_diff_plus": geom_result_diff_plus, "result_diff_min": geom_result_diff_min, - "remark": remark + "remark": remark, } def _evaluate(self, id_theme, geom_predicted): @@ -1196,8 +1295,8 @@ def _evaluate(self, id_theme, geom_predicted): function that evaluates a predicted geometry and returns a properties-dictionary """ threshold_od_percentage = 1 - #threshold_area = 5 - #threshold_percentage = 1 + # threshold_area = 5 + # threshold_percentage = 1 properties = { NEW_FORMULA_FIELD_NAME: "", EVALUATION_FIELD_NAME: Evaluation.TO_CHECK_4, @@ -1207,20 +1306,17 @@ def _evaluate(self, id_theme, geom_predicted): EQUAL_REFERENCE_FEATURES_FIELD_NAME: None, DIFF_PERCENTAGE_FIELD_NAME: None, DIFF_AREA_FIELD_NAME: None, - } actual_formula = self.get_brdr_formula(geom_predicted) - properties[NEW_FORMULA_FIELD_NAME] = json.dumps( - actual_formula - ) + properties[NEW_FORMULA_FIELD_NAME] = json.dumps(actual_formula) base_formula = None if ( - id_theme in self.dict_thematic_properties - and BASE_FORMULA_FIELD_NAME in self.dict_thematic_properties[id_theme] + id_theme in self.dict_thematic_properties + and BASE_FORMULA_FIELD_NAME in self.dict_thematic_properties[id_theme] ): - base_formula = json.loads(self.dict_thematic_properties[id_theme][ - BASE_FORMULA_FIELD_NAME - ]) + base_formula = json.loads( + self.dict_thematic_properties[id_theme][BASE_FORMULA_FIELD_NAME] + ) if base_formula is None or actual_formula is None: properties[EVALUATION_FIELD_NAME] = Evaluation.NO_PREDICTION_5 @@ -1228,33 +1324,39 @@ def _evaluate(self, id_theme, geom_predicted): properties[FULL_BASE_FIELD_NAME] = base_formula["full"] properties[FULL_ACTUAL_FIELD_NAME] = actual_formula["full"] od_alike = False - if base_formula["reference_od"] is None and actual_formula["reference_od"] is None: + if ( + base_formula["reference_od"] is None + and actual_formula["reference_od"] is None + ): od_alike = True - elif base_formula["reference_od"] is None or actual_formula["reference_od"] is None: + elif ( + base_formula["reference_od"] is None + or actual_formula["reference_od"] is None + ): od_alike = False elif ( - abs( - base_formula["reference_od"]["area"] - - actual_formula["reference_od"]["area"] - ) - * 100 - / base_formula["reference_od"]["area"] + abs( + base_formula["reference_od"]["area"] + - actual_formula["reference_od"]["area"] + ) + * 100 + / base_formula["reference_od"]["area"] ) < threshold_od_percentage: od_alike = True properties[OD_ALIKE_FIELD_NAME] = od_alike equal_reference_features = True if ( - base_formula["reference_features"].keys() - == actual_formula["reference_features"].keys() + base_formula["reference_features"].keys() + == actual_formula["reference_features"].keys() ): equal_reference_features = True max_diff_area_reference_feature = 0 max_diff_percentage_reference_feature = 0 for key in base_formula["reference_features"].keys(): if ( - base_formula["reference_features"][key]["full"] - != actual_formula["reference_features"][key]["full"] + base_formula["reference_features"][key]["full"] + != actual_formula["reference_features"][key]["full"] ): equal_reference_features = False @@ -1263,28 +1365,44 @@ def _evaluate(self, id_theme, geom_predicted): - actual_formula["reference_features"][key]["area"] ) diff_percentage_reference_feature = ( - abs( - base_formula["reference_features"][key]["area"] - - actual_formula["reference_features"][key]["area"] - ) - * 100 - / base_formula["reference_features"][key]["area"] + abs( + base_formula["reference_features"][key]["area"] + - actual_formula["reference_features"][key]["area"] + ) + * 100 + / base_formula["reference_features"][key]["area"] ) if diff_area_reference_feature > max_diff_area_reference_feature: max_diff_area_reference_feature = diff_area_reference_feature - if diff_percentage_reference_feature > max_diff_percentage_reference_feature: - max_diff_percentage_reference_feature = diff_percentage_reference_feature + if ( + diff_percentage_reference_feature + > max_diff_percentage_reference_feature + ): + max_diff_percentage_reference_feature = ( + diff_percentage_reference_feature + ) properties[EQUAL_REFERENCE_FEATURES_FIELD_NAME] = equal_reference_features properties[DIFF_AREA_FIELD_NAME] = max_diff_area_reference_feature - properties[DIFF_PERCENTAGE_FIELD_NAME] = max_diff_percentage_reference_feature - #EVALUATION - if equal_reference_features and od_alike and base_formula["full"] and actual_formula["full"]: + properties[DIFF_PERCENTAGE_FIELD_NAME] = ( + max_diff_percentage_reference_feature + ) + # EVALUATION + if ( + equal_reference_features + and od_alike + and base_formula["full"] + and actual_formula["full"] + ): properties[EVALUATION_FIELD_NAME] = Evaluation.EQUALITY_EQUAL_FORMULA_FULL_1 - elif equal_reference_features and od_alike and base_formula["full"] == actual_formula["full"]: + elif ( + equal_reference_features + and od_alike + and base_formula["full"] == actual_formula["full"] + ): properties[EVALUATION_FIELD_NAME] = Evaluation.EQUALITY_EQUAL_FORMULA_2 elif base_formula["full"] and actual_formula["full"] and od_alike: properties[EVALUATION_FIELD_NAME] = Evaluation.EQUALITY_FULL_3 - #elif base_formula["full"] == actual_formula["full"] and od_alike:#TODO evaluate when not-full-parcels? + # elif base_formula["full"] == actual_formula["full"] and od_alike:#TODO evaluate when not-full-parcels? # properties[EVALUATION_FIELD_NAME] = Evaluation.EQUALITY_NON_FULL else: properties[EVALUATION_FIELD_NAME] = Evaluation.TO_CHECK_4 @@ -1327,7 +1445,7 @@ def _calculate_geom_by_intersection_and_reference( threshold_overlap_percentage, threshold_exclusion_percentage, threshold_exclusion_area, - mitre_limit, + mitre_limit, ): """ Calculates the geometry based on intersection and reference geometries. @@ -1377,8 +1495,16 @@ def _calculate_geom_by_intersection_and_reference( return Polygon(), Polygon(), Polygon() geom_difference = safe_difference(geom_reference, geom_intersection) - geom_relevant_intersection = buffer_neg(geom_intersection, buffer_distance,mitre_limit=mitre_limit,) - geom_relevant_difference = buffer_neg(geom_difference, buffer_distance,mitre_limit=mitre_limit,) + geom_relevant_intersection = buffer_neg( + geom_intersection, + buffer_distance, + mitre_limit=mitre_limit, + ) + geom_relevant_difference = buffer_neg( + geom_difference, + buffer_distance, + mitre_limit=mitre_limit, + ) if ( not geom_relevant_intersection.is_empty and not geom_relevant_difference.is_empty @@ -1390,15 +1516,24 @@ def _calculate_geom_by_intersection_and_reference( geom_reference, safe_intersection( geom_difference, - buffer_neg_pos(geom_difference, buffer_distance,mitre_limit=mitre_limit,), + buffer_neg_pos( + geom_difference, + buffer_distance, + mitre_limit=mitre_limit, + ), ), ), ) geom = safe_intersection( geom_x, buffer_pos( - buffer_neg_pos(geom_x, buffer_distance,mitre_limit=mitre_limit,), - buffer_distance,mitre_limit=mitre_limit, + buffer_neg_pos( + geom_x, + buffer_distance, + mitre_limit=mitre_limit, + ), + buffer_distance, + mitre_limit=mitre_limit, ), ) # when calculating for OD, we create a 'virtual parcel'. When calculating this @@ -1407,7 +1542,7 @@ def _calculate_geom_by_intersection_and_reference( # in the result. The function below tries to exclude these non-logical parts. # see eo_id 206363 with relevant distance=0.2m and SNAP_ALL_SIDE if is_openbaar_domein: - geom = _get_relevant_polygons_from_geom(geom, buffer_distance,mitre_limit) + geom = _get_relevant_polygons_from_geom(geom, buffer_distance, mitre_limit) elif not geom_relevant_intersection.is_empty and geom_relevant_difference.is_empty: geom = geom_reference elif geom_relevant_intersection.is_empty and not geom_relevant_difference.is_empty: @@ -1427,7 +1562,9 @@ def _calculate_geom_by_intersection_and_reference( return geom, geom_relevant_intersection, geom_relevant_difference -def _get_relevant_polygons_from_geom(geometry: BaseGeometry, buffer_distance: float,mitre_limit): +def _get_relevant_polygons_from_geom( + geometry: BaseGeometry, buffer_distance: float, mitre_limit +): """ Get only the relevant parts (polygon) from a geometry. Points, Lines and Polygons smaller than relevant distance are excluded from the @@ -1445,13 +1582,17 @@ def _get_relevant_polygons_from_geom(geometry: BaseGeometry, buffer_distance: fl # Ensure each sub-geometry is valid. g = make_valid(g) if str(g.geom_type) in ["Polygon", "MultiPolygon"]: - relevant_geom = buffer_neg(g, buffer_distance,mitre_limit=mitre_limit,) + relevant_geom = buffer_neg( + g, + buffer_distance, + mitre_limit=mitre_limit, + ) if relevant_geom is not None and not relevant_geom.is_empty: array.append(g) return safe_unary_union(array) -def _equal_geom_in_array(geom, geom_array,correction_distance,mitre_limit): +def _equal_geom_in_array(geom, geom_array, correction_distance, mitre_limit): """ Check if a predicted geometry is equal to other predicted geometries in a list. Equality is defined as there is the symmetrical difference is smaller than the CORRECTION DISTANCE @@ -1459,7 +1600,11 @@ def _equal_geom_in_array(geom, geom_array,correction_distance,mitre_limit): """ for g in geom_array: # if safe_equals(geom,g): - if buffer_neg(safe_symmetric_difference(geom, g), correction_distance,mitre_limit=mitre_limit,).is_empty: + if buffer_neg( + safe_symmetric_difference(geom, g), + correction_distance, + mitre_limit=mitre_limit, + ).is_empty: return True return False @@ -1526,4 +1671,4 @@ def _check_equality( return True, Evaluation.EQUALITY_EQUAL_FORMULA_2 if base_formula["full"] and actual_formula["full"] and od_alike: return True, Evaluation.EQUALITY_FULL_3 - return False, Evaluation.NO_PREDICTION_5 \ No newline at end of file + return False, Evaluation.NO_PREDICTION_5 diff --git a/brdr/grb.py b/brdr/grb.py index 0939e0d..1a1f784 100644 --- a/brdr/grb.py +++ b/brdr/grb.py @@ -148,7 +148,11 @@ def get_affected_by_grb_change( ) logging.info("Number of filtered features: " + str(len(thematic_intersections))) for key, geom in dict_thematic.items(): - affected.append(key) if key in thematic_intersections else unaffected.append(key) + ( + affected.append(key) + if key in thematic_intersections + else unaffected.append(key) + ) return affected, unaffected @@ -358,13 +362,17 @@ def update_to_actual_grb( for feature in featurecollection["features"]: id_theme = feature["properties"][id_theme_fieldname] geom = shape(feature["geometry"]) - #logger.feedback_debug("id theme: " + id_theme) - #logger.feedback_debug("geometry (wkt): " + geom.wkt) + # logger.feedback_debug("id theme: " + id_theme) + # logger.feedback_debug("geometry (wkt): " + geom.wkt) dict_thematic[id_theme] = geom dict_thematic_props[id_theme] = feature["properties"] try: - dict_thematic_props[id_theme][BASE_FORMULA_FIELD_NAME] =feature["properties"][base_formula_field] - base_formula = json.loads(dict_thematic_props[id_theme][BASE_FORMULA_FIELD_NAME]) + dict_thematic_props[id_theme][BASE_FORMULA_FIELD_NAME] = feature[ + "properties" + ][base_formula_field] + base_formula = json.loads( + dict_thematic_props[id_theme][BASE_FORMULA_FIELD_NAME] + ) logger.feedback_debug("formula: " + str(base_formula)) except Exception: raise Exception("Formula -attribute-field (json) cannot be loaded") @@ -372,14 +380,10 @@ def update_to_actual_grb( logger.feedback_debug(str(dict_thematic_props[id_theme])) if ( LAST_VERSION_DATE in base_formula - and base_formula[LAST_VERSION_DATE] - is not None - and base_formula[LAST_VERSION_DATE] - != "" + and base_formula[LAST_VERSION_DATE] is not None + and base_formula[LAST_VERSION_DATE] != "" ): - str_lvd = base_formula[ - LAST_VERSION_DATE - ] + str_lvd = base_formula[LAST_VERSION_DATE] lvd = datetime.strptime(str_lvd, DATE_FORMAT).date() if lvd < last_version_date: last_version_date = lvd @@ -400,10 +404,9 @@ def update_to_actual_grb( one_by_one=False, geometry_thematic_union=base_aligner_result.get_thematic_union(), crs=base_aligner_result.CRS, - ) + ) logger.feedback_info( - "Number of possible affected OE-thematic during timespan: " - + str(len(affected)) + "Number of possible affected OE-thematic during timespan: " + str(len(affected)) ) if len(affected) == 0: logger.feedback_info( @@ -414,7 +417,7 @@ def update_to_actual_grb( logger.feedback_debug(str(base_formula_field)) # Initiate a Aligner to reference thematic features to the actual borders - actual_aligner = Aligner(feedback=feedback,max_workers=None) + actual_aligner = Aligner(feedback=feedback, max_workers=None) actual_aligner.load_thematic_data( DictLoader(data_dict=dict_thematic, data_dict_properties=dict_thematic_props) ) @@ -425,9 +428,13 @@ def update_to_actual_grb( actual_aligner.relevant_distances = ( np.arange(0, max_distance_for_actualisation * 100, 10, dtype=int) / 100 ) - dict_evaluated, prop_dictionary = actual_aligner.compare(ids_to_compare= affected) + dict_evaluated, prop_dictionary = actual_aligner.compare(ids_to_compare=affected) - return actual_aligner.get_results_as_geojson(resulttype=AlignerResultType.EVALUATED_PREDICTIONS,formula=True,attributes=attributes) + return actual_aligner.get_results_as_geojson( + resulttype=AlignerResultType.EVALUATED_PREDICTIONS, + formula=True, + attributes=attributes, + ) class GRBActualLoader(GeoJsonLoader): @@ -442,7 +449,9 @@ def __init__(self, grb_type: GRBType, aligner, partition: int = 1000): def load_data(self): if not self.aligner.dict_thematic: raise ValueError("Thematic data not loaded") - geom_union = buffer_pos(self.aligner.get_thematic_union(), GRB_MAX_REFERENCE_BUFFER) + geom_union = buffer_pos( + self.aligner.get_thematic_union(), GRB_MAX_REFERENCE_BUFFER + ) collection, id_property = get_collection_grb_actual( grb_type=self.grb_type, geometry=geom_union, @@ -471,7 +480,9 @@ def __init__(self, year: str, aligner, partition=1000): def load_data(self): if not self.aligner.dict_thematic: raise ValueError("Thematic data not loaded") - geom_union = buffer_pos(self.aligner.get_thematic_union(), GRB_MAX_REFERENCE_BUFFER) + geom_union = buffer_pos( + self.aligner.get_thematic_union(), GRB_MAX_REFERENCE_BUFFER + ) collection = get_collection_grb_fiscal_parcels( year=self.year, geometry=geom_union, @@ -485,7 +496,9 @@ def load_data(self): class GRBSpecificDateParcelLoader(GeoJsonLoader): def __init__(self, date, aligner, partition=1000): - logging.warning("Loader for GRB parcel-situation on specific date (experimental); Use it with care!!!") + logging.warning( + "Loader for GRB parcel-situation on specific date (experimental); Use it with care!!!" + ) try: date = datetime.strptime(date, DATE_FORMAT).date() if date.year >= datetime.now().year: @@ -507,7 +520,9 @@ def __init__(self, date, aligner, partition=1000): def load_data(self): if not self.aligner.dict_thematic: raise ValueError("Thematic data not loaded") - geom_union = buffer_pos(self.aligner.get_thematic_union(), GRB_MAX_REFERENCE_BUFFER) + geom_union = buffer_pos( + self.aligner.get_thematic_union(), GRB_MAX_REFERENCE_BUFFER + ) collection = get_collection_grb_parcels_by_date( date=self.date, geometry=geom_union, diff --git a/brdr/utils.py b/brdr/utils.py index c3a2e65..86ad429 100644 --- a/brdr/utils.py +++ b/brdr/utils.py @@ -522,9 +522,9 @@ def merge_process_results( for rel_dist, process_result in dict_results.items(): for key in process_result: value = process_result[key] # noqa - if isinstance(value,str): + if isinstance(value,str) and value !="": existing_remark: str = grouped_results[id_theme_global][rel_dist][ key] # noqa - grouped_results[id_theme_global][rel_dist][key] = existing_remark + '|' + str(value) + grouped_results[id_theme_global][rel_dist][key] = existing_remark + " | " + str(value) continue elif isinstance(value,BaseGeometry): geom = value diff --git a/examples/example_multipolygon.py b/examples/example_multipolygon.py index e189d51..0bb9450 100644 --- a/examples/example_multipolygon.py +++ b/examples/example_multipolygon.py @@ -13,7 +13,9 @@ aligner0.load_thematic_data( GeoJsonFileLoader("../tests/testdata/multipolygon.geojson", "theme_identifier") ) -aligner0.dict_thematic,dict_multi_as_single = multipolygons_to_singles(aligner0.dict_thematic) +aligner0.dict_thematic, dict_multi_as_single = multipolygons_to_singles( + aligner0.dict_thematic +) aligner0.load_thematic_data( DictLoader( aligner0.dict_thematic, diff --git a/examples/example_parcel_change_detector.py b/examples/example_parcel_change_detector.py index 8cc7f56..33d3a48 100644 --- a/examples/example_parcel_change_detector.py +++ b/examples/example_parcel_change_detector.py @@ -2,8 +2,7 @@ from datetime import datetime from brdr.aligner import Aligner -from brdr.constants import EVALUATION_FIELD_NAME, RELEVANT_DISTANCE_FIELD_NAME, BASE_FORMULA_FIELD_NAME, \ - NEW_FORMULA_FIELD_NAME +from brdr.constants import EVALUATION_FIELD_NAME, RELEVANT_DISTANCE_FIELD_NAME, NEW_FORMULA_FIELD_NAME from brdr.grb import GRBFiscalParcelLoader from brdr.grb import update_to_actual_grb from brdr.oe import OnroerendErfgoedLoader diff --git a/examples/example_update_to_actual_grb.py b/examples/example_update_to_actual_grb.py index 5dbc7e2..832ec5b 100644 --- a/examples/example_update_to_actual_grb.py +++ b/examples/example_update_to_actual_grb.py @@ -59,12 +59,14 @@ GRBFiscalParcelLoader(year=base_year, aligner=base_aligner) ) base_process_result = base_aligner.process(relevant_distance=2) -fcs = base_aligner.get_results_as_geojson(formula=True,attributes=True) +fcs = base_aligner.get_results_as_geojson(formula=True, attributes=True) featurecollection_base_result = fcs["result"] print(featurecollection_base_result) # Update Featurecollection to actual version featurecollection = update_to_actual_grb( - featurecollection_base_result, base_aligner.name_thematic_id,base_formula_field=NEW_FORMULA_FIELD_NAME + featurecollection_base_result, + base_aligner.name_thematic_id, + base_formula_field=NEW_FORMULA_FIELD_NAME, ) # Print results for feature in featurecollection["result"]["features"]: diff --git a/tests/test_aligner.py b/tests/test_aligner.py index c8239e5..48bf207 100644 --- a/tests/test_aligner.py +++ b/tests/test_aligner.py @@ -7,7 +7,6 @@ from shapely import from_wkt from shapely.geometry import Polygon from shapely.geometry import shape -from shapely.geometry.multipolygon import MultiPolygon from shapely.predicates import equals from brdr.aligner import Aligner From 250c5bc87248aba3de226eac87d479c5c570ec03 Mon Sep 17 00:00:00 2001 From: dieuska Date: Wed, 2 Oct 2024 14:46:49 +0200 Subject: [PATCH 19/26] str to any (for flexible IDs) --- brdr/loader.py | 8 ++++---- brdr/utils.py | 14 +++++++------- examples/__init__.py | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/brdr/loader.py b/brdr/loader.py index ad1ba0f..29a78a5 100644 --- a/brdr/loader.py +++ b/brdr/loader.py @@ -14,10 +14,10 @@ class Loader(ABC): def __init__(self): - self.data_dict: dict[str, BaseGeometry] = {} - self.data_dict_properties: dict[str, dict] = {} - self.data_dict_source: dict[str, str] = {} - self.versiondate_info: dict[str, str] = None + self.data_dict: dict[any, BaseGeometry] = {} + self.data_dict_properties: dict[any, dict] = {} + self.data_dict_source: dict[any, str] = {} + self.versiondate_info: dict[any, str] = None def load_data(self): self.data_dict = {x: make_valid(self.data_dict[x]) for x in self.data_dict} diff --git a/brdr/utils.py b/brdr/utils.py index 86ad429..04f3b33 100644 --- a/brdr/utils.py +++ b/brdr/utils.py @@ -27,10 +27,10 @@ def get_series_geojson_dict( - series_dict: dict[str, dict[float, ProcessResult]], + series_dict: dict[any, dict[float, ProcessResult]], crs: str, id_field: str, - series_prop_dict: dict[str, dict[float, any]] = None, + series_prop_dict: dict[any, dict[float, any]] = None, geom_attributes=True, ): """ @@ -317,8 +317,8 @@ def _numerical_derivative(x, y): def diffs_from_dict_series( - dict_series: dict[str, dict[float, ProcessResult]], - dict_thematic: dict[str, BaseGeometry], + dict_series: dict[any, dict[float, ProcessResult]], + dict_thematic: dict[any, BaseGeometry], diff_metric: DiffMetric = DiffMetric.CHANGES_AREA, ): """ @@ -496,9 +496,9 @@ def _add_bbox_to_url(url, crs=DEFAULT_CRS, bbox=None): def merge_process_results( - result_dict: dict[str, dict[float, ProcessResult]], + result_dict: dict[any, dict[float, ProcessResult]], dict_multi_as_single:dict -) -> dict[str, dict[float, ProcessResult]]: +) -> dict[any, dict[float, ProcessResult]]: """ Merges processresults in a dictionary from multiple themeIDs into a single themeID. @@ -509,7 +509,7 @@ def merge_process_results( theme IDs and values are merged geometries and remarks. """ - grouped_results: dict[str, dict[float, ProcessResult]] = {} + grouped_results: dict[any, dict[float, ProcessResult]] = {} for id_theme, dict_results in result_dict.items(): if id_theme in dict_multi_as_single.keys(): diff --git a/examples/__init__.py b/examples/__init__.py index 64baa2f..5a7f239 100644 --- a/examples/__init__.py +++ b/examples/__init__.py @@ -89,7 +89,7 @@ def _make_map(ax, processresult, thematic_dict, reference_dict): def show_map( - dict_results: dict[str, dict[float, ProcessResult]], + dict_results: dict[any, dict[float, ProcessResult]], dict_thematic, dict_reference, ): From 75f2793badfafe144a9a4b225f90c7e33baa6a9b Mon Sep 17 00:00:00 2001 From: dieuska Date: Thu, 3 Oct 2024 09:28:00 +0200 Subject: [PATCH 20/26] new_formula to formula --- brdr/aligner.py | 14 +- brdr/constants.py | 2 +- brdr/utils.py | 216 +++++++++------------ examples/example_parcel_change_detector.py | 4 +- examples/example_update_to_actual_grb.py | 4 +- tests/test_aligner.py | 4 +- 6 files changed, 107 insertions(+), 137 deletions(-) diff --git a/brdr/aligner.py b/brdr/aligner.py index 69298aa..13816fb 100644 --- a/brdr/aligner.py +++ b/brdr/aligner.py @@ -24,7 +24,7 @@ LAST_VERSION_DATE, VERSION_DATE, DATE_FORMAT, - NEW_FORMULA_FIELD_NAME, + FORMULA_FIELD_NAME, EVALUATION_FIELD_NAME, FULL_BASE_FIELD_NAME, FULL_ACTUAL_FIELD_NAME, @@ -63,11 +63,11 @@ class Aligner: """ - This class is used to compare the thematic data with the reference data. + This class is used to compare and align the thematic data with the reference data. The reference data can be loaded in different ways, for example by using the GRB data. - The thematic data can be loaded by using a geojson file. - The class can be used to compare the thematic data with the reference data. + The thematic data can be loaded by using different Loaders: DictLoader, GeojsonLoader,... + The class can be used to compare and aligne the thematic data with the reference data. """ def __init__( @@ -790,7 +790,7 @@ def get_results_as_geojson( result = process_results["result"] formula = self.get_brdr_formula(result) prop_dictionary[theme_id][relevant_distance][ - NEW_FORMULA_FIELD_NAME + FORMULA_FIELD_NAME ] = json.dumps(formula) return get_series_geojson_dict( dict_series, @@ -1298,7 +1298,7 @@ def _evaluate(self, id_theme, geom_predicted): # threshold_area = 5 # threshold_percentage = 1 properties = { - NEW_FORMULA_FIELD_NAME: "", + FORMULA_FIELD_NAME: "", EVALUATION_FIELD_NAME: Evaluation.TO_CHECK_4, FULL_BASE_FIELD_NAME: None, FULL_ACTUAL_FIELD_NAME: None, @@ -1308,7 +1308,7 @@ def _evaluate(self, id_theme, geom_predicted): DIFF_AREA_FIELD_NAME: None, } actual_formula = self.get_brdr_formula(geom_predicted) - properties[NEW_FORMULA_FIELD_NAME] = json.dumps(actual_formula) + properties[FORMULA_FIELD_NAME] = json.dumps(actual_formula) base_formula = None if ( id_theme in self.dict_thematic_properties diff --git a/brdr/constants.py b/brdr/constants.py index d49a1ad..5eac700 100644 --- a/brdr/constants.py +++ b/brdr/constants.py @@ -9,7 +9,7 @@ PREFIX_FIELDNAME = "brdr_" BASE_FORMULA_FIELD_NAME = PREFIX_FIELDNAME + "base_formula" -NEW_FORMULA_FIELD_NAME = PREFIX_FIELDNAME + "new_formula" +FORMULA_FIELD_NAME = PREFIX_FIELDNAME + "formula" EVALUATION_FIELD_NAME = PREFIX_FIELDNAME + "evaluation" DIFF_PERCENTAGE_FIELD_NAME = PREFIX_FIELDNAME + "diff_percentage" DIFF_AREA_FIELD_NAME = PREFIX_FIELDNAME + "diff_area" diff --git a/brdr/utils.py b/brdr/utils.py index 04f3b33..6b59ba6 100644 --- a/brdr/utils.py +++ b/brdr/utils.py @@ -3,14 +3,8 @@ import numpy as np import requests -from geojson import Feature -from geojson import FeatureCollection -from geojson import dump -from shapely import GeometryCollection -from shapely import make_valid -from shapely import node -from shapely import polygonize -from shapely import unary_union +from geojson import Feature, FeatureCollection, dump +from shapely import GeometryCollection, make_valid, node, polygonize, unary_union from shapely.geometry import shape from shapely.geometry.base import BaseGeometry @@ -34,7 +28,17 @@ def get_series_geojson_dict( geom_attributes=True, ): """ - Convert a series of process results to a GeoJSON feature collections. + Convert a series of process results to a GeoJSON feature collection. + + Args: + series_dict (dict): Dictionary containing process results. + crs (str): Coordinate reference system. + id_field (str): Field name for the ID. + series_prop_dict (dict, optional): Dictionary containing series properties. + geom_attributes (bool, optional): Whether to include geometry attributes. + + Returns: + dict: Dictionary of GeoJSON feature collections. """ features_list_dict = {} @@ -50,7 +54,7 @@ def get_series_geojson_dict( properties[REMARK_FIELD_NAME] = process_result["remark"] for results_type, geom in process_result.items(): - if not isinstance(geom,BaseGeometry): + if not isinstance(geom, BaseGeometry): continue if results_type not in features_list_dict: features_list_dict[results_type] = [] @@ -75,13 +79,12 @@ def _feature_from_geom( Args: geom (BaseGeometry): The geometry to convert. - properties (dict): The properties to include in the feature. - geom_attributes (bool): Whether to include geometry attributes (default True). + properties (dict, optional): The properties to include in the feature. + geom_attributes (bool, optional): Whether to include geometry attributes. Returns: Feature: The GeoJSON feature. """ - properties = dict(properties or {}) if geom_attributes: area = geom.area @@ -94,8 +97,17 @@ def _feature_from_geom( def geojson_from_dict(dictionary, crs, id_field, prop_dict=None, geom_attributes=True): """ - get a geojson (featurecollection) from a dictionary of ids(keys) and geometries - (values) + Get a GeoJSON (FeatureCollection) from a dictionary of IDs (keys) and geometries (values). + + Args: + dictionary (dict): Dictionary of geometries. + crs (str): Coordinate reference system. + id_field (str): Field name for the ID. + prop_dict (dict, optional): Dictionary of properties. + geom_attributes (bool, optional): Whether to include geometry attributes. + + Returns: + FeatureCollection: The GeoJSON FeatureCollection. """ features = [] for key, geom in dictionary.items(): @@ -108,6 +120,13 @@ def geojson_from_dict(dictionary, crs, id_field, prop_dict=None, geom_attributes def write_geojson(path_to_file, geojson): + """ + Write a GeoJSON object to a file. + + Args: + path_to_file (str): Path to the output file. + geojson (FeatureCollection): The GeoJSON object to write. + """ parent = os.path.dirname(path_to_file) os.makedirs(parent, exist_ok=True) with open(path_to_file, "w") as f: @@ -116,32 +135,15 @@ def write_geojson(path_to_file, geojson): def multipolygons_to_singles(dict_geoms): """ - Converts a dictionary of shapely-geometries to a dictionary containing only single - polygons. - - This function iterates through a dictionary where values are Shapely-geometries - and performs the following: - - * **Polygons:** Preserves the key and geometry from the original dictionary. - * **MultiPolygons with one polygon:** Preserves the key and extracts the single - polygon. - * **MultiPolygons with multiple polygons:** - * Creates new keys for each polygon by appending a suffix (_index) to the - original key. - * Assigns the individual polygons from the MultiPolygon to the newly created - keys. + Convert a dictionary of Shapely geometries to a dictionary containing only single polygons. Args: - dict_geoms (dict): A dictionary where keys are identifiers and values are - GeoJSON geometries. + dict_geoms (dict): Dictionary of geometries. Returns: - dict: A new dictionary containing only single polygons (as Polygon geometries). - Keys are created based on the logic described above. - - Notes: - * Geometries that are not Polygons or MultiPolygons are excluded with a warning - message printed. + tuple: A tuple containing: + - dict: Dictionary of single polygons. + - dict: Dictionary mapping new keys to original keys. """ resulting_dict_geoms = {} dict_multi_as_single = {} @@ -162,37 +164,18 @@ def multipolygons_to_singles(dict_geoms): i = i + 1 else: logging.debug("geom excluded: " + str(geom) + " for key: " + str(key)) - return resulting_dict_geoms,dict_multi_as_single + return resulting_dict_geoms, dict_multi_as_single def polygonize_reference_data(dict_ref): """ - Creates a new dictionary with non-overlapping polygons based on a reference data - dictionary. - - This function is designed to handle situations where the original reference data - dictionary might contain: - - * Overlapping polygons: It creates new, non-overlapping polygons by combining all - reference borders. - * Multiple overlapping references: This function is useful when combining references - like parcels and buildings that might overlap. - - **Important:** The original reference IDs are lost in the process of creating new - non-overlapping polygons. New unique keys are assigned instead. + Create a new dictionary with non-overlapping polygons based on a reference data dictionary. Args: - dict_ref (dict): A dictionary where keys are identifiers and values are Shapely - geometries (assumed to be Polygons or MultiPolygons). + dict_ref (dict): Dictionary of reference geometries. Returns: - dict: A new dictionary containing non-overlapping polygons derived from the - original reference data. - Keys are unique strings (reference IDs are lost). - - Notes: - * Geometries that are not Polygons or MultiPolygons are excluded with a warning - message printed. + dict: Dictionary of non-overlapping polygons. """ arr_ref = [] for key in dict_ref: @@ -213,27 +196,16 @@ def polygonize_reference_data(dict_ref): def get_breakpoints_zerostreak(x, y): """ - Determine the extremes and zero_streaks of a graph based on the derivative, and - return: - * the breakpoints: extremes (breakpoints) of graph where 'change' occurs - * the zero_streaks: ranges where the derivative is zero, ranges of relevant_distance - where 'no-change' occurs + Determine the extremes and zero_streaks of a graph based on the derivative. - Parameters: - x (numpy.ndarray): The x values of the graph. - derivative (numpy.ndarray): The y values of the graph. + Args: + x (numpy.ndarray): The x values of the graph. + y (numpy.ndarray): The y values of the graph. Returns: - extremes: A list of tuples for breakpoints: - * relevant distance where extreme occurs - * extreme value - * minimum or maximum - zero_streaks: A list of tuples for zero_streaks: - * relevant distance where zero_streak starts - * relevant distance where zero_streak ends - * center of start- and end- zero_streak - * counter of #relevant_distances where zero-streak holds on - * extreme value for zero_streak + tuple: A tuple containing: + - list: List of breakpoints (extremes). + - list: List of zero_streaks. """ derivative = _numerical_derivative(x, y) # plt.plot(x, y, label="y") @@ -322,53 +294,15 @@ def diffs_from_dict_series( diff_metric: DiffMetric = DiffMetric.CHANGES_AREA, ): """ - Calculates a dictionary containing difference metrics for thematic elements based on - a distance series. - - This function analyzes the changes in thematic elements (represented by - thematic_ids in `dict_thematic`) across different distances provided in the - `dict_series`. It calculates a difference metric for each thematic element at - each distance and returns a dictionary summarizing these differences. + Calculates a dictionary containing difference metrics for thematic elements based on a distance series. - Args: - dict_series (dict): A dictionary where thematic_ids are distances and - values are tuples of two dictionaries. - - The first dictionary in the tuple represents thematic - element areas for a specific distance. - - The second dictionary represents the difference in - areas from the original thematic data for a specific - distance. - dict_thematic (dict): A dictionary where thematic_ids are thematic element - identifiers and values are GeoJSON geometry objects representing the - original thematic data. - diff_metric (DiffMetric): The metric used to determine the difference between - the thematic and reference data. + Parameters: + dict_series (dict): A dictionary where keys are thematic IDs and values are dictionaries mapping relative distances to ProcessResult objects. + dict_thematic (dict): A dictionary where keys are thematic IDs and values are BaseGeometry objects representing the original geometries. + diff_metric (DiffMetric, optional): The metric to use for calculating differences. Default is DiffMetric.CHANGES_AREA. Returns: - dict: A dictionary containing difference metrics for each thematic element - (`key`) across different distances. - The structure is as follows: - { - 'thematic_key1': { - distance1: difference_metric1, - distance2: difference_metric2, - ... - }, - 'thematic_key2': { - distance1: difference_metric1, - distance2: difference_metric2, - ... - }, - ... - } - - - `difference_metric`: This value depends on the chosen calculation for - thematic element change. The docstring provides examples like area - difference, percentage change, and absolute difference. - - Raises: - KeyError: If a thematic element key is missing from the results in - `dict_series`. + dict: A dictionary where keys are thematic IDs and values are dictionaries mapping relative distances to calculated difference metrics. """ diffs = {} # all the relevant distances used to calculate the series @@ -451,6 +385,18 @@ def get_collection(ref_url, limit): def geojson_to_dicts(collection, id_property): + """ + Converts a GeoJSON collection into dictionaries of geometries and properties. + + Parameters: + collection (dict): The GeoJSON collection to convert. + id_property (str): The property name to use as the key for the dictionaries. + + Returns: + tuple: Two dictionaries: + - data_dict (dict): A dictionary where keys are the id_property values and values are the geometries. + - data_dict_properties (dict): A dictionary where keys are the id_property values and values are the properties. + """ data_dict = {} data_dict_properties = {} if collection is None or "features" not in collection: @@ -466,6 +412,19 @@ def geojson_to_dicts(collection, id_property): def get_collection_by_partition( url, geometry, partition=1000, limit=DOWNLOAD_LIMIT, crs=DEFAULT_CRS ): + """ + Retrieves a collection of geographic data by partitioning the input geometry. + + Parameters: + url (str): The base URL for the data source. + geometry (object): The geometric area to partition and retrieve data for. If None, retrieves data for the entire area. + partition (int, optional): The number of partitions to divide the geometry into. Default is 1000. If less than 1, no partitioning is done. + limit (int, optional): The maximum number of items to retrieve. Default is DOWNLOAD_LIMIT. + crs (str, optional): The coordinate reference system to use. Default is DEFAULT_CRS. + + Returns: + dict: A collection of geographic data, potentially partitioned by the input geometry. + """ collection = {} if geometry is None: collection = get_collection( @@ -489,6 +448,17 @@ def get_collection_by_partition( def _add_bbox_to_url(url, crs=DEFAULT_CRS, bbox=None): + """ + Adds a bounding box (bbox) parameter to the URL for geographic data requests. + + Parameters: + url (str): The base URL for the data source. + crs (str, optional): The coordinate reference system to use. Default is DEFAULT_CRS. + bbox (str, optional): The bounding box coordinates to add to the URL. If None, no bbox is added. + + Returns: + str: The updated URL with the bbox parameter included, if provided. + """ # Load the Base reference data if bbox is not None: url = url + "&bbox-crs=" + crs + "&bbox=" + bbox @@ -505,7 +475,7 @@ def merge_process_results( Args: result_dict (dict): A dictionary where keys are theme IDs and values are process results - Returns: dict: A new dictionary with merged geometries and remarks (processresults, where keys are global + Returns: dict: A new dictionary with merged geometries and remarks (processresults), where keys are global theme IDs and values are merged geometries and remarks. """ diff --git a/examples/example_parcel_change_detector.py b/examples/example_parcel_change_detector.py index 33d3a48..6b2a7fc 100644 --- a/examples/example_parcel_change_detector.py +++ b/examples/example_parcel_change_detector.py @@ -2,7 +2,7 @@ from datetime import datetime from brdr.aligner import Aligner -from brdr.constants import EVALUATION_FIELD_NAME, RELEVANT_DISTANCE_FIELD_NAME, NEW_FORMULA_FIELD_NAME +from brdr.constants import EVALUATION_FIELD_NAME, RELEVANT_DISTANCE_FIELD_NAME, FORMULA_FIELD_NAME from brdr.grb import GRBFiscalParcelLoader from brdr.grb import update_to_actual_grb from brdr.oe import OnroerendErfgoedLoader @@ -83,7 +83,7 @@ fcs = update_to_actual_grb( featurecollection_base_result, base_aligner.name_thematic_id, - base_formula_field=NEW_FORMULA_FIELD_NAME, + base_formula_field=FORMULA_FIELD_NAME, max_distance_for_actualisation=max_distance_for_actualisation, ) diff --git a/examples/example_update_to_actual_grb.py b/examples/example_update_to_actual_grb.py index 832ec5b..cd79b85 100644 --- a/examples/example_update_to_actual_grb.py +++ b/examples/example_update_to_actual_grb.py @@ -1,5 +1,5 @@ from brdr.aligner import Aligner -from brdr.constants import EVALUATION_FIELD_NAME, NEW_FORMULA_FIELD_NAME +from brdr.constants import EVALUATION_FIELD_NAME, FORMULA_FIELD_NAME from brdr.grb import GRBFiscalParcelLoader from brdr.grb import update_to_actual_grb from brdr.loader import GeoJsonLoader @@ -66,7 +66,7 @@ featurecollection = update_to_actual_grb( featurecollection_base_result, base_aligner.name_thematic_id, - base_formula_field=NEW_FORMULA_FIELD_NAME, + base_formula_field=FORMULA_FIELD_NAME, ) # Print results for feature in featurecollection["result"]["features"]: diff --git a/tests/test_aligner.py b/tests/test_aligner.py index 48bf207..f8d5aa2 100644 --- a/tests/test_aligner.py +++ b/tests/test_aligner.py @@ -10,7 +10,7 @@ from shapely.predicates import equals from brdr.aligner import Aligner -from brdr.constants import NEW_FORMULA_FIELD_NAME +from brdr.constants import FORMULA_FIELD_NAME from brdr.enums import GRBType, AlignerResultType from brdr.enums import OpenbaarDomeinStrategy from brdr.geometry_utils import _grid_bounds @@ -379,7 +379,7 @@ def test_evaluate(self): "result" ] thematic_dict_formula[key] = { - NEW_FORMULA_FIELD_NAME: base_aligner.get_brdr_formula( + FORMULA_FIELD_NAME: base_aligner.get_brdr_formula( thematic_dict_result[key] ) } From 8c3fcf118f6b9b223faae968d65c6b272ba2adb6 Mon Sep 17 00:00:00 2001 From: dieuska Date: Thu, 3 Oct 2024 10:29:40 +0200 Subject: [PATCH 21/26] default OD strategy adapted and added docs --- brdr/aligner.py | 150 ++++++++++++++++++++++---------- brdr/grb.py | 10 ++- brdr/utils.py | 40 +++++---- examples/example_evaluate.py | 2 +- examples/example_evaluate_ao.py | 2 +- tests/test_aligner.py | 2 +- 6 files changed, 141 insertions(+), 65 deletions(-) diff --git a/brdr/aligner.py b/brdr/aligner.py index 13816fb..a8b417a 100644 --- a/brdr/aligner.py +++ b/brdr/aligner.py @@ -77,7 +77,7 @@ def __init__( relevant_distance=1, relevant_distances=np.arange(0, 200, 10, dtype=int) / 100, threshold_overlap_percentage=50, - od_strategy=OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE, + od_strategy=OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE, crs=DEFAULT_CRS, multi_as_single_modus=True, threshold_exclusion_area=0, @@ -97,22 +97,31 @@ def __init__( feedback in QGIS. Defaults to None. relevant_distance (int, optional): The relevant distance (in meters) for processing. Defaults to 1. + relevant_distances ([],optional): Relevant distances (in meters) for + processing od_strategy (int, optional): The strategy to determine how to handle information outside the reference polygons (Openbaar Domein) - (default 1: SNAP_SINGLE_SIDE) + (default: SNAP_FULL_AREA_ALL_SIDE) threshold_overlap_percentage (int, optional): Threshold (%) to determine from which overlapping-percentage a reference-polygon has to be included when there aren't relevant intersections or relevant differences (default 50%). When setting this parameter to '-1' the original border for will be returned for cases where nor relevant intersections and relevant differences are found - od_strategy (int, optional): Determines how the algorithm deals with parts - of the geometry that are not on the - reference (default 1: SNAP_SINGLE_SIDE) crs (str, optional): Coordinate Reference System (CRS) of the data. (default EPSG:31370) + multi_as_single_modus (boolean, optional): Modus to handle multipolygons (Default=True): + True: input-multipolygons will be split-up into single polygons and handled by the algorithm. After executing the algorithm, the results are merged together. + False: Multipolygons are directly processed by the algorithm + threshold_exclusion_percentage (int, optional): Percentage for excluding candidate reference-polygons when overlap(%) is smaller than the threshold(Default=0) + threshold_exclusion_area (int, optional):Area in m² for excluding candidate reference-polygons when overlap(m²) is smaller than the threshold (Default=0) + buffer_multiplication_factor (float, optional): Multiplication factor, used to buffer the thematic objects searching for reference borders (buffer= buffer_multiplication_factor*relevant_distance)(Default=1.01) + threshold_circle_ratio (float, optional): Threshold-value to exclude circles getting processed (perfect circle = 1) based on POLSPY-POPPER algorithm(Default=0.98) + correction_distance (float, optional): Distance used in a pos_neg_buffer to remove slivers (technical correction) (Default= 0.01 = 1cm ) + mitre_limit (int, optional):buffer-parameter - The mitre ratio is the ratio of the distance from the corner to the end of the mitred offset corner. + When two line segments meet at a sharp angle, a miter join will extend far beyond the original geometry. (and in the extreme case will be infinitely far.) To prevent unreasonable geometry, the mitre limit allows controlling the maximum length of the join corner. + Corners with a ratio which exceed the limit will be beveled(Default=10) area_limit (int, optional): Maximum area for processing. (default 100000) - max_workers (int, optional): Amount of workers that is used in ThreadPoolExecutor when processing objects for multiple relevant distances. (default None) - + max_workers (int, optional): Amount of workers that is used in ThreadPoolExecutor (for parallel execution) when processing objects for multiple relevant distances. (default None). If set to -1, no parallel exececution is used. """ self.logger = Logger(feedback) @@ -121,10 +130,10 @@ def __init__( self.relevant_distances = relevant_distances self.od_strategy = od_strategy self.threshold_overlap_percentage = threshold_overlap_percentage - # Area in m² for excluding candidate reference when overlap(m²) is smaller than the + # Area in m² for excluding candidate reference-polygons when overlap(m²) is smaller than the # threshold self.threshold_exclusion_area = threshold_exclusion_area - # Percentage for excluding candidate reference when overlap(%) is smaller than the + # Percentage for excluding candidate reference-polygons when overlap(%) is smaller than the # threshold self.threshold_exclusion_percentage = threshold_exclusion_percentage self.area_limit = area_limit @@ -200,11 +209,21 @@ def __init__( ########################################### def load_thematic_data(self, loader: Loader): + """ + Loads the thematic features into the aligner + :param loader: + :return: + """ self.dict_thematic, self.dict_thematic_properties, self.dict_thematic_source = ( loader.load_data() ) def load_reference_data(self, loader: Loader): + """ + Loads the reference features into the aligner, and prepares the reference-data for processing + :param loader: + :return: + """ ( self.dict_reference, self.dict_reference_properties, @@ -219,7 +238,7 @@ def process_geometry( self, input_geometry: BaseGeometry, relevant_distance: float = 1, - od_strategy=OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE, + od_strategy=OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE, threshold_overlap_percentage=50, ) -> ProcessResult: """ @@ -227,10 +246,16 @@ def process_geometry( Args: input_geometry (BaseGeometry): The input geometric object. - relevant_distance - od_strategy - threshold_overlap_percentage (int): The buffer distance (positive - or negative). + relevant_distance: The relevant distance (in meters) for processing + od_strategy (int, optional): The strategy to determine how to handle + information outside the reference polygons (Openbaar Domein) + (default: SNAP_FULL_AREA_ALL_SIDE) + threshold_overlap_percentage (int, optional): Threshold (%) to determine + from which overlapping-percentage a reference-polygon has to be included + when there aren't relevant intersections or relevant differences + (default 50%). + When setting this parameter to '-1' the original border for will be returned for cases where nor relevant intersections and relevant differences are found + Returns: ProcessResult : A dict containing the resulting geometries: @@ -243,9 +268,7 @@ def process_geometry( geometry * relevant_intersection (BaseGeometry): The relevant_intersection * relevant_difference (BaseGeometry): The relevant_difference - Notes: - - - Example: + * remark (str): remarks collected when processing the geoemetry """ if self.area_limit and input_geometry.area > self.area_limit: message = "The input geometry is too large to process." @@ -329,7 +352,7 @@ def process( dict_thematic=None, relevant_distances: Iterable[float] = None, relevant_distance=1, - od_strategy=OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE, + od_strategy=OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE, threshold_overlap_percentage=50, ) -> dict[any, dict[float, ProcessResult]]: """ @@ -339,10 +362,15 @@ def process( Args: relevant_distances (Iterable[float]): A series of relevant distances (in meters) to process - od_strategy (int, optional): The strategy for overlap detection. - Defaults to 1. - threshold_overlap_percentage (float, optional): The threshold percentage for - considering full overlap. Defaults to 50. + od_strategy (int, optional): The strategy to determine how to handle + information outside the reference polygons (Openbaar Domein) + (default: SNAP_FULL_AREA_ALL_SIDE) + threshold_overlap_percentage (int, optional): Threshold (%) to determine + from which overlapping-percentage a reference-polygon has to be included + when there aren't relevant intersections or relevant differences + (default 50%). + When setting this parameter to '-1' the original border for will be returned for cases where nor relevant intersections and relevant differences are found + Returns: dict: A dictionary, for every thematic ID a dictionary with the results for all distances @@ -459,7 +487,7 @@ def predictor( self, dict_thematic=None, relevant_distances=np.arange(0, 300, 10, dtype=int) / 100, - od_strategy=OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE, + od_strategy=OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE, threshold_overlap_percentage=50, ): """ @@ -501,11 +529,24 @@ def predictor( element based on the element key (using `filter_resulting_series_by_key`). Args: + + relevant_distances (np.ndarray, optional): A series of relevant distances + (in meters) to process. : A NumPy array of distances to + be analyzed. + od_strategy (int, optional): The strategy to determine how to handle + information outside the reference polygons (Openbaar Domein) + (default: SNAP_FULL_AREA_ALL_SIDE) + threshold_overlap_percentage (int, optional): Threshold (%) to determine + from which overlapping-percentage a reference-polygon has to be included + when there aren't relevant intersections or relevant differences + (default 50%). + When setting this parameter to '-1' the original border for will be returned for cases where nor relevant intersections and relevant differences are found + relevant_distances (np.ndarray, optional): A NumPy array of distances to be analyzed. Defaults to np.arange(0.1, 5.05, 0.1). od_strategy (OpenbaarDomeinStrategy, optional): A strategy for handling open data in the processing (implementation specific). Defaults to - OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE. + OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE. threshold_overlap_percentage (int, optional): A percentage threshold for considering full overlap in the processing (implementation specific). Defaults to 50. @@ -521,10 +562,8 @@ def predictor( - Values: dicts containing results (likely specific to your implementation) from the distance series for the corresponding distance. - - Logs: - - Debug logs the thematic element key being processed. """ + if dict_thematic is None: dict_thematic = self.dict_thematic dict_predictions = defaultdict(dict) @@ -584,12 +623,12 @@ def predictor( diffs_dict, ) - def compare( + def evaluate( self, ids_to_compare=None, ): """ - Compares and evaluate input-geometries (with formula).Attributes are added to evaluate and decide if new + Compares and evaluate input-geometries (with formula). Attributes are added to evaluate and decide if new proposals can be used affected: list with all IDs to evaluate. all other IDs will be unchanged. If None (default), all self.dict_thematic will be evaluated. """ @@ -649,13 +688,21 @@ def get_brdr_formula(self, geometry: BaseGeometry, with_geom=False): Returns: dict: A dictionary containing formula-related data: - - 'full': True if the intersection is the same as the reference - geometry, else False. - - 'area': Area of the intersection or reference geometry. - - 'percentage': Percentage of intersection area relative to the - reference geometry. - - 'geometry': GeoJSON representation of the intersection (if - with_geom is True). + - "alignment_date": datetime.now().strftime(DATE_FORMAT), + - "brdr_version": str(__version__), + - "reference_source": self.dict_reference_source, + - "full": True if the geometry exists out of all full reference-polygons, else False. + - "area": Area of the geometry. + - "reference_features": { + array of all the reference features the geometry is composed of: + - 'full': True if the intersection is the same as the reference + geometry, else False. + - 'area': Area of the intersection or reference geometry. + - 'percentage': Percentage of intersection area relative to the + reference geometry. + - 'geometry': GeoJSON representation of the intersection (if + with_geom is True).}, + - "reference_od": Discription of the OD-part of the geometry (= not covered by reference-features), """ dict_formula = { "alignment_date": datetime.now().strftime(DATE_FORMAT), @@ -756,9 +803,11 @@ def get_results_as_geojson( attributes=False, ): """ - get a geojson of a dictionary containing the resulting geometries for all - 'serial' relevant distances. If no dict_series is given, the dict_result returned. - Optional: The descriptive formula is added as an attribute to the result""" + get a geojson of a dictionary containing the resulting geometries for all + 'serial' relevant distances. The resulttype can be chosen. + formula (boolean, Optional): The descriptive formula is added as an attribute to the result + attributes (boolean, Optional): The original attributes/properties are added to the result + """ prop_dictionary = None if resulttype == AlignerResultType.PROCESSRESULTS: dict_series = self.dict_processresults @@ -801,7 +850,7 @@ def get_results_as_geojson( def get_input_as_geojson(self, inputtype=AlignerInputType.REFERENCE): """ - get a geojson of the reference polygons + get a geojson of the input polygons (thematic or reference-polygons) """ if inputtype == AlignerInputType.THEMATIC: @@ -829,7 +878,7 @@ def save_results( self, path, resulttype=AlignerResultType.PROCESSRESULTS, formula=True ): """ - Exports analysis results as GeoJSON files. + Exports analysis results (as geojson) to path. This function exports 6 GeoJSON files containing the analysis results to the specified `path`. @@ -864,6 +913,10 @@ def save_results( ) def get_thematic_union(self): + """ + returns a unary_unioned geometry from all the thematic geometries + :return: + """ if self.thematic_union is None: self.thematic_union = safe_unary_union(list(self.dict_thematic.values())) return self.thematic_union @@ -871,9 +924,7 @@ def get_thematic_union(self): def _prepare_reference_data(self): """ Prepares reference data for spatial queries and analysis. - It performs the following tasks: - 1. **Optimizes spatial queries:** - Creates a Spatial Relationship Tree (STRtree) using `STRtree` for efficient spatial queries against the reference data in @@ -902,6 +953,12 @@ def _prepare_reference_data(self): def _calculate_intersection_between_geometry_and_od( self, geometry, relevant_distance ): + """ + Calculates the intersecting parts between a thematic geometry and the openbaardomein( domain, not coverd by reference-polygons) + :param geometry: + :param relevant_distance: + :return: + """ # Calculate the intersection between thematic and Openbaar Domein buffer_distance = relevant_distance / 2 relevant_intersection_array = [] @@ -1099,6 +1156,10 @@ def _od_snap_all_side(self, geometry, relevant_distance): return geom_thematic_od, relevant_difference_array, relevant_intersection_array def _get_reference_union(self): + """ + returns a unary_unioned geometry from all the referene geometries + :return: + """ if self.reference_union is None: self.reference_union = safe_unary_union(list(self.dict_reference.values())) return self.reference_union @@ -1128,6 +1189,7 @@ def _postprocess_preresult( output geometry * result_diff_min (BaseGeometry): The resulting negative difference output geometry + * remark (str): Remark when processing the geometry """ # Process array remark = "" @@ -1295,8 +1357,6 @@ def _evaluate(self, id_theme, geom_predicted): function that evaluates a predicted geometry and returns a properties-dictionary """ threshold_od_percentage = 1 - # threshold_area = 5 - # threshold_percentage = 1 properties = { FORMULA_FIELD_NAME: "", EVALUATION_FIELD_NAME: Evaluation.TO_CHECK_4, diff --git a/brdr/grb.py b/brdr/grb.py index 1a1f784..f8a0cf1 100644 --- a/brdr/grb.py +++ b/brdr/grb.py @@ -352,6 +352,14 @@ def update_to_actual_grb( """ Function to update a thematic featurecollection to the most actual version of GRB. Important to notice that the featurecollection needs a 'formula' for the base-alignment. + + :param featurecollection: Thematic featurecollection + :param id_theme_fieldname: property-fieldname that states which property has to be used as unique ID + :param base_formula_field: Name of the property-field that holds the original/base formula of the geometry, that has to be compared with the actual formula. + :param max_distance_for_actualisation: Maximum relevant distance that is used to search and evaluate resulting geometries. All relevant distance between 0 and this max_distance are used to search, with a interval of 0.1m. + :param feedback: (default None): a QGIS feedback can be added to push all the logging to QGIS + :param attributes: (boolean, default=True): States of all original attributes has to be added to the result + :return: featurecollection """ logger = Logger(feedback) # Load featurecollection into a shapely_dict: @@ -428,7 +436,7 @@ def update_to_actual_grb( actual_aligner.relevant_distances = ( np.arange(0, max_distance_for_actualisation * 100, 10, dtype=int) / 100 ) - dict_evaluated, prop_dictionary = actual_aligner.compare(ids_to_compare=affected) + dict_evaluated, prop_dictionary = actual_aligner.evaluate(ids_to_compare=affected) return actual_aligner.get_results_as_geojson( resulttype=AlignerResultType.EVALUATED_PREDICTIONS, diff --git a/brdr/utils.py b/brdr/utils.py index 6b59ba6..3c3aa37 100644 --- a/brdr/utils.py +++ b/brdr/utils.py @@ -13,7 +13,8 @@ DEFAULT_CRS, DOWNLOAD_LIMIT, RELEVANT_DISTANCE_FIELD_NAME, - NR_CALCULATION_FIELD_NAME, REMARK_FIELD_NAME, + NR_CALCULATION_FIELD_NAME, + REMARK_FIELD_NAME, ) from brdr.enums import DiffMetric from brdr.geometry_utils import get_partitions, get_bbox @@ -449,15 +450,15 @@ def get_collection_by_partition( def _add_bbox_to_url(url, crs=DEFAULT_CRS, bbox=None): """ - Adds a bounding box (bbox) parameter to the URL for geographic data requests. + Adds a bounding box (bbox) parameter to the URL for geographic data requests. - Parameters: - url (str): The base URL for the data source. - crs (str, optional): The coordinate reference system to use. Default is DEFAULT_CRS. - bbox (str, optional): The bounding box coordinates to add to the URL. If None, no bbox is added. + Parameters: + url (str): The base URL for the data source. + crs (str, optional): The coordinate reference system to use. Default is DEFAULT_CRS. + bbox (str, optional): The bounding box coordinates to add to the URL. If None, no bbox is added. - Returns: - str: The updated URL with the bbox parameter included, if provided. + Returns: + str: The updated URL with the bbox parameter included, if provided. """ # Load the Base reference data if bbox is not None: @@ -466,8 +467,7 @@ def _add_bbox_to_url(url, crs=DEFAULT_CRS, bbox=None): def merge_process_results( - result_dict: dict[any, dict[float, ProcessResult]], - dict_multi_as_single:dict + result_dict: dict[any, dict[float, ProcessResult]], dict_multi_as_single: dict ) -> dict[any, dict[float, ProcessResult]]: """ Merges processresults in a dictionary from multiple themeIDs into a single themeID. @@ -485,22 +485,30 @@ def merge_process_results( if id_theme in dict_multi_as_single.keys(): id_theme_global = dict_multi_as_single[id_theme] else: - id_theme_global=id_theme + id_theme_global = id_theme if id_theme_global not in grouped_results: grouped_results[id_theme_global] = dict_results else: for rel_dist, process_result in dict_results.items(): for key in process_result: value = process_result[key] # noqa - if isinstance(value,str) and value !="": - existing_remark: str = grouped_results[id_theme_global][rel_dist][ key] # noqa - grouped_results[id_theme_global][rel_dist][key] = existing_remark + " | " + str(value) + if isinstance(value, str) and value != "": + existing_remark: str = grouped_results[id_theme_global][ + rel_dist + ][ + key + ] # noqa + grouped_results[id_theme_global][rel_dist][key] = ( + existing_remark + " | " + str(value) + ) continue - elif isinstance(value,BaseGeometry): + elif isinstance(value, BaseGeometry): geom = value if geom.is_empty or geom is None: continue - existing: BaseGeometry = grouped_results[id_theme_global][rel_dist][ + existing: BaseGeometry = grouped_results[id_theme_global][ + rel_dist + ][ key ] # noqa grouped_results[id_theme_global][rel_dist][key] = unary_union( diff --git a/examples/example_evaluate.py b/examples/example_evaluate.py index be929b9..125912f 100644 --- a/examples/example_evaluate.py +++ b/examples/example_evaluate.py @@ -56,7 +56,7 @@ GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner) ) actual_aligner.relevant_distances = np.arange(0, 200, 10, dtype=int) / 100 -dict_evaluated, prop_dictionary = actual_aligner.compare(ids_to_compare=affected) +dict_evaluated, prop_dictionary = actual_aligner.evaluate(ids_to_compare=affected) fc = actual_aligner.get_results_as_geojson(resulttype=AlignerResultType.EVALUATED_PREDICTIONS,formula=True, attributes=True) print(fc["result"]) diff --git a/examples/example_evaluate_ao.py b/examples/example_evaluate_ao.py index a0b5e00..8f10775 100644 --- a/examples/example_evaluate_ao.py +++ b/examples/example_evaluate_ao.py @@ -53,7 +53,7 @@ GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner) ) actual_aligner.relevant_distances = np.arange(0, 200, 10, dtype=int) / 100 -dict_evaluated, prop_dictionary = actual_aligner.compare(ids_to_compare=affected) +dict_evaluated, prop_dictionary = actual_aligner.evaluate(ids_to_compare=affected) fc = actual_aligner.get_results_as_geojson(resulttype=AlignerResultType.EVALUATED_PREDICTIONS,formula=True, attributes=True) print(fc["result"]) diff --git a/tests/test_aligner.py b/tests/test_aligner.py index f8d5aa2..1b36a30 100644 --- a/tests/test_aligner.py +++ b/tests/test_aligner.py @@ -411,7 +411,7 @@ def test_evaluate(self): ) ) actual_aligner.relevant_distances = np.arange(0, 200, 10, dtype=int) / 100 - dict_evaluated, prop_dictionary = actual_aligner.compare( + dict_evaluated, prop_dictionary = actual_aligner.evaluate( ids_to_compare=affected ) From ed3771e0ace47b71de186a64b794ef57ed2b1c19 Mon Sep 17 00:00:00 2001 From: dieuska Date: Thu, 3 Oct 2024 12:19:08 +0200 Subject: [PATCH 22/26] #100 cleanedup examples --- brdr/utils.py | 3 +- examples/example_131635.py | 30 ---- examples/example_aligners.py | 60 ------- examples/example_ao.py | 35 ----- examples/example_combined_borders_adp_gbg.py | 19 +-- examples/example_dictloader_properties.py | 45 ++++++ examples/example_eo.py | 50 ------ examples/example_evaluate.py | 110 +++++++------ examples/example_evaluate_ao.py | 64 -------- ...le_aligner.py => example_geojsonloader.py} | 20 ++- examples/example_grbspecificloader.py | 41 ++--- examples/example_integer_id.py | 39 ----- examples/example_local_data.py | 23 --- examples/example_multi_to_single.py | 46 ------ examples/example_multi_to_single_modus.py | 53 +++++++ examples/example_multipolygon.py | 38 ----- examples/example_onroerenderfgoed.py | 26 +++ examples/example_parcel_change_detector.py | 22 ++- examples/example_parcel_vs_building.py | 6 +- examples/example_predictor.py | 24 +-- .../example_process_relevant_distances.py | 64 ++++++++ examples/example_speedtest.py | 4 + examples/example_update_to_actual_grb.py | 148 +++++++++--------- 23 files changed, 402 insertions(+), 568 deletions(-) delete mode 100644 examples/example_131635.py delete mode 100644 examples/example_aligners.py delete mode 100644 examples/example_ao.py create mode 100644 examples/example_dictloader_properties.py delete mode 100644 examples/example_eo.py delete mode 100644 examples/example_evaluate_ao.py rename examples/{example_aligner.py => example_geojsonloader.py} (87%) delete mode 100644 examples/example_integer_id.py delete mode 100644 examples/example_local_data.py delete mode 100644 examples/example_multi_to_single.py create mode 100644 examples/example_multi_to_single_modus.py delete mode 100644 examples/example_multipolygon.py create mode 100644 examples/example_onroerenderfgoed.py create mode 100644 examples/example_process_relevant_distances.py diff --git a/brdr/utils.py b/brdr/utils.py index 3c3aa37..1406e5c 100644 --- a/brdr/utils.py +++ b/brdr/utils.py @@ -148,8 +148,7 @@ def multipolygons_to_singles(dict_geoms): """ resulting_dict_geoms = {} dict_multi_as_single = {} - for key in dict_geoms: - geom = dict_geoms[key] + for key, geom in dict_geoms.items(): if str(geom.geom_type) == "Polygon": resulting_dict_geoms[key] = geom elif str(geom.geom_type) == "MultiPolygon": diff --git a/examples/example_131635.py b/examples/example_131635.py deleted file mode 100644 index 4afe314..0000000 --- a/examples/example_131635.py +++ /dev/null @@ -1,30 +0,0 @@ -from brdr.aligner import Aligner -from brdr.enums import GRBType, AlignerInputType -from brdr.grb import GRBActualLoader -from brdr.oe import OnroerendErfgoedLoader -from examples import print_brdr_formula -from examples import show_map - -if __name__ == "__main__": - # TODO - # EXAMPLE for a thematic Polygon (aanduid_id 131635) - - # Initiate brdr - aligner = Aligner() - # Load thematic data & reference data - loader = OnroerendErfgoedLoader([131635]) - aligner.load_thematic_data(loader) - loader = GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=aligner) - aligner.load_reference_data(loader) - ref_geojson = aligner.get_input_as_geojson(inputtype=AlignerInputType.REFERENCE) - - # RESULTS - rel_dist = 2 - dict_results = aligner.process(relevant_distance=rel_dist, od_strategy=4) - fcs = aligner.get_results_as_geojson() - print(fcs["result"]) - # put resulting tuple in a dictionary - aligner.save_results("output/", formula=True) - - show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) - print_brdr_formula(dict_results, aligner) diff --git a/examples/example_aligners.py b/examples/example_aligners.py deleted file mode 100644 index e316008..0000000 --- a/examples/example_aligners.py +++ /dev/null @@ -1,60 +0,0 @@ -from brdr.aligner import Aligner -from brdr.enums import OpenbaarDomeinStrategy, GRBType -from brdr.grb import GRBActualLoader -from brdr.loader import GeoJsonFileLoader -from brdr.utils import diffs_from_dict_series -from examples import plot_series -from examples import show_map - -if __name__ == "__main__": - # Initiate brdr - aligner = Aligner() - # Load thematic data - aligner.load_thematic_data( - GeoJsonFileLoader("../tests/testdata/themelayer_referenced.geojson", "id_theme") - ) - - # Use GRB adp-parcels as reference polygons adp= administratieve percelen - aligner.load_reference_data( - GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=aligner) - ) - - # Example how to use the Aligner - rel_dist = 10 - dict_results = aligner.process( - relevant_distance=rel_dist, - od_strategy=OpenbaarDomeinStrategy.SNAP_FULL_AREA_SINGLE_SIDE, - ) - aligner.save_results("output/") - show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) - - rel_dist = 6 - dict_results = aligner.process( - relevant_distance=rel_dist, od_strategy=OpenbaarDomeinStrategy.SNAP_ALL_SIDE - ) - - aligner.save_results("output/") - show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) - # for key in r: - # x.get_formula(r[key]) - - # Example how to use a series (for histogram) - series = [0.1, 0.2, 0.3, 0.4, 0.5, 1, 2] - dict_series = aligner.process( - relevant_distances=series, od_strategy=2, threshold_overlap_percentage=50 - ) - resulting_areas = diffs_from_dict_series(dict_series, aligner.dict_thematic) - plot_series(series, resulting_areas) - - # Example how to use the Aligner with threshold_overlap_percentage=-1 (original - # border will be used for cases where relevant zones cannot be used for - # determination) - rel_dist = 6 - dict_results = aligner.process( - relevant_distance=rel_dist, - od_strategy=OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE, - threshold_overlap_percentage=-1, - ) - - aligner.save_results("output/") - show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) diff --git a/examples/example_ao.py b/examples/example_ao.py deleted file mode 100644 index 0f2366e..0000000 --- a/examples/example_ao.py +++ /dev/null @@ -1,35 +0,0 @@ -import numpy as np - -from brdr.aligner import Aligner -from brdr.enums import GRBType -from brdr.grb import GRBActualLoader -from brdr.oe import OnroerendErfgoedLoader -from examples import show_map, plot_series - -if __name__ == "__main__": - # EXAMPLE to test the algorithm for erfgoedobject with relevant distance 0.2m and - # od_strategy SNAP_ALL_SIDE - - # Initiate brdr - aligner = Aligner() - # Load thematic data & reference data - aanduidingsobjecten = [117798, 116800, 117881] - - loader = OnroerendErfgoedLoader(aanduidingsobjecten) - aligner.load_thematic_data(loader) - loader = GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=aligner) - aligner.load_reference_data(loader) - - series = np.arange(0, 500, 20, dtype=int) / 100 - # predict which relevant distances are interesting to propose as resulting geometry - dict_series, dict_predictions, diffs = aligner.predictor( - relevant_distances=series, od_strategy=2, threshold_overlap_percentage=50 - ) - for key in dict_predictions.keys(): - diff = {key: diffs[key]} - plot_series(series, diff) - show_map( - {key: dict_predictions[key]}, - {key: aligner.dict_thematic[key]}, - aligner.dict_reference, - ) diff --git a/examples/example_combined_borders_adp_gbg.py b/examples/example_combined_borders_adp_gbg.py index c6ee219..146f555 100644 --- a/examples/example_combined_borders_adp_gbg.py +++ b/examples/example_combined_borders_adp_gbg.py @@ -5,17 +5,18 @@ from brdr.utils import polygonize_reference_data, geojson_to_dicts from examples import show_map, print_brdr_formula -# example to test what happens if we combine borders -# (so thematic data can use both polygons) - -# If we just combine sets of polygons (fe parcels and buildings) these polygons will -# overlap, and gives some unwanted/unexpected results. The reference data can be -# preprocessed with 'shapely-polygonize', to create a new set of non-overlapping -# polygons. These non-overlapping polygons can be used as reference-data to align the -# theme-data. - if __name__ == "__main__": + """ + # example to test what happens if we combine borders + # (so thematic data can use both polygons) + + # If we just combine sets of polygons (fe parcels and buildings) these polygons will + # overlap, and gives some unwanted/unexpected results. The reference data can be + # preprocessed with 'shapely-polygonize', to create a new set of non-overlapping + # polygons. These non-overlapping polygons can be used as reference-data to align the + # theme-data. + """ # Initiate brdr aligner = Aligner() diff --git a/examples/example_dictloader_properties.py b/examples/example_dictloader_properties.py new file mode 100644 index 0000000..2c67583 --- /dev/null +++ b/examples/example_dictloader_properties.py @@ -0,0 +1,45 @@ +from brdr.aligner import Aligner +from brdr.enums import OpenbaarDomeinStrategy, AlignerResultType +from brdr.geometry_utils import geom_from_wkt +from brdr.loader import DictLoader + +if __name__ == "__main__": + """ + Example to load dat with a DictLoader, and also adding the properties to the result + """ + # CREATE AN ALIGNER + aligner = Aligner(crs="EPSG:31370", multi_as_single_modus=True) + # ADD A THEMATIC POLYGON TO THEMATIC DICTIONARY and LOAD into Aligner + id = 1 + thematic_dict = {id: geom_from_wkt("POLYGON ((0 0, 0 9, 5 10, 10 0, 0 0))")} + # Add properties + thematic_dict_properties = { + id: {"propA": 1, "propB": 1.1, "propC": "dit is tekst", "propD": None} + } + loader = DictLoader( + data_dict=thematic_dict, data_dict_properties=thematic_dict_properties + ) + aligner.load_thematic_data(loader) + # ADD A REFERENCE POLYGON TO REFERENCE DICTIONARY and LOAD into Aligner + reference_dict = {100: geom_from_wkt("POLYGON ((0 1, 0 10,8 10,10 1,0 1))")} + loader = DictLoader(reference_dict) + aligner.load_reference_data(loader) + # EXECUTE THE ALIGNMENT + relevant_distance = 1 + process_result = aligner.process( + relevant_distance=relevant_distance, + od_strategy=OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE, + threshold_overlap_percentage=50, + ) + # PRINT RESULTS IN WKT + print("result: " + process_result[id][relevant_distance]["result"].wkt) + print( + "added area: " + process_result[id][relevant_distance]["result_diff_plus"].wkt + ) + print( + "removed area: " + process_result[id][relevant_distance]["result_diff_min"].wkt + ) + fcs = aligner.get_results_as_geojson( + resulttype=AlignerResultType.PROCESSRESULTS, formula=True, attributes=True + ) + print(fcs["result"]) diff --git a/examples/example_eo.py b/examples/example_eo.py deleted file mode 100644 index dbfd77a..0000000 --- a/examples/example_eo.py +++ /dev/null @@ -1,50 +0,0 @@ -import numpy as np - -from brdr.aligner import Aligner -from brdr.enums import GRBType, AlignerResultType -from brdr.grb import GRBActualLoader -from brdr.oe import OnroerendErfgoedLoader, OEType -from brdr.utils import write_geojson -from examples import show_map, plot_series - -if __name__ == "__main__": - # EXAMPLE to test the algorithm for erfgoedobject with relevant distance 0.2m and - # od_strategy SNAP_ALL_SIDE - - # Initiate brdr - aligner = Aligner() - # Load thematic data & reference data - # dict_theme = get_oe_dict_by_ids([206363], oetype='erfgoedobjecten') - - erfgoedobjecten = [ - # 206407, - # 206403, - # 206372, - # 206369, - # 206377, - # 206371, - # 206370, - # 206368, - 206786 - ] - loader = OnroerendErfgoedLoader(objectids=erfgoedobjecten, oetype=OEType.EO) - aligner.load_thematic_data(loader) - aligner.load_reference_data(GRBActualLoader(aligner=aligner, grb_type=GRBType.ADP)) - - series = np.arange(0, 200, 20, dtype=int) / 100 - # predict which relevant distances are interesting to propose as resulting geometry - dict_series, dict_predictions, diffs = aligner.predictor( - relevant_distances=series, od_strategy=2, threshold_overlap_percentage=50 - ) - fcs = aligner.get_results_as_geojson(resulttype=AlignerResultType.PREDICTIONS) - write_geojson("output/predicted.geojson", fcs["result"]) - write_geojson("output/predicted_diff.geojson", fcs["result_diff"]) - - for key in dict_predictions.keys(): - diff = {key: diffs[key]} - plot_series(series, diff) - show_map( - {key: dict_predictions[key]}, - {key: aligner.dict_thematic[key]}, - aligner.dict_reference, - ) diff --git a/examples/example_evaluate.py b/examples/example_evaluate.py index 125912f..249b2a7 100644 --- a/examples/example_evaluate.py +++ b/examples/example_evaluate.py @@ -2,7 +2,6 @@ from datetime import date import numpy as np -from shapely import from_wkt from brdr.aligner import Aligner from brdr.constants import EVALUATION_FIELD_NAME, BASE_FORMULA_FIELD_NAME @@ -12,56 +11,65 @@ from brdr.grb import get_affected_by_grb_change from brdr.loader import DictLoader, GeoJsonFileLoader -thematic_dict = { - "theme_id_1": from_wkt( - "Polygon ((174072.91453437806922011 179188.47430499014444649, 174121.17416846146807075 179179.98909460185677744, 174116.93156326730968431 179156.47799081765697338, 174110.56765547610120848 179152.58893605635967106, 174069.37903004963300191 179159.30639428040012717, 174069.37903004963300191 179159.30639428040012717, 174070.97000699743512087 179169.7361320493509993, 174072.91453437806922011 179188.47430499014444649))" +# Press the green button in the gutter to run the script. +if __name__ == "__main__": + """ + EXAMPLE of the 'evaluate()-function of 'brdr': This function evaluates thematic objects with a former brdr_formula and compares them with an actual formula; and adds evaluation-properties to the result + """ + #initiate a base Aligner, to align thematic objects on an older version of the parcels (year 2022) + base_aligner = Aligner() + #Load thematic data + loader = GeoJsonFileLoader("themelayer.geojson", "theme_identifier") + base_aligner.load_thematic_data(loader) + base_year = "2022" + #Load reference data + base_aligner.load_reference_data( + GRBFiscalParcelLoader(year=base_year, aligner=base_aligner) ) -} -base_aligner = Aligner() -loader = GeoJsonFileLoader("themelayer.geojson", "theme_identifier") -base_aligner.load_thematic_data(loader) -base_year = "2022" -base_aligner.load_reference_data( - GRBFiscalParcelLoader(year=base_year, aligner=base_aligner) -) -relevant_distance = 2 -base_process_result = base_aligner.process(relevant_distance=relevant_distance) -thematic_dict_formula = {} -thematic_dict_result = {} -for key in base_process_result: - thematic_dict_result[key] = base_process_result[key][relevant_distance]["result"] - thematic_dict_formula[key] = { - BASE_FORMULA_FIELD_NAME: json.dumps(base_aligner.get_brdr_formula(thematic_dict_result[key])) - } - print(key + ": " + thematic_dict_result[key].wkt) - print(key + ": " + str(thematic_dict_formula[key])) -base_aligner_result = Aligner() -base_aligner_result.load_thematic_data(DictLoader(thematic_dict_result)) -affected, unaffected = get_affected_by_grb_change( - dict_thematic = thematic_dict_result, - grb_type=GRBType.ADP, - date_start=date(2022, 1, 1), - date_end=date.today(), - one_by_one=False, -) -if len(affected)==0: - print("No affected dicts") - exit() -print("Affected_IDs: " + str(affected)) -actual_aligner = Aligner() -actual_aligner.load_thematic_data( - DictLoader(data_dict=thematic_dict_result, data_dict_properties=thematic_dict_formula) -) -actual_aligner.load_reference_data( - GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner) -) -actual_aligner.relevant_distances = np.arange(0, 200, 10, dtype=int) / 100 -dict_evaluated, prop_dictionary = actual_aligner.evaluate(ids_to_compare=affected) + relevant_distance = 2 + #Align the thematic object on the parcelborders of 2022, to simulate a base-situation + base_process_result = base_aligner.process(relevant_distance=2) -fc = actual_aligner.get_results_as_geojson(resulttype=AlignerResultType.EVALUATED_PREDICTIONS,formula=True, attributes=True) -print(fc["result"]) + #Collect the base-situation (base-geometries and the brdr_formula from that moment + thematic_dict_formula = {} + thematic_dict_result = {} + for key in base_process_result: + thematic_dict_result[key] = base_process_result[key][relevant_distance]["result"] + thematic_dict_formula[key] = { + BASE_FORMULA_FIELD_NAME: json.dumps(base_aligner.get_brdr_formula(thematic_dict_result[key])) + } + print(key + ": " + thematic_dict_result[key].wkt) + print(key + ": " + str(thematic_dict_formula[key])) -for feature in fc["result"]["features"]: - id = feature["properties"][actual_aligner.name_thematic_id] - evaluation = feature["properties"][EVALUATION_FIELD_NAME] - print(id + ": " + evaluation) + #(OPTIONAL) Check for changes in the period 2022-now of the reference-parcels (GRB/Flanders-specific function) + affected, unaffected = get_affected_by_grb_change( + dict_thematic = thematic_dict_result, + grb_type=GRBType.ADP, + date_start=date(2022, 1, 1), + date_end=date.today(), + one_by_one=False, + ) + if len(affected)==0: + print("No affected dicts") + exit() + print("Affected_IDs: " + str(affected)) + + #Start an aligner to align thematic objects on the actual parcels + actual_aligner = Aligner(relevant_distances=np.arange(0, 200, 10, dtype=int) / 100) + #Load the thematic objects (aligned on 2022) and also give the brdr_formula from 2022 as property + actual_aligner.load_thematic_data( + DictLoader(data_dict=thematic_dict_result, data_dict_properties=thematic_dict_formula) + ) + #Load reference data; the actual parcels + actual_aligner.load_reference_data( + GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner) + ) + #Use the EVALUATE-function + dict_evaluated, prop_dictionary = actual_aligner.evaluate(ids_to_compare=affected) + + # SHOW the EVALUATED results + fc = actual_aligner.get_results_as_geojson(resulttype=AlignerResultType.EVALUATED_PREDICTIONS,formula=True, attributes=True) + print(fc["result"]) + + for feature in fc["result"]["features"]: + print(feature["properties"][actual_aligner.name_thematic_id] + ": " + feature["properties"][EVALUATION_FIELD_NAME]) diff --git a/examples/example_evaluate_ao.py b/examples/example_evaluate_ao.py deleted file mode 100644 index 8f10775..0000000 --- a/examples/example_evaluate_ao.py +++ /dev/null @@ -1,64 +0,0 @@ -import json -from datetime import date - -import numpy as np - -from brdr.aligner import Aligner -from brdr.constants import EVALUATION_FIELD_NAME, BASE_FORMULA_FIELD_NAME -from brdr.enums import GRBType, AlignerResultType -from brdr.grb import ( - get_affected_by_grb_change, - GRBFiscalParcelLoader, - GRBActualLoader, -) -from brdr.loader import DictLoader -from brdr.oe import OnroerendErfgoedLoader - -base_aligner = Aligner() -loader = OnroerendErfgoedLoader([120288,120108]) -base_aligner.load_thematic_data(loader) -base_year = "2022" -base_aligner.load_reference_data( - GRBFiscalParcelLoader(year=base_year, aligner=base_aligner) -) -relevant_distance = 3 -base_process_result = base_aligner.process(relevant_distance=relevant_distance) -thematic_dict_formula = {} -thematic_dict_result = {} -for key in base_process_result: - thematic_dict_result[key] = base_process_result[key][relevant_distance]["result"] - thematic_dict_formula[key] = { - BASE_FORMULA_FIELD_NAME: json.dumps(base_aligner.get_brdr_formula(thematic_dict_result[key])) - } - print(key + ": " + thematic_dict_result[key].wkt) - print(key + ": " + str(thematic_dict_formula[key])) -base_aligner_result = Aligner() -base_aligner_result.load_thematic_data(DictLoader(thematic_dict_result)) -affected, unaffected = get_affected_by_grb_change( - dict_thematic = thematic_dict_result, - grb_type=GRBType.ADP, - date_start=date(2022, 1, 1), - date_end=date.today(), - one_by_one=False, -) -if len(affected)==0: - print("No affected dicts") - exit() -print("Affected_IDs: " + str(affected)) -actual_aligner = Aligner() -actual_aligner.load_thematic_data( - DictLoader(data_dict=thematic_dict_result, data_dict_properties=thematic_dict_formula) -) -actual_aligner.load_reference_data( - GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner) -) -actual_aligner.relevant_distances = np.arange(0, 200, 10, dtype=int) / 100 -dict_evaluated, prop_dictionary = actual_aligner.evaluate(ids_to_compare=affected) - -fc = actual_aligner.get_results_as_geojson(resulttype=AlignerResultType.EVALUATED_PREDICTIONS,formula=True, attributes=True) -print(fc["result"]) - -for feature in fc["result"]["features"]: - id = feature["properties"][actual_aligner.name_thematic_id] - evaluation = feature["properties"][EVALUATION_FIELD_NAME] - print(id + ": " + evaluation) \ No newline at end of file diff --git a/examples/example_aligner.py b/examples/example_geojsonloader.py similarity index 87% rename from examples/example_aligner.py rename to examples/example_geojsonloader.py index ed08939..3f61590 100644 --- a/examples/example_aligner.py +++ b/examples/example_geojsonloader.py @@ -1,13 +1,17 @@ from brdr.aligner import Aligner -from brdr.enums import OpenbaarDomeinStrategy, GRBType +from brdr.enums import GRBType from brdr.grb import GRBActualLoader from brdr.loader import GeoJsonLoader -from examples import show_map +from examples import show_map, print_brdr_formula if __name__ == "__main__": + """ + #EXAMPLE of a Geojson, aligned by 'brdr' + """ + + # Initiate brdr aligner = Aligner() - # Load thematic data thematic_json = { "type": "FeatureCollection", @@ -45,14 +49,14 @@ loader = GeoJsonLoader(_input=thematic_json, id_property="theme_identifier") aligner.load_thematic_data(loader) + # Load reference data: The actual GRB-parcels loader = GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=aligner) aligner.load_reference_data(loader) # Example how to use the Aligner - rel_dist = 6 - dict_results = aligner.process( - relevant_distance=rel_dist, - od_strategy=OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE, - ) + dict_results = aligner.process(relevant_distance=6) + + #show results aligner.save_results("output/") show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) + print_brdr_formula(dict_results, aligner) diff --git a/examples/example_grbspecificloader.py b/examples/example_grbspecificloader.py index 88c7564..4cf151e 100644 --- a/examples/example_grbspecificloader.py +++ b/examples/example_grbspecificloader.py @@ -8,26 +8,27 @@ from brdr.loader import DictLoader from brdr.utils import write_geojson -aligner = Aligner() -thematic_dict = { - "theme_id_1": from_wkt( - "Polygon ((172283.76869662097305991 174272.85233648214489222, 172276.89871930953813717 174278.68436246179044247, 172274.71383684969623573 174280.57171753142029047, 172274.63047763772192411 174280.64478165470063686, 172272.45265833073062822 174282.52660570573061705, 172269.33533191855531186 174285.22093996312469244, 172265.55258252174826339 174288.49089696351438761, 172258.77032718938426115 174294.22654021997004747, 172258.63259260458289646 174294.342757155187428, 172254.93673790179309435 174288.79932878911495209, 172248.71360730109154247 174279.61860501393675804, 172248.96566232520854101 174279.43056782521307468, 172255.25363882273086347 174274.73737183399498463, 172257.08298882702365518 174273.37133203260600567, 172259.32325354730710387 174271.69890458136796951, 172261.65807284769834951 174269.9690355472266674, 172266.35596220899606124 174266.4871726930141449, 172273.34350050613284111 174261.30863015633076429, 172289.60360219911672175 174249.35944479051977396, 172293.30328181147342548 174246.59864199347794056, 172297.34760522318538278 174253.10583685990422964, 172289.53060952731175348 174259.6846851697191596, 172292.86485871637705714 174265.19099397677928209, 172283.76869662097305991 174272.85233648214489222))" - ) -} -#EXAMPLE to use a GRBSpecificLoader, that retrieves the parcels for a specific data, based on the 2 fiscal parcel-situations of the year before and after -#Based on the date, the referencelayer will be different -date = "2023-05-03" -date = "2023-08-03" -loader = DictLoader(thematic_dict) -aligner.load_thematic_data(loader) -loader = GRBSpecificDateParcelLoader(date=date, aligner=aligner) -aligner.load_reference_data(loader) +if __name__ == "__main__": + aligner = Aligner() + thematic_dict = { + "theme_id_1": from_wkt( + "Polygon ((172283.76869662097305991 174272.85233648214489222, 172276.89871930953813717 174278.68436246179044247, 172274.71383684969623573 174280.57171753142029047, 172274.63047763772192411 174280.64478165470063686, 172272.45265833073062822 174282.52660570573061705, 172269.33533191855531186 174285.22093996312469244, 172265.55258252174826339 174288.49089696351438761, 172258.77032718938426115 174294.22654021997004747, 172258.63259260458289646 174294.342757155187428, 172254.93673790179309435 174288.79932878911495209, 172248.71360730109154247 174279.61860501393675804, 172248.96566232520854101 174279.43056782521307468, 172255.25363882273086347 174274.73737183399498463, 172257.08298882702365518 174273.37133203260600567, 172259.32325354730710387 174271.69890458136796951, 172261.65807284769834951 174269.9690355472266674, 172266.35596220899606124 174266.4871726930141449, 172273.34350050613284111 174261.30863015633076429, 172289.60360219911672175 174249.35944479051977396, 172293.30328181147342548 174246.59864199347794056, 172297.34760522318538278 174253.10583685990422964, 172289.53060952731175348 174259.6846851697191596, 172292.86485871637705714 174265.19099397677928209, 172283.76869662097305991 174272.85233648214489222))" + ) + } + #EXAMPLE to use a GRBSpecificLoader, that retrieves the parcels for a specific data, based on the 2 fiscal parcel-situations of the year before and after + #Based on the date, the referencelayer will be different + date = "2023-05-03" + date = "2023-08-03" + loader = DictLoader(thematic_dict) + aligner.load_thematic_data(loader) + loader = GRBSpecificDateParcelLoader(date=date, aligner=aligner) + aligner.load_reference_data(loader) -# aligner.process() -# aligner.save_results(path = "output/") + # aligner.process() + # aligner.save_results(path = "output/") -fc = aligner.get_input_as_geojson( - inputtype=AlignerInputType.REFERENCE, -) -write_geojson(os.path.join("output/", "grb_adp_"+ date +".geojson"), fc) + fc = aligner.get_input_as_geojson( + inputtype=AlignerInputType.REFERENCE, + ) + write_geojson(os.path.join("output/", "grb_adp_" + date + ".geojson"), fc) diff --git a/examples/example_integer_id.py b/examples/example_integer_id.py deleted file mode 100644 index e0355fb..0000000 --- a/examples/example_integer_id.py +++ /dev/null @@ -1,39 +0,0 @@ -from brdr.aligner import Aligner -from brdr.enums import OpenbaarDomeinStrategy, AlignerResultType -from brdr.geometry_utils import geom_from_wkt -from brdr.loader import DictLoader - -# CREATE AN ALIGNER -aligner = Aligner( - crs="EPSG:31370", -) -aligner.multi_as_single_modus = True -# ADD A THEMATIC POLYGON TO THEMATIC DICTIONARY and LOAD into Aligner -id = 1 -thematic_dict = {id: geom_from_wkt("POLYGON ((0 0, 0 9, 5 10, 10 0, 0 0))")} -thematic_dict_properties = { - id: {"propA": 1, "propB": 1.1, "propC": "dit is tekst", "propD": None} -} -loader = DictLoader( - data_dict=thematic_dict, data_dict_properties=thematic_dict_properties -) -aligner.load_thematic_data(loader) -# ADD A REFERENCE POLYGON TO REFERENCE DICTIONARY and LOAD into Aligner -reference_dict = {100: geom_from_wkt("POLYGON ((0 1, 0 10,8 10,10 1,0 1))")} -loader = DictLoader(reference_dict) -aligner.load_reference_data(loader) -# EXECUTE THE ALIGNMENT -relevant_distance = 1 -process_result = aligner.process( - relevant_distance=relevant_distance, - od_strategy=OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE, - threshold_overlap_percentage=50, -) -# PRINT RESULTS IN WKT -print("result: " + process_result[id][relevant_distance]["result"].wkt) -print("added area: " + process_result[id][relevant_distance]["result_diff_plus"].wkt) -print("removed area: " + process_result[id][relevant_distance]["result_diff_min"].wkt) -fcs = aligner.get_results_as_geojson( - resulttype=AlignerResultType.PROCESSRESULTS, formula=True, attributes=True -) -print(fcs["result"]) diff --git a/examples/example_local_data.py b/examples/example_local_data.py deleted file mode 100644 index 23599fe..0000000 --- a/examples/example_local_data.py +++ /dev/null @@ -1,23 +0,0 @@ -from brdr.aligner import Aligner -from brdr.enums import OpenbaarDomeinStrategy -from brdr.loader import GeoJsonFileLoader - -if __name__ == "__main__": - # Initiate brdr - aligner = Aligner() - # Load local thematic data and reference data - loader = GeoJsonFileLoader( - "../tests/testdata/themelayer_referenced.geojson", "id_theme" - ) - aligner.load_thematic_data(loader) - loader = GeoJsonFileLoader("../tests/testdata/reference_leuven.geojson", "capakey") - aligner.load_reference_data(loader) - # Example how to use the Aligner - rel_dist = 1 - dict_results = aligner.process( - relevant_distance=rel_dist, - od_strategy=OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE, - ) - - aligner.save_results("output/") - # show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) diff --git a/examples/example_multi_to_single.py b/examples/example_multi_to_single.py deleted file mode 100644 index 3a14748..0000000 --- a/examples/example_multi_to_single.py +++ /dev/null @@ -1,46 +0,0 @@ -from brdr.aligner import Aligner -from brdr.enums import GRBType -from brdr.grb import GRBActualLoader -from brdr.oe import OnroerendErfgoedLoader, OEType -from examples import print_brdr_formula -from examples import show_map - -# This example shows the usage of the setting 'multi_as_single_modus' -# True: (default): All polygons inside a MultiPolygon will be processed seperataly by the algorithm, and merged after processing. -# False: Multipolygon will be processed directly by the algorithm - -# Example (ErfgoedObject): https://inventaris.onroerenderfgoed.be/erfgoedobjecten/305858 -loader = OnroerendErfgoedLoader([305858], oetype=OEType.EO) -relevant_distance = 5 # rd is taken very high to show the difference -od_strategy = 4 - -# EXAMPLE of "multi_as_single_modus"=FALSE -print("EXAMPLE with 'multi_as_single_modus'=False") -aligner = Aligner() -aligner.multi_as_single_modus = False -aligner.load_thematic_data(loader) -aligner.load_reference_data( - GRBActualLoader(aligner=aligner, grb_type=GRBType.GBG, partition=1000) -) -dict_results = aligner.process( - relevant_distance=relevant_distance, od_strategy=od_strategy -) -aligner.save_results("output/") -print_brdr_formula(dict_results, aligner) -show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) - - -# WITH "multi_as_single_modus"=True -print("EXAMPLE with 'multi_as_single_modus'=True") -aligner = Aligner() -aligner.multi_as_single_modus = True -aligner.load_thematic_data(loader) -aligner.load_reference_data( - GRBActualLoader(aligner=aligner, grb_type=GRBType.GBG, partition=1000) -) -dict_results = aligner.process( - relevant_distance=relevant_distance, od_strategy=od_strategy -) -aligner.save_results("output/") -print_brdr_formula(dict_results, aligner) -show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) diff --git a/examples/example_multi_to_single_modus.py b/examples/example_multi_to_single_modus.py new file mode 100644 index 0000000..16f4231 --- /dev/null +++ b/examples/example_multi_to_single_modus.py @@ -0,0 +1,53 @@ +from brdr.aligner import Aligner +from brdr.enums import GRBType, OpenbaarDomeinStrategy +from brdr.grb import GRBActualLoader +from brdr.oe import OnroerendErfgoedLoader, OEType +from examples import print_brdr_formula +from examples import show_map + +if __name__ == "__main__": + """ + # This example shows the usage of the setting 'multi_as_single_modus' + # True: (default): All polygons inside a MultiPolygon will be processed seperataly by the algorithm, and merged after processing. + # False: Multipolygon will be processed directly by the algorithm + + """ + # Example (ErfgoedObject): https://inventaris.onroerenderfgoed.be/erfgoedobjecten/305858 + loader = OnroerendErfgoedLoader([305858], oetype=OEType.EO) + relevant_distance = 5 # rd is taken very high to show the difference + od_strategy = OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE + threshold_circle_ratio = 0.75 # default it is 0.98, but because it are not fully circles in this example we put this on 0.75 + + # EXAMPLE of "multi_as_single_modus"=FALSE + print("EXAMPLE with 'multi_as_single_modus'=False") + aligner = Aligner( + multi_as_single_modus=False, + relevant_distance=relevant_distance, + od_strategy=od_strategy, + threshold_circle_ratio=threshold_circle_ratio, + ) + aligner.load_thematic_data(loader) + aligner.load_reference_data( + GRBActualLoader(aligner=aligner, grb_type=GRBType.ADP, partition=1000) + ) + dict_results = aligner.process() + aligner.save_results("output/") + print_brdr_formula(dict_results, aligner) + show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) + + # WITH "multi_as_single_modus"=True + print("EXAMPLE with 'multi_as_single_modus'=True") + aligner = Aligner( + multi_as_single_modus=True, + relevant_distance=relevant_distance, + od_strategy=od_strategy, + threshold_circle_ratio=threshold_circle_ratio, + ) + aligner.load_thematic_data(loader) + aligner.load_reference_data( + GRBActualLoader(aligner=aligner, grb_type=GRBType.ADP, partition=1000) + ) + dict_results = aligner.process() + aligner.save_results("output/") + print_brdr_formula(dict_results, aligner) + show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) diff --git a/examples/example_multipolygon.py b/examples/example_multipolygon.py deleted file mode 100644 index 0bb9450..0000000 --- a/examples/example_multipolygon.py +++ /dev/null @@ -1,38 +0,0 @@ -# Initiate brdr -from brdr.aligner import Aligner -from brdr.enums import GRBType, AlignerResultType -from brdr.grb import GRBActualLoader -from brdr.loader import DictLoader, GeoJsonFileLoader -from brdr.utils import multipolygons_to_singles -from brdr.utils import write_geojson - -aligner0 = Aligner() - -# Load thematic data - -aligner0.load_thematic_data( - GeoJsonFileLoader("../tests/testdata/multipolygon.geojson", "theme_identifier") -) -aligner0.dict_thematic, dict_multi_as_single = multipolygons_to_singles( - aligner0.dict_thematic -) -aligner0.load_thematic_data( - DictLoader( - aligner0.dict_thematic, - ) -) -# gebruik de actuele adp-percelen adp= administratieve percelen -aligner = Aligner() -aligner.load_thematic_data(DictLoader(aligner0.dict_thematic)) - -aligner.load_reference_data( - GRBActualLoader(aligner=aligner, grb_type=GRBType.ADP, partition=1000) -) - -dict_series, dict_predictions, diffs = aligner.predictor() -fcs = aligner.get_results_as_geojson( - resulttype=AlignerResultType.PREDICTIONS, formula=True -) -aligner.save_results("output/") -write_geojson("output/predicted.geojson", fcs["result"]) -write_geojson("output/predicted_diff.geojson", fcs["result_diff"]) diff --git a/examples/example_onroerenderfgoed.py b/examples/example_onroerenderfgoed.py new file mode 100644 index 0000000..9ddd800 --- /dev/null +++ b/examples/example_onroerenderfgoed.py @@ -0,0 +1,26 @@ +from brdr.aligner import Aligner +from brdr.enums import GRBType +from brdr.grb import GRBActualLoader +from brdr.oe import OnroerendErfgoedLoader, OEType +from examples import print_brdr_formula +from examples import show_map + +if __name__ == "__main__": + # EXAMPLE for a thematic Polygon from Onroerend Erfgoed (https://inventaris.onroerenderfgoed.be/aanduidingsobjecten/131635) + + # Initiate brdr + aligner = Aligner() + # Load thematic data from Onroerend Erfgoed + loader = OnroerendErfgoedLoader( objectids = [131635] , oetype=OEType.AO) + aligner.load_thematic_data(loader) + # Load reference data: The actual GRB-parcels + loader = GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=aligner) + aligner.load_reference_data(loader) + + # PROCESS + dict_results = aligner.process(relevant_distance=2) + + # GET/SHOW results + aligner.save_results("output/", formula=True) + show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) + print_brdr_formula(dict_results, aligner) diff --git a/examples/example_parcel_change_detector.py b/examples/example_parcel_change_detector.py index 6b2a7fc..b1d9b8c 100644 --- a/examples/example_parcel_change_detector.py +++ b/examples/example_parcel_change_detector.py @@ -8,14 +8,16 @@ from brdr.oe import OnroerendErfgoedLoader from brdr.utils import write_geojson -# This code shows an example how the aligner can be used inside a flow of -# parcel change detection: -# * it can be used to do a first alignment of the original features -# (in this case, based on parcel version adpF2023 (1st January 2023) -# * it can be used to do a new alignment on the actual version of the parcels adp -# * it can be used to convert the geometries to a formula, to compare and -# evaluate if equality is detected after alignement - +if __name__ == "__main__": + """ + # This code shows an example how the aligner can be used inside a flow of + # parcel change detection: + # * it can be used to do a first alignment of the original features + # (in this case, based on parcel version adpF2022 (1st January 2022) + # * it can be used to do a new alignment on the actual version of the parcels adp + # * it can be used to convert the geometries to a formula, to compare and + # evaluate if equality is detected after alignement + """ counter_excluded = 0 # PARAMS # ========= @@ -23,10 +25,6 @@ limit = 10000 bbox = [172800, 170900, 173000, 171100] bbox = [172000, 172000, 174000, 174000] -# bbox = "170000,170000,175000,174900" -# bbox = "100000,195000,105000,195900" -# bbox = "150000,210000,155000,214900" -# bbox = "173500,173500,174000,174000" # example "aanduid_id" = 34195 base_year = "2022" # relevant distance that is used to align the original geometries to the # reference-polygons of the base-year diff --git a/examples/example_parcel_vs_building.py b/examples/example_parcel_vs_building.py index 0d1fdc6..2003b75 100644 --- a/examples/example_parcel_vs_building.py +++ b/examples/example_parcel_vs_building.py @@ -7,9 +7,11 @@ from brdr.utils import diffs_from_dict_series from examples import plot_series -# example to check if we can notice if it is better to align to a building instead of a -# parcel if __name__ == "__main__": + """ + # example to check if we can notice if it is better to align to a building instead of a + # parcel + """ # Initiate brdr aligner_x = Aligner() # Load thematic data & reference data (parcels) diff --git a/examples/example_predictor.py b/examples/example_predictor.py index e7d60ac..29127c4 100644 --- a/examples/example_predictor.py +++ b/examples/example_predictor.py @@ -1,14 +1,15 @@ import numpy as np from brdr.aligner import Aligner -from brdr.enums import GRBType, AlignerResultType +from brdr.enums import GRBType, AlignerResultType, OpenbaarDomeinStrategy from brdr.grb import GRBActualLoader from brdr.loader import GeoJsonFileLoader +from examples import show_map, plot_series # Press the green button in the gutter to run the script. if __name__ == "__main__": """ - example to use the predictor-function to automatically predict which resulting + EXAMPLE to use the predictor-function to automatically predict which resulting geometries are interesting to look at (based on detection of breakpoints and relevant distances of 'no-change') """ @@ -19,21 +20,26 @@ "../tests/testdata/test_wanted_changes.geojson", "theme_id" ) aligner.load_thematic_data(loader) + # Load reference data loader = GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=aligner) aligner.load_reference_data(loader) + #PREDICT the 'stable' relevant distances, for a series of relevant distances series = np.arange(0, 300, 10, dtype=int) / 100 # predict which relevant distances are interesting to propose as resulting geometry dict_series, dict_predictions, diffs = aligner.predictor( - relevant_distances=series, od_strategy=4, threshold_overlap_percentage=50 + relevant_distances=series, od_strategy=OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE, threshold_overlap_percentage=50 ) + + #SHOW results of the predictions fcs = aligner.get_results_as_geojson( resulttype=AlignerResultType.PREDICTIONS, formula=False ) print(fcs["result"]) - # for key in dict_predictions: - # show_map( - # {key:dict_predictions[key]}, - # {key: aligner.dict_thematic[key]}, - # aligner.dict_reference, - # ) + for key in dict_predictions: + plot_series(series, {key:diffs[key]}) + show_map( + {key:dict_predictions[key]}, + {key: aligner.dict_thematic[key]}, + aligner.dict_reference, + ) diff --git a/examples/example_process_relevant_distances.py b/examples/example_process_relevant_distances.py new file mode 100644 index 0000000..1c7a8fe --- /dev/null +++ b/examples/example_process_relevant_distances.py @@ -0,0 +1,64 @@ +from brdr.aligner import Aligner +from brdr.enums import OpenbaarDomeinStrategy, GRBType +from brdr.grb import GRBActualLoader +from brdr.loader import GeoJsonLoader +from brdr.utils import diffs_from_dict_series +from examples import plot_series +from examples import show_map + +if __name__ == "__main__": + # EXAMPLE to process a series of relevant distances + # Initiate brdr + aligner = Aligner() + # Load thematic data + thematic_json = { + "type": "FeatureCollection", + "name": "test", + "crs": {"type": "name", "properties": {"name": "urn:ogc:def:crs:EPSG::31370"}}, + "features": [ + { + "type": "Feature", + "properties": {"fid": 1100, "id": 1100, "theme_identifier": "1100"}, + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [170020.885142877610633, 171986.324472956912359], + [170078.339491307124263, 172031.344329243671382], + [170049.567976467020344, 172070.009247593494365], + [170058.413533725659363, 172089.43287940043956], + [170071.570170061604585, 172102.403589786874363], + [170061.212614970601862, 172153.235667688539252], + [170008.670597244432429, 172137.344562214449979], + [169986.296310421457747, 172121.231194059771951], + [169979.658902874827618, 172095.676061166130239], + [169980.509304589155363, 172078.495172578957863], + [169985.424311356124235, 172065.162689057324314], + [169971.609697333740769, 172057.093288242496783], + [170020.885142877610633, 171986.324472956912359], + ] + ] + ], + }, + } + ], + } + + loader = GeoJsonLoader(_input=thematic_json, id_property="theme_identifier") + aligner.load_thematic_data(loader) + # Load reference data: The actual GRB-parcels + aligner.load_reference_data( + GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=aligner) + ) + # PROCESS a series of relevant distances + relevant_distances = [0.5, 1, 3, 6] + dict_results = aligner.process( + relevant_distances=relevant_distances, + od_strategy=OpenbaarDomeinStrategy.SNAP_ALL_SIDE, + threshold_overlap_percentage=50, + ) + # SHOW results: map and plotted changes + show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) + resulting_areas = diffs_from_dict_series(dict_results, aligner.dict_thematic) + plot_series(relevant_distances, resulting_areas) diff --git a/examples/example_speedtest.py b/examples/example_speedtest.py index 973d191..d1b6da8 100644 --- a/examples/example_speedtest.py +++ b/examples/example_speedtest.py @@ -6,6 +6,10 @@ def main(): + """ + EXAMPLE of a test to measure the speed of the aligner + :return: + """ # Initiate brdr aligner = Aligner(relevant_distance=2, max_workers=None) iterations = 10 diff --git a/examples/example_update_to_actual_grb.py b/examples/example_update_to_actual_grb.py index cd79b85..1940e14 100644 --- a/examples/example_update_to_actual_grb.py +++ b/examples/example_update_to_actual_grb.py @@ -4,76 +4,84 @@ from brdr.grb import update_to_actual_grb from brdr.loader import GeoJsonLoader -# Create a featurecollection (aligned on 2022), to use for the 'update_to_actual_grb' -base_year = "2022" -base_aligner = Aligner() -name_thematic_id = "theme_identifier" -loader = GeoJsonLoader( - _input={ - "type": "FeatureCollection", - "name": "extract", - "crs": {"type": "name", "properties": {"name": "urn:ogc:def:crs:EPSG::31370"}}, - "features": [ - { - "type": "Feature", - "properties": { - "testattribute": "test", - "nr_calculations": 1, - "ID": "206285", - "relevant_distance": 2.0, - "area": 503.67736346047076, - "perimeter": 125.74541473322422, - "shape_index": 0.24965468741597097, - }, - "geometry": { - "type": "MultiPolygon", - "coordinates": [ - [ + +if __name__ == "__main__": + """ + EXAMPLE of the use of GRB (flanders-specific) function: Update_to_actual_grb + """ + # Create a featurecollection (aligned on 2022), to use for the 'update_to_actual_grb' + base_year = "2022" + base_aligner = Aligner() + name_thematic_id = "theme_identifier" + loader = GeoJsonLoader( + _input={ + "type": "FeatureCollection", + "name": "extract", + "crs": { + "type": "name", + "properties": {"name": "urn:ogc:def:crs:EPSG::31370"}, + }, + "features": [ + { + "type": "Feature", + "properties": { + "testattribute": "test", + "nr_calculations": 1, + "ID": "206285", + "relevant_distance": 2.0, + "area": 503.67736346047076, + "perimeter": 125.74541473322422, + "shape_index": 0.24965468741597097, + }, + "geometry": { + "type": "MultiPolygon", + "coordinates": [ [ - [138539.326299999986077, 193994.138199999986682], - [138529.3663, 193995.566400000010617], - [138522.0997, 193996.6084], - [138514.984399999986636, 193997.6287], - [138505.8261, 193996.615], - [138498.8406, 193996.4314], - [138492.9442, 193996.289500000013504], - [138491.224599999986822, 193996.2481], - [138491.4111, 194004.814699999988079], - [138514.368500000011409, 194005.1297], - [138520.2585, 194004.5753], - [138520.3946, 194005.5833], - [138520.542599999986123, 194009.731999999989057], - [138541.4173, 194007.7292], - [138539.326299999986077, 193994.138199999986682], + [ + [138539.326299999986077, 193994.138199999986682], + [138529.3663, 193995.566400000010617], + [138522.0997, 193996.6084], + [138514.984399999986636, 193997.6287], + [138505.8261, 193996.615], + [138498.8406, 193996.4314], + [138492.9442, 193996.289500000013504], + [138491.224599999986822, 193996.2481], + [138491.4111, 194004.814699999988079], + [138514.368500000011409, 194005.1297], + [138520.2585, 194004.5753], + [138520.3946, 194005.5833], + [138520.542599999986123, 194009.731999999989057], + [138541.4173, 194007.7292], + [138539.326299999986077, 193994.138199999986682], + ] ] - ] - ], - }, - } - ], - }, - id_property="ID", -) -base_aligner.load_thematic_data(loader) -base_aligner.load_reference_data( - GRBFiscalParcelLoader(year=base_year, aligner=base_aligner) -) -base_process_result = base_aligner.process(relevant_distance=2) -fcs = base_aligner.get_results_as_geojson(formula=True, attributes=True) -featurecollection_base_result = fcs["result"] -print(featurecollection_base_result) -# Update Featurecollection to actual version -featurecollection = update_to_actual_grb( - featurecollection_base_result, - base_aligner.name_thematic_id, - base_formula_field=FORMULA_FIELD_NAME, -) -# Print results -for feature in featurecollection["result"]["features"]: - print( - feature["properties"][name_thematic_id] - + ": " - + feature["properties"][EVALUATION_FIELD_NAME] + ], + }, + } + ], + }, + id_property="ID", + ) + base_aligner.load_thematic_data(loader) + base_aligner.load_reference_data( + GRBFiscalParcelLoader(year=base_year, aligner=base_aligner) + ) + base_process_result = base_aligner.process(relevant_distance=2) + fcs = base_aligner.get_results_as_geojson(formula=True, attributes=True) + featurecollection_base_result = fcs["result"] + print(featurecollection_base_result) + # Update Featurecollection to actual version + featurecollection = update_to_actual_grb( + featurecollection_base_result, + base_aligner.name_thematic_id, + base_formula_field=FORMULA_FIELD_NAME, ) -geojson = featurecollection["result"] -print(geojson) + # Print results + for feature in featurecollection["result"]["features"]: + print( + feature["properties"][name_thematic_id] + + ": " + + feature["properties"][EVALUATION_FIELD_NAME] + ) + geojson = featurecollection["result"] + print(geojson) From 7a81e6198aae3aa3796ed751d8c39a8f84688006 Mon Sep 17 00:00:00 2001 From: dieuska Date: Thu, 3 Oct 2024 15:52:12 +0200 Subject: [PATCH 23/26] fix constant formula field name --- brdr/aligner.py | 20 +++++++++++--------- brdr/constants.py | 2 +- brdr/grb.py | 16 +++++++--------- examples/example_evaluate.py | 7 ++++--- tests/test_aligner.py | 7 ++++--- 5 files changed, 27 insertions(+), 25 deletions(-) diff --git a/brdr/aligner.py b/brdr/aligner.py index a8b417a..3cd4aec 100644 --- a/brdr/aligner.py +++ b/brdr/aligner.py @@ -19,7 +19,7 @@ from shapely.geometry.base import BaseGeometry from brdr import __version__ -from brdr.constants import DEFAULT_CRS, BASE_FORMULA_FIELD_NAME +from brdr.constants import DEFAULT_CRS from brdr.constants import ( LAST_VERSION_DATE, VERSION_DATE, @@ -625,19 +625,20 @@ def predictor( def evaluate( self, - ids_to_compare=None, + ids_to_evaluate=None, base_formula_field= FORMULA_FIELD_NAME ): """ + Compares and evaluate input-geometries (with formula). Attributes are added to evaluate and decide if new proposals can be used affected: list with all IDs to evaluate. all other IDs will be unchanged. If None (default), all self.dict_thematic will be evaluated. """ - if ids_to_compare is None: - ids_to_compare = list(self.dict_thematic.keys()) + if ids_to_evaluate is None: + ids_to_evaluate = list(self.dict_thematic.keys()) dict_affected = {} dict_unaffected = {} for id_theme, geom in self.dict_thematic.items(): - if id_theme in ids_to_compare: + if id_theme in ids_to_evaluate: dict_affected[id_theme] = geom else: dict_unaffected[id_theme] = geom @@ -656,6 +657,7 @@ def evaluate( props = self._evaluate( id_theme=theme_id, geom_predicted=dict_predictions_results[dist]["result"], + base_formula_field=base_formula_field, ) dict_predictions_evaluated[theme_id][dist] = dict_affected_predictions[ theme_id @@ -668,7 +670,7 @@ def evaluate( for theme_id, geom in dict_unaffected.items(): dict_predictions_evaluated[theme_id] = {} prop_dictionary[theme_id] = {relevant_distance: {}} - props = self._evaluate(id_theme=theme_id, geom_predicted=geom) + props = self._evaluate(id_theme=theme_id, geom_predicted=geom,base_formula_field=base_formula_field) props[EVALUATION_FIELD_NAME] = Evaluation.NO_CHANGE_6 dict_predictions_evaluated[theme_id][relevant_distance] = {"result": geom} prop_dictionary[theme_id][relevant_distance] = props @@ -1352,7 +1354,7 @@ def _postprocess_preresult( "remark": remark, } - def _evaluate(self, id_theme, geom_predicted): + def _evaluate(self, id_theme, geom_predicted,base_formula_field=FORMULA_FIELD_NAME): """ function that evaluates a predicted geometry and returns a properties-dictionary """ @@ -1372,10 +1374,10 @@ def _evaluate(self, id_theme, geom_predicted): base_formula = None if ( id_theme in self.dict_thematic_properties - and BASE_FORMULA_FIELD_NAME in self.dict_thematic_properties[id_theme] + and base_formula_field in self.dict_thematic_properties[id_theme] ): base_formula = json.loads( - self.dict_thematic_properties[id_theme][BASE_FORMULA_FIELD_NAME] + self.dict_thematic_properties[id_theme][base_formula_field] ) if base_formula is None or actual_formula is None: diff --git a/brdr/constants.py b/brdr/constants.py index 5eac700..d3d0508 100644 --- a/brdr/constants.py +++ b/brdr/constants.py @@ -8,7 +8,7 @@ MULTI_SINGLE_ID_SEPARATOR = "*$*" PREFIX_FIELDNAME = "brdr_" -BASE_FORMULA_FIELD_NAME = PREFIX_FIELDNAME + "base_formula" +BASE_FORMULA_FIELD_NAME = PREFIX_FIELDNAME + "base_formula"# for use in grb_actualisation FORMULA_FIELD_NAME = PREFIX_FIELDNAME + "formula" EVALUATION_FIELD_NAME = PREFIX_FIELDNAME + "evaluation" DIFF_PERCENTAGE_FIELD_NAME = PREFIX_FIELDNAME + "diff_percentage" diff --git a/brdr/grb.py b/brdr/grb.py index f8a0cf1..c85fe59 100644 --- a/brdr/grb.py +++ b/brdr/grb.py @@ -14,7 +14,7 @@ LAST_VERSION_DATE, DATE_FORMAT, VERSION_DATE, - BASE_FORMULA_FIELD_NAME, + FORMULA_FIELD_NAME, BASE_FORMULA_FIELD_NAME, ) from brdr.constants import DOWNLOAD_LIMIT from brdr.constants import GRB_BUILDING_ID @@ -344,7 +344,7 @@ def get_collection_grb_parcels_by_date( def update_to_actual_grb( featurecollection, id_theme_fieldname, - base_formula_field=BASE_FORMULA_FIELD_NAME, + base_formula_field=FORMULA_FIELD_NAME, max_distance_for_actualisation=2, feedback=None, attributes=True, @@ -375,12 +375,10 @@ def update_to_actual_grb( dict_thematic[id_theme] = geom dict_thematic_props[id_theme] = feature["properties"] try: - dict_thematic_props[id_theme][BASE_FORMULA_FIELD_NAME] = feature[ - "properties" - ][base_formula_field] - base_formula = json.loads( - dict_thematic_props[id_theme][BASE_FORMULA_FIELD_NAME] - ) + base_formula_string = feature["properties"][base_formula_field] + dict_thematic_props[id_theme][BASE_FORMULA_FIELD_NAME] = base_formula_string + base_formula = json.loads(base_formula_string) + logger.feedback_debug("formula: " + str(base_formula)) except Exception: raise Exception("Formula -attribute-field (json) cannot be loaded") @@ -436,7 +434,7 @@ def update_to_actual_grb( actual_aligner.relevant_distances = ( np.arange(0, max_distance_for_actualisation * 100, 10, dtype=int) / 100 ) - dict_evaluated, prop_dictionary = actual_aligner.evaluate(ids_to_compare=affected) + dict_evaluated, prop_dictionary = actual_aligner.evaluate(ids_to_evaluate=affected,base_formula_field=BASE_FORMULA_FIELD_NAME) return actual_aligner.get_results_as_geojson( resulttype=AlignerResultType.EVALUATED_PREDICTIONS, diff --git a/examples/example_evaluate.py b/examples/example_evaluate.py index 249b2a7..28f2cde 100644 --- a/examples/example_evaluate.py +++ b/examples/example_evaluate.py @@ -4,7 +4,7 @@ import numpy as np from brdr.aligner import Aligner -from brdr.constants import EVALUATION_FIELD_NAME, BASE_FORMULA_FIELD_NAME +from brdr.constants import EVALUATION_FIELD_NAME from brdr.enums import GRBType, AlignerResultType from brdr.grb import GRBActualLoader from brdr.grb import GRBFiscalParcelLoader @@ -22,6 +22,7 @@ loader = GeoJsonFileLoader("themelayer.geojson", "theme_identifier") base_aligner.load_thematic_data(loader) base_year = "2022" + name_formula = "base_formula" #Load reference data base_aligner.load_reference_data( GRBFiscalParcelLoader(year=base_year, aligner=base_aligner) @@ -36,7 +37,7 @@ for key in base_process_result: thematic_dict_result[key] = base_process_result[key][relevant_distance]["result"] thematic_dict_formula[key] = { - BASE_FORMULA_FIELD_NAME: json.dumps(base_aligner.get_brdr_formula(thematic_dict_result[key])) + name_formula: json.dumps(base_aligner.get_brdr_formula(thematic_dict_result[key])) } print(key + ": " + thematic_dict_result[key].wkt) print(key + ": " + str(thematic_dict_formula[key])) @@ -65,7 +66,7 @@ GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner) ) #Use the EVALUATE-function - dict_evaluated, prop_dictionary = actual_aligner.evaluate(ids_to_compare=affected) + dict_evaluated, prop_dictionary = actual_aligner.evaluate(ids_to_evaluate=affected,base_formula_field=name_formula) # SHOW the EVALUATED results fc = actual_aligner.get_results_as_geojson(resulttype=AlignerResultType.EVALUATED_PREDICTIONS,formula=True, attributes=True) diff --git a/tests/test_aligner.py b/tests/test_aligner.py index 1b36a30..7a48fa4 100644 --- a/tests/test_aligner.py +++ b/tests/test_aligner.py @@ -1,3 +1,4 @@ +import json import os import unittest from datetime import date @@ -379,8 +380,8 @@ def test_evaluate(self): "result" ] thematic_dict_formula[key] = { - FORMULA_FIELD_NAME: base_aligner.get_brdr_formula( - thematic_dict_result[key] + FORMULA_FIELD_NAME: json.dumps(base_aligner.get_brdr_formula( + thematic_dict_result[key]) ) } print(key + ": " + thematic_dict_result[key].wkt) @@ -412,7 +413,7 @@ def test_evaluate(self): ) actual_aligner.relevant_distances = np.arange(0, 200, 10, dtype=int) / 100 dict_evaluated, prop_dictionary = actual_aligner.evaluate( - ids_to_compare=affected + ids_to_evaluate=affected,base_formula_field=FORMULA_FIELD_NAME ) fc = get_series_geojson_dict( From 61a9054c0befc03eca88cb2fbbe6d68ab3bf9433 Mon Sep 17 00:00:00 2001 From: dieuska Date: Thu, 3 Oct 2024 16:50:27 +0200 Subject: [PATCH 24/26] commit for release_040 --- CHANGES.md | 17 +++++++ brdr/__init__.py | 2 +- brdr/aligner.py | 23 +++++----- brdr/constants.py | 4 +- brdr/geometry_utils.py | 16 ++++--- brdr/grb.py | 7 ++- examples/example_evaluate.py | 52 ++++++++++++++-------- examples/example_geojsonloader.py | 3 +- examples/example_onroerenderfgoed.py | 2 +- examples/example_parcel_change_detector.py | 22 +++++---- examples/example_parcel_vs_building.py | 8 +++- pyproject.toml | 2 +- tests/test_aligner.py | 35 +++++++-------- 13 files changed, 121 insertions(+), 72 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7248ff5..027f7c4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -40,7 +40,24 @@ - Bugfixing: - adding a safe_equals-function to catch GEOsException bug [#71] +# 0.4.0 +! Not Backwards compatable ! + +- Refactoring: + - Possibility for parallel processing [#97] + - Changed Aligner constants to init-settings [#83] + - Refactored ID-handling so strings,integers,... can be used as ID[#110] + - Cleaned examples [#100] + + - Functionalities: + - Added evaluation-attributes to evaluate()-function [#99] + - processing-remarks available in geojson-output [#103] + - Added warning when input/output changed from polygon/multipolygon [#107] + +- Bugfixing: + - Bugfix on version_date [#96] + - Bugfix on disappearing features [#105] diff --git a/brdr/__init__.py b/brdr/__init__.py index 493f741..6a9beea 100644 --- a/brdr/__init__.py +++ b/brdr/__init__.py @@ -1 +1 @@ -__version__ = "0.3.0" +__version__ = "0.4.0" diff --git a/brdr/aligner.py b/brdr/aligner.py index 3cd4aec..43c3536 100644 --- a/brdr/aligner.py +++ b/brdr/aligner.py @@ -623,10 +623,7 @@ def predictor( diffs_dict, ) - def evaluate( - self, - ids_to_evaluate=None, base_formula_field= FORMULA_FIELD_NAME - ): + def evaluate(self, ids_to_evaluate=None, base_formula_field=FORMULA_FIELD_NAME): """ Compares and evaluate input-geometries (with formula). Attributes are added to evaluate and decide if new @@ -657,7 +654,7 @@ def evaluate( props = self._evaluate( id_theme=theme_id, geom_predicted=dict_predictions_results[dist]["result"], - base_formula_field=base_formula_field, + base_formula_field=base_formula_field, ) dict_predictions_evaluated[theme_id][dist] = dict_affected_predictions[ theme_id @@ -670,7 +667,11 @@ def evaluate( for theme_id, geom in dict_unaffected.items(): dict_predictions_evaluated[theme_id] = {} prop_dictionary[theme_id] = {relevant_distance: {}} - props = self._evaluate(id_theme=theme_id, geom_predicted=geom,base_formula_field=base_formula_field) + props = self._evaluate( + id_theme=theme_id, + geom_predicted=geom, + base_formula_field=base_formula_field, + ) props[EVALUATION_FIELD_NAME] = Evaluation.NO_CHANGE_6 dict_predictions_evaluated[theme_id][relevant_distance] = {"result": geom} prop_dictionary[theme_id][relevant_distance] = props @@ -840,9 +841,9 @@ def get_results_as_geojson( ): # and not (theme_id in prop_dictionary and relevant_distance in prop_dictionary[theme_id] and NEW_FORMULA_FIELD_NAME in prop_dictionary[theme_id][relevant_distance]): result = process_results["result"] formula = self.get_brdr_formula(result) - prop_dictionary[theme_id][relevant_distance][ - FORMULA_FIELD_NAME - ] = json.dumps(formula) + prop_dictionary[theme_id][relevant_distance][FORMULA_FIELD_NAME] = ( + json.dumps(formula) + ) return get_series_geojson_dict( dict_series, crs=self.CRS, @@ -1354,7 +1355,9 @@ def _postprocess_preresult( "remark": remark, } - def _evaluate(self, id_theme, geom_predicted,base_formula_field=FORMULA_FIELD_NAME): + def _evaluate( + self, id_theme, geom_predicted, base_formula_field=FORMULA_FIELD_NAME + ): """ function that evaluates a predicted geometry and returns a properties-dictionary """ diff --git a/brdr/constants.py b/brdr/constants.py index d3d0508..4fbdfb1 100644 --- a/brdr/constants.py +++ b/brdr/constants.py @@ -8,7 +8,9 @@ MULTI_SINGLE_ID_SEPARATOR = "*$*" PREFIX_FIELDNAME = "brdr_" -BASE_FORMULA_FIELD_NAME = PREFIX_FIELDNAME + "base_formula"# for use in grb_actualisation +BASE_FORMULA_FIELD_NAME = ( + PREFIX_FIELDNAME + "base_formula" +) # for use in grb_actualisation FORMULA_FIELD_NAME = PREFIX_FIELDNAME + "formula" EVALUATION_FIELD_NAME = PREFIX_FIELDNAME + "evaluation" DIFF_PERCENTAGE_FIELD_NAME = PREFIX_FIELDNAME + "diff_percentage" diff --git a/brdr/geometry_utils.py b/brdr/geometry_utils.py index 2840522..89b1aaa 100644 --- a/brdr/geometry_utils.py +++ b/brdr/geometry_utils.py @@ -23,7 +23,7 @@ from shapely.prepared import prep -def buffer_neg_pos(geometry, buffer_value,mitre_limit=5): +def buffer_neg_pos(geometry, buffer_value, mitre_limit=5): """ Computes two buffers accordingly: one with a negative buffer value and another with a positive buffer value. This function can be used the check where relevant areas @@ -56,18 +56,18 @@ def buffer_neg_pos(geometry, buffer_value,mitre_limit=5): buffer( geometry, -buffer_value, - #quad_segs=QUAD_SEGMENTS, + # quad_segs=QUAD_SEGMENTS, join_style="mitre", mitre_limit=mitre_limit, ), buffer_value, - #quad_segs=QUAD_SEGMENTS, + # quad_segs=QUAD_SEGMENTS, join_style="mitre", mitre_limit=mitre_limit, ) -def buffer_neg(geometry, buffer_value,mitre_limit=5): +def buffer_neg(geometry, buffer_value, mitre_limit=5): """ Computes the negative buffer of a given geometric object. @@ -94,13 +94,13 @@ def buffer_neg(geometry, buffer_value,mitre_limit=5): return buffer( geometry, -buffer_value, - #quad_segs=QUAD_SEGMENTS, + # quad_segs=QUAD_SEGMENTS, join_style="mitre", mitre_limit=mitre_limit, ) -def buffer_pos(geometry, buffer_value,mitre_limit=5): +def buffer_pos(geometry, buffer_value, mitre_limit=5): """ Computes the positive buffer of a given geometric object. @@ -127,7 +127,7 @@ def buffer_pos(geometry, buffer_value,mitre_limit=5): return buffer( geometry, buffer_value, - #quad_segs=QUAD_SEGMENTS, + # quad_segs=QUAD_SEGMENTS, join_style="mitre", mitre_limit=mitre_limit, ) @@ -506,9 +506,11 @@ def fill_and_remove_gaps(input_geometry, buffer_value): ix_part = ix_part + 1 return cleaned_geometry + def safe_unary_union(geometries): return make_valid(unary_union(geometries)) + def get_bbox(geometry): """ Get the BBOX (string) of a shapely geometry diff --git a/brdr/grb.py b/brdr/grb.py index c85fe59..56a34c1 100644 --- a/brdr/grb.py +++ b/brdr/grb.py @@ -14,7 +14,8 @@ LAST_VERSION_DATE, DATE_FORMAT, VERSION_DATE, - FORMULA_FIELD_NAME, BASE_FORMULA_FIELD_NAME, + FORMULA_FIELD_NAME, + BASE_FORMULA_FIELD_NAME, ) from brdr.constants import DOWNLOAD_LIMIT from brdr.constants import GRB_BUILDING_ID @@ -434,7 +435,9 @@ def update_to_actual_grb( actual_aligner.relevant_distances = ( np.arange(0, max_distance_for_actualisation * 100, 10, dtype=int) / 100 ) - dict_evaluated, prop_dictionary = actual_aligner.evaluate(ids_to_evaluate=affected,base_formula_field=BASE_FORMULA_FIELD_NAME) + dict_evaluated, prop_dictionary = actual_aligner.evaluate( + ids_to_evaluate=affected, base_formula_field=BASE_FORMULA_FIELD_NAME + ) return actual_aligner.get_results_as_geojson( resulttype=AlignerResultType.EVALUATED_PREDICTIONS, diff --git a/examples/example_evaluate.py b/examples/example_evaluate.py index 28f2cde..520f711 100644 --- a/examples/example_evaluate.py +++ b/examples/example_evaluate.py @@ -16,61 +16,77 @@ """ EXAMPLE of the 'evaluate()-function of 'brdr': This function evaluates thematic objects with a former brdr_formula and compares them with an actual formula; and adds evaluation-properties to the result """ - #initiate a base Aligner, to align thematic objects on an older version of the parcels (year 2022) + # initiate a base Aligner, to align thematic objects on an older version of the parcels (year 2022) base_aligner = Aligner() - #Load thematic data + # Load thematic data loader = GeoJsonFileLoader("themelayer.geojson", "theme_identifier") base_aligner.load_thematic_data(loader) base_year = "2022" name_formula = "base_formula" - #Load reference data + # Load reference data base_aligner.load_reference_data( GRBFiscalParcelLoader(year=base_year, aligner=base_aligner) ) relevant_distance = 2 - #Align the thematic object on the parcelborders of 2022, to simulate a base-situation + # Align the thematic object on the parcelborders of 2022, to simulate a base-situation base_process_result = base_aligner.process(relevant_distance=2) - #Collect the base-situation (base-geometries and the brdr_formula from that moment + # Collect the base-situation (base-geometries and the brdr_formula from that moment thematic_dict_formula = {} thematic_dict_result = {} for key in base_process_result: - thematic_dict_result[key] = base_process_result[key][relevant_distance]["result"] + thematic_dict_result[key] = base_process_result[key][relevant_distance][ + "result" + ] thematic_dict_formula[key] = { - name_formula: json.dumps(base_aligner.get_brdr_formula(thematic_dict_result[key])) + name_formula: json.dumps( + base_aligner.get_brdr_formula(thematic_dict_result[key]) + ) } print(key + ": " + thematic_dict_result[key].wkt) print(key + ": " + str(thematic_dict_formula[key])) - #(OPTIONAL) Check for changes in the period 2022-now of the reference-parcels (GRB/Flanders-specific function) + # (OPTIONAL) Check for changes in the period 2022-now of the reference-parcels (GRB/Flanders-specific function) affected, unaffected = get_affected_by_grb_change( - dict_thematic = thematic_dict_result, + dict_thematic=thematic_dict_result, grb_type=GRBType.ADP, date_start=date(2022, 1, 1), date_end=date.today(), one_by_one=False, ) - if len(affected)==0: + if len(affected) == 0: print("No affected dicts") exit() print("Affected_IDs: " + str(affected)) - #Start an aligner to align thematic objects on the actual parcels + # Start an aligner to align thematic objects on the actual parcels actual_aligner = Aligner(relevant_distances=np.arange(0, 200, 10, dtype=int) / 100) - #Load the thematic objects (aligned on 2022) and also give the brdr_formula from 2022 as property + # Load the thematic objects (aligned on 2022) and also give the brdr_formula from 2022 as property actual_aligner.load_thematic_data( - DictLoader(data_dict=thematic_dict_result, data_dict_properties=thematic_dict_formula) + DictLoader( + data_dict=thematic_dict_result, data_dict_properties=thematic_dict_formula + ) ) - #Load reference data; the actual parcels + # Load reference data; the actual parcels actual_aligner.load_reference_data( GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner) ) - #Use the EVALUATE-function - dict_evaluated, prop_dictionary = actual_aligner.evaluate(ids_to_evaluate=affected,base_formula_field=name_formula) + # Use the EVALUATE-function + dict_evaluated, prop_dictionary = actual_aligner.evaluate( + ids_to_evaluate=affected, base_formula_field=name_formula + ) # SHOW the EVALUATED results - fc = actual_aligner.get_results_as_geojson(resulttype=AlignerResultType.EVALUATED_PREDICTIONS,formula=True, attributes=True) + fc = actual_aligner.get_results_as_geojson( + resulttype=AlignerResultType.EVALUATED_PREDICTIONS, + formula=True, + attributes=True, + ) print(fc["result"]) for feature in fc["result"]["features"]: - print(feature["properties"][actual_aligner.name_thematic_id] + ": " + feature["properties"][EVALUATION_FIELD_NAME]) + print( + feature["properties"][actual_aligner.name_thematic_id] + + ": " + + feature["properties"][EVALUATION_FIELD_NAME] + ) diff --git a/examples/example_geojsonloader.py b/examples/example_geojsonloader.py index 3f61590..4ae7adf 100644 --- a/examples/example_geojsonloader.py +++ b/examples/example_geojsonloader.py @@ -9,7 +9,6 @@ #EXAMPLE of a Geojson, aligned by 'brdr' """ - # Initiate brdr aligner = Aligner() # Load thematic data @@ -56,7 +55,7 @@ # Example how to use the Aligner dict_results = aligner.process(relevant_distance=6) - #show results + # show results aligner.save_results("output/") show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) print_brdr_formula(dict_results, aligner) diff --git a/examples/example_onroerenderfgoed.py b/examples/example_onroerenderfgoed.py index 9ddd800..c862786 100644 --- a/examples/example_onroerenderfgoed.py +++ b/examples/example_onroerenderfgoed.py @@ -11,7 +11,7 @@ # Initiate brdr aligner = Aligner() # Load thematic data from Onroerend Erfgoed - loader = OnroerendErfgoedLoader( objectids = [131635] , oetype=OEType.AO) + loader = OnroerendErfgoedLoader(objectids=[131635], oetype=OEType.AO) aligner.load_thematic_data(loader) # Load reference data: The actual GRB-parcels loader = GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=aligner) diff --git a/examples/example_parcel_change_detector.py b/examples/example_parcel_change_detector.py index b1d9b8c..375d915 100644 --- a/examples/example_parcel_change_detector.py +++ b/examples/example_parcel_change_detector.py @@ -2,7 +2,11 @@ from datetime import datetime from brdr.aligner import Aligner -from brdr.constants import EVALUATION_FIELD_NAME, RELEVANT_DISTANCE_FIELD_NAME, FORMULA_FIELD_NAME +from brdr.constants import ( + EVALUATION_FIELD_NAME, + RELEVANT_DISTANCE_FIELD_NAME, + FORMULA_FIELD_NAME, +) from brdr.grb import GRBFiscalParcelLoader from brdr.grb import update_to_actual_grb from brdr.oe import OnroerendErfgoedLoader @@ -49,7 +53,7 @@ + str(len(base_aligner.dict_thematic)) ) base_aligner.load_reference_data( - GRBFiscalParcelLoader(year=base_year, aligner=base_aligner,partition=1000) + GRBFiscalParcelLoader(year=base_year, aligner=base_aligner, partition=1000) ) print("Reference-data loaded") # Exclude objects bigger than specified area @@ -59,9 +63,7 @@ if base_aligner.dict_thematic[key].area > excluded_area: keys_to_exclude.append(key) counter_excluded = counter_excluded + 1 - print( - "geometrie excluded; bigger than " + str(excluded_area) + ": " + key - ) + print("geometrie excluded; bigger than " + str(excluded_area) + ": " + key) for x in keys_to_exclude: del base_aligner.dict_thematic[x] @@ -85,15 +87,17 @@ max_distance_for_actualisation=max_distance_for_actualisation, ) -write_geojson(os.path.join("output/", "parcel_change_detector_with.geojson"), fcs["result"]) +write_geojson( + os.path.join("output/", "parcel_change_detector_with.geojson"), fcs["result"] +) counter_equality = 0 counter_equality_by_alignment = 0 counter_difference = 0 counter_no_change = 0 -#TODO: counter_difference collects al the 'TO_CHECK's' but these are multiple proposals, so clean up the stats -#TODO: Move this as general output from the updater? +# TODO: counter_difference collects al the 'TO_CHECK's' but these are multiple proposals, so clean up the stats +# TODO: Move this as general output from the updater? for feature in fcs["result"]["features"]: if EVALUATION_FIELD_NAME in feature["properties"].keys(): ev = feature["properties"][EVALUATION_FIELD_NAME] @@ -104,7 +108,7 @@ elif ev.startswith("equal") and rd > 0: counter_equality_by_alignment = counter_equality_by_alignment + 1 elif ev.startswith("no_change"): - counter_no_change= counter_no_change + 1 + counter_no_change = counter_no_change + 1 else: counter_difference = counter_difference + 1 diff --git a/examples/example_parcel_vs_building.py b/examples/example_parcel_vs_building.py index 2003b75..ffc07c7 100644 --- a/examples/example_parcel_vs_building.py +++ b/examples/example_parcel_vs_building.py @@ -37,9 +37,13 @@ # Example how to use a series (for histogram) series = np.arange(0, 300, 10, dtype=int) / 100 - x_dict_series = aligner_x.process(relevant_distances=series, od_strategy=4, threshold_overlap_percentage=50) + x_dict_series = aligner_x.process( + relevant_distances=series, od_strategy=4, threshold_overlap_percentage=50 + ) x_resulting_areas = diffs_from_dict_series(x_dict_series, aligner_x.dict_thematic) - y_dict_series = aligner_y.process(relevant_distances=series, od_strategy=4, threshold_overlap_percentage=50) + y_dict_series = aligner_y.process( + relevant_distances=series, od_strategy=4, threshold_overlap_percentage=50 + ) y_resulting_areas = diffs_from_dict_series(y_dict_series, aligner_y.dict_thematic) # plot_diffs(series,x_resulting_areas) # plot_diffs(series,y_resulting_areas) diff --git a/pyproject.toml b/pyproject.toml index 057bcc2..11cbb85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "brdr" -version = "0.3.0" +version = "0.4.0" description = "BRDR - a Python library to assist in realigning (multi-)polygons (OGC Simple Features) to reference borders " readme = { file = "README.md", content-type = "text/markdown" } license = { file = "LICENSE" } diff --git a/tests/test_aligner.py b/tests/test_aligner.py index 7a48fa4..82e212b 100644 --- a/tests/test_aligner.py +++ b/tests/test_aligner.py @@ -272,7 +272,7 @@ def test_process_interior_ring(self): def test_process_circle(self): geometry = Point(0, 0).buffer(3) - #geometry = MultiPolygon([geometry]) + # geometry = MultiPolygon([geometry]) thematic_dict = {"key": geometry} self.sample_aligner.load_thematic_data(DictLoader(thematic_dict)) # LOAD REFERENCE DICTIONARY @@ -346,14 +346,12 @@ def test_fully_aligned_input(self): self.sample_aligner.load_reference_data(DictLoader({"ref_id_1": aligned_shape})) relevant_distance = 1 result = self.sample_aligner.process(relevant_distance=relevant_distance) - assert equals(result["theme_id_1"][relevant_distance].get("result"),aligned_shape) - assert result["theme_id_1"][relevant_distance].get("result_diff").is_empty - assert ( - result["theme_id_1"][relevant_distance].get("result_diff_min").is_empty - ) - assert ( - result["theme_id_1"][relevant_distance].get("result_diff_plus").is_empty + assert equals( + result["theme_id_1"][relevant_distance].get("result"), aligned_shape ) + assert result["theme_id_1"][relevant_distance].get("result_diff").is_empty + assert result["theme_id_1"][relevant_distance].get("result_diff_min").is_empty + assert result["theme_id_1"][relevant_distance].get("result_diff_plus").is_empty def test_evaluate(self): thematic_dict = { @@ -380,8 +378,8 @@ def test_evaluate(self): "result" ] thematic_dict_formula[key] = { - FORMULA_FIELD_NAME: json.dumps(base_aligner.get_brdr_formula( - thematic_dict_result[key]) + FORMULA_FIELD_NAME: json.dumps( + base_aligner.get_brdr_formula(thematic_dict_result[key]) ) } print(key + ": " + thematic_dict_result[key].wkt) @@ -413,7 +411,7 @@ def test_evaluate(self): ) actual_aligner.relevant_distances = np.arange(0, 200, 10, dtype=int) / 100 dict_evaluated, prop_dictionary = actual_aligner.evaluate( - ids_to_evaluate=affected,base_formula_field=FORMULA_FIELD_NAME + ids_to_evaluate=affected, base_formula_field=FORMULA_FIELD_NAME ) fc = get_series_geojson_dict( @@ -427,15 +425,16 @@ def test_evaluate(self): def test_remark_for_poly_multipoly(self): shape = from_wkt( - "MultiPolygon(((48893.03662109375 214362.93756103515625, 48890.8258056640625 214368.482666015625, 48890.7159423828125 214368.44110107421875, 48887.6488037109375 214367.2845458984375, 48886.3800048828125 214368.68017578125, 48885.1068115234375 214370.08062744140625, 48884.3330078125 214369.782470703125, 48882.563720703125 214369.10064697265625, 48882.1116943359375 214370.1346435546875, 48878.5626220703125 214368.70196533203125, 48877.839111328125 214368.40997314453125, 48877.2352294921875 214369.79376220703125, 48876.7911376953125 214369.60687255859375, 48875.0850830078125 214373.62353515625, 48875.478759765625 214373.8182373046875, 48881.5286865234375 214376.81109619140625, 48885.10546875 214372.36151123046875, 48887.0050048828125 214370.08538818359375, 48888.4698486328125 214368.330078125, 48890.366943359375 214369.2685546875, 48901.0638427734375 214374.56024169921875, 48905.0159912109375 214369.61175537109375, 48904.472900390625 214367.53851318359375, 48893.03662109375 214362.93756103515625)))") - self.sample_aligner.load_thematic_data( - DictLoader({"theme_id_1": shape}) + "MultiPolygon(((48893.03662109375 214362.93756103515625, 48890.8258056640625 214368.482666015625, 48890.7159423828125 214368.44110107421875, 48887.6488037109375 214367.2845458984375, 48886.3800048828125 214368.68017578125, 48885.1068115234375 214370.08062744140625, 48884.3330078125 214369.782470703125, 48882.563720703125 214369.10064697265625, 48882.1116943359375 214370.1346435546875, 48878.5626220703125 214368.70196533203125, 48877.839111328125 214368.40997314453125, 48877.2352294921875 214369.79376220703125, 48876.7911376953125 214369.60687255859375, 48875.0850830078125 214373.62353515625, 48875.478759765625 214373.8182373046875, 48881.5286865234375 214376.81109619140625, 48885.10546875 214372.36151123046875, 48887.0050048828125 214370.08538818359375, 48888.4698486328125 214368.330078125, 48890.366943359375 214369.2685546875, 48901.0638427734375 214374.56024169921875, 48905.0159912109375 214369.61175537109375, 48904.472900390625 214367.53851318359375, 48893.03662109375 214362.93756103515625)))" + ) + self.sample_aligner.load_thematic_data(DictLoader({"theme_id_1": shape})) + self.sample_aligner.load_reference_data( + GRBActualLoader( + grb_type=GRBType.ADP, partition=1000, aligner=self.sample_aligner + ) ) - self.sample_aligner.load_reference_data(GRBActualLoader( grb_type=GRBType.ADP, partition=1000, aligner=self.sample_aligner - )) self.sample_aligner.process(relevant_distances=[2]) - assert self.sample_aligner.dict_processresults["theme_id_1"][2]["remark"]!="" - + assert self.sample_aligner.dict_processresults["theme_id_1"][2]["remark"] != "" def test_fully_aligned_geojson_output(self): aligned_shape = from_wkt( From e20e4e6715aac40bc2a3954fee2da0ed06a34407 Mon Sep 17 00:00:00 2001 From: dieuska Date: Thu, 3 Oct 2024 16:57:17 +0200 Subject: [PATCH 25/26] black formatted --- brdr/loader.py | 4 +++- examples/example_grbspecificloader.py | 5 ++--- examples/example_predictor.py | 12 +++++++----- tests/test_grb.py | 14 ++++++++------ tests/test_integration.py | 6 +++++- 5 files changed, 25 insertions(+), 16 deletions(-) diff --git a/brdr/loader.py b/brdr/loader.py index 29a78a5..dc00c22 100644 --- a/brdr/loader.py +++ b/brdr/loader.py @@ -36,7 +36,9 @@ def load_data(self): ], DATE_FORMAT, ) - self.data_dict_properties[key][VERSION_DATE] = datetime.strftime(date,DATE_FORMAT) + self.data_dict_properties[key][VERSION_DATE] = datetime.strftime( + date, DATE_FORMAT + ) return self.data_dict, self.data_dict_properties, self.data_dict_source diff --git a/examples/example_grbspecificloader.py b/examples/example_grbspecificloader.py index 4cf151e..1fb7cbb 100644 --- a/examples/example_grbspecificloader.py +++ b/examples/example_grbspecificloader.py @@ -15,8 +15,8 @@ "Polygon ((172283.76869662097305991 174272.85233648214489222, 172276.89871930953813717 174278.68436246179044247, 172274.71383684969623573 174280.57171753142029047, 172274.63047763772192411 174280.64478165470063686, 172272.45265833073062822 174282.52660570573061705, 172269.33533191855531186 174285.22093996312469244, 172265.55258252174826339 174288.49089696351438761, 172258.77032718938426115 174294.22654021997004747, 172258.63259260458289646 174294.342757155187428, 172254.93673790179309435 174288.79932878911495209, 172248.71360730109154247 174279.61860501393675804, 172248.96566232520854101 174279.43056782521307468, 172255.25363882273086347 174274.73737183399498463, 172257.08298882702365518 174273.37133203260600567, 172259.32325354730710387 174271.69890458136796951, 172261.65807284769834951 174269.9690355472266674, 172266.35596220899606124 174266.4871726930141449, 172273.34350050613284111 174261.30863015633076429, 172289.60360219911672175 174249.35944479051977396, 172293.30328181147342548 174246.59864199347794056, 172297.34760522318538278 174253.10583685990422964, 172289.53060952731175348 174259.6846851697191596, 172292.86485871637705714 174265.19099397677928209, 172283.76869662097305991 174272.85233648214489222))" ) } - #EXAMPLE to use a GRBSpecificLoader, that retrieves the parcels for a specific data, based on the 2 fiscal parcel-situations of the year before and after - #Based on the date, the referencelayer will be different + # EXAMPLE to use a GRBSpecificLoader, that retrieves the parcels for a specific data, based on the 2 fiscal parcel-situations of the year before and after + # Based on the date, the referencelayer will be different date = "2023-05-03" date = "2023-08-03" loader = DictLoader(thematic_dict) @@ -31,4 +31,3 @@ inputtype=AlignerInputType.REFERENCE, ) write_geojson(os.path.join("output/", "grb_adp_" + date + ".geojson"), fc) - diff --git a/examples/example_predictor.py b/examples/example_predictor.py index 29127c4..276cc04 100644 --- a/examples/example_predictor.py +++ b/examples/example_predictor.py @@ -24,22 +24,24 @@ loader = GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=aligner) aligner.load_reference_data(loader) - #PREDICT the 'stable' relevant distances, for a series of relevant distances + # PREDICT the 'stable' relevant distances, for a series of relevant distances series = np.arange(0, 300, 10, dtype=int) / 100 # predict which relevant distances are interesting to propose as resulting geometry dict_series, dict_predictions, diffs = aligner.predictor( - relevant_distances=series, od_strategy=OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE, threshold_overlap_percentage=50 + relevant_distances=series, + od_strategy=OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE, + threshold_overlap_percentage=50, ) - #SHOW results of the predictions + # SHOW results of the predictions fcs = aligner.get_results_as_geojson( resulttype=AlignerResultType.PREDICTIONS, formula=False ) print(fcs["result"]) for key in dict_predictions: - plot_series(series, {key:diffs[key]}) + plot_series(series, {key: diffs[key]}) show_map( - {key:dict_predictions[key]}, + {key: dict_predictions[key]}, {key: aligner.dict_thematic[key]}, aligner.dict_reference, ) diff --git a/tests/test_grb.py b/tests/test_grb.py index 0998d48..98e911d 100644 --- a/tests/test_grb.py +++ b/tests/test_grb.py @@ -101,7 +101,7 @@ def test_get_geoms_affected_by_grb_change(self): } aligner = Aligner() aligner.load_thematic_data(DictLoader(thematic_dict)) - affected,unaffected = get_affected_by_grb_change( + affected, unaffected = get_affected_by_grb_change( dict_thematic=thematic_dict, grb_type=GRBType.ADP, date_start=date.today() - timedelta(days=1), @@ -110,7 +110,7 @@ def test_get_geoms_affected_by_grb_change(self): ) assert len(affected) == 0 - affected,unaffected = get_affected_by_grb_change( + affected, unaffected = get_affected_by_grb_change( dict_thematic=thematic_dict, grb_type=GRBType.ADP, date_start=date.today() - timedelta(days=1000), @@ -130,7 +130,7 @@ def test_get_geoms_affected_by_grb_change(self): } aligner2 = Aligner() aligner2.load_thematic_data(DictLoader(thematic_dict2)) - affected,unaffected = get_affected_by_grb_change( + affected, unaffected = get_affected_by_grb_change( dict_thematic=thematic_dict2, grb_type=GRBType.ADP, date_start=date.today() - timedelta(days=1000), @@ -152,7 +152,7 @@ def test_get_geoms_affected_by_grb_change_bulk(self): } aligner = Aligner() aligner.load_thematic_data(DictLoader(thematic_dict)) - affected,unaffected = get_affected_by_grb_change( + affected, unaffected = get_affected_by_grb_change( dict_thematic=thematic_dict, grb_type=GRBType.ADP, date_start=date.today() - timedelta(days=1), @@ -161,7 +161,7 @@ def test_get_geoms_affected_by_grb_change_bulk(self): ) assert len(affected) == 0 - affected,unaffected = get_affected_by_grb_change( + affected, unaffected = get_affected_by_grb_change( dict_thematic=thematic_dict, grb_type=GRBType.ADP, date_start=date.today() - timedelta(days=1000), @@ -233,7 +233,9 @@ def test_update_to_actual_grb(self): # Update Featurecollection to actual version featurecollection = update_to_actual_grb( - featurecollection_base_result, name_thematic_id,base_formula_field="brdr_base_formula" + featurecollection_base_result, + name_thematic_id, + base_formula_field="brdr_base_formula", ) # Print results for feature in featurecollection["result"]["features"]: diff --git a/tests/test_integration.py b/tests/test_integration.py index 1da1a90..c5489c0 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -55,7 +55,11 @@ def test_webservice_brdr(self): series = np.arange(0, 61, 1, dtype=float) / 10 - dict_series = aligner.process(relevant_distances=series, od_strategy=openbaardomein_strategy, threshold_overlap_percentage=50) + dict_series = aligner.process( + relevant_distances=series, + od_strategy=openbaardomein_strategy, + threshold_overlap_percentage=50, + ) dict_diffs = diffs_from_dict_series( dict_series, aligner.dict_thematic, DiffMetric.CHANGES_AREA ) From 41a228191a20d13e6b877d23e15db880a74730f3 Mon Sep 17 00:00:00 2001 From: dieuska Date: Thu, 3 Oct 2024 16:58:43 +0200 Subject: [PATCH 26/26] black_formatted --- tests/test_utils.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index bc185cc..0e7e175 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -8,6 +8,7 @@ from brdr.oe import get_oe_dict_by_ids from brdr.typings import ProcessResult from brdr.utils import diffs_from_dict_series + # from brdr.utils import filter_dict_by_key from brdr.utils import get_breakpoints_zerostreak from brdr.utils import get_collection @@ -41,26 +42,26 @@ def test_get_breakpoints_zerostreak_no_zerostreaks(self): def test_multipolygons_to_singles_empty_dict(self): data = {} - result,dict_multi_as_single = multipolygons_to_singles(data) + result, dict_multi_as_single = multipolygons_to_singles(data) self.assertEqual(result, {}) def test_multipolygons_to_singles_with_point(self): geometry1 = shapely.geometry.Polygon([(0, 0), (1, 0), (1, 1), (0, 1)]) geometry2 = shapely.geometry.Point(0, 0) data = {"test_id1": geometry1, "test_id2": geometry2} - result,dict_multi_as_single = multipolygons_to_singles(data) + result, dict_multi_as_single = multipolygons_to_singles(data) self.assertEqual(result, {"test_id1": geometry1}) def test_multipolygons_to_singles_single_polygon(self): geometry = shapely.geometry.Polygon([(0, 0), (1, 0), (1, 1), (0, 1)]) data = {"test_id": geometry} - result,dict_multi_as_single = multipolygons_to_singles(data) + result, dict_multi_as_single = multipolygons_to_singles(data) self.assertEqual(result, data) def test_multipolygons_to_singles_multipolygon_single_poly(self): geometry = shapely.geometry.Polygon([(0, 0), (1, 0), (1, 1), (0, 1)]) data = {"test_id": shapely.geometry.MultiPolygon([geometry])} - result,dict_multi_as_single = multipolygons_to_singles(data) + result, dict_multi_as_single = multipolygons_to_singles(data) self.assertEqual(result, {"test_id": geometry}) def test_polygonize_reference_data_no_overlap(self): @@ -168,7 +169,7 @@ def test_merge_process_results(self): key_1 = "key" + MULTI_SINGLE_ID_SEPARATOR + "1" key_2 = "key" + MULTI_SINGLE_ID_SEPARATOR + "2" key_3 = "key_3" - dict_multi_as_single = {key_1:"key",key_2:"key"} + dict_multi_as_single = {key_1: "key", key_2: "key"} process_result_1 = ProcessResult() process_result_1["result"] = Polygon([(0, 0), (10, 0), (10, 10), (0, 10)]) process_result_2 = ProcessResult() @@ -180,5 +181,7 @@ def test_merge_process_results(self): key_2: {0: process_result_2}, key_3: {0: process_result_3}, } - merged_testdict = merge_process_results(result_dict=testdict,dict_multi_as_single = dict_multi_as_single) + merged_testdict = merge_process_results( + result_dict=testdict, dict_multi_as_single=dict_multi_as_single + ) assert len(merged_testdict.keys()) == 2