diff --git a/CHANGES.md b/CHANGES.md index 7248ff5..027f7c4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -40,7 +40,24 @@ - Bugfixing: - adding a safe_equals-function to catch GEOsException bug [#71] +# 0.4.0 +! Not Backwards compatable ! + +- Refactoring: + - Possibility for parallel processing [#97] + - Changed Aligner constants to init-settings [#83] + - Refactored ID-handling so strings,integers,... can be used as ID[#110] + - Cleaned examples [#100] + + - Functionalities: + - Added evaluation-attributes to evaluate()-function [#99] + - processing-remarks available in geojson-output [#103] + - Added warning when input/output changed from polygon/multipolygon [#107] + +- Bugfixing: + - Bugfix on version_date [#96] + - Bugfix on disappearing features [#105] diff --git a/brdr/__init__.py b/brdr/__init__.py index 493f741..6a9beea 100644 --- a/brdr/__init__.py +++ b/brdr/__init__.py @@ -1 +1 @@ -__version__ = "0.3.0" +__version__ = "0.4.0" diff --git a/brdr/aligner.py b/brdr/aligner.py index 5ff3cfe..43c3536 100644 --- a/brdr/aligner.py +++ b/brdr/aligner.py @@ -2,6 +2,8 @@ import logging import os from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import wait from datetime import datetime from math import pi from typing import Iterable @@ -14,30 +16,30 @@ from shapely import make_valid from shapely import remove_repeated_points from shapely import to_geojson -from shapely import unary_union from shapely.geometry.base import BaseGeometry from brdr import __version__ +from brdr.constants import DEFAULT_CRS from brdr.constants import ( - BUFFER_MULTIPLICATION_FACTOR, LAST_VERSION_DATE, VERSION_DATE, DATE_FORMAT, - THRESHOLD_EXCLUSION_PERCENTAGE, - THRESHOLD_EXCLUSION_AREA, FORMULA_FIELD_NAME, EVALUATION_FIELD_NAME, + FULL_BASE_FIELD_NAME, + FULL_ACTUAL_FIELD_NAME, + DIFF_PERCENTAGE_FIELD_NAME, + DIFF_AREA_FIELD_NAME, + OD_ALIKE_FIELD_NAME, + EQUAL_REFERENCE_FEATURES_FIELD_NAME, ) -from brdr.constants import CORR_DISTANCE -from brdr.constants import DEFAULT_CRS -from brdr.constants import THRESHOLD_CIRCLE_RATIO from brdr.enums import ( OpenbaarDomeinStrategy, Evaluation, AlignerResultType, AlignerInputType, ) -from brdr.geometry_utils import buffer_neg +from brdr.geometry_utils import buffer_neg, safe_unary_union from brdr.geometry_utils import buffer_neg_pos from brdr.geometry_utils import buffer_pos from brdr.geometry_utils import fill_and_remove_gaps @@ -61,11 +63,11 @@ class Aligner: """ - This class is used to compare the thematic data with the reference data. + This class is used to compare and align the thematic data with the reference data. The reference data can be loaded in different ways, for example by using the GRB data. - The thematic data can be loaded by using a geojson file. - The class can be used to compare the thematic data with the reference data. + The thematic data can be loaded by using different Loaders: DictLoader, GeojsonLoader,... + The class can be used to compare and aligne the thematic data with the reference data. """ def __init__( @@ -75,9 +77,17 @@ def __init__( relevant_distance=1, relevant_distances=np.arange(0, 200, 10, dtype=int) / 100, threshold_overlap_percentage=50, - od_strategy=OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE, + od_strategy=OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE, crs=DEFAULT_CRS, + multi_as_single_modus=True, + threshold_exclusion_area=0, + threshold_exclusion_percentage=0, + buffer_multiplication_factor=1.01, + threshold_circle_ratio=0.98, + correction_distance=0.01, + mitre_limit=10, area_limit=None, + max_workers=None, ): """ Initializes the Aligner object @@ -87,39 +97,73 @@ def __init__( feedback in QGIS. Defaults to None. relevant_distance (int, optional): The relevant distance (in meters) for processing. Defaults to 1. + relevant_distances ([],optional): Relevant distances (in meters) for + processing od_strategy (int, optional): The strategy to determine how to handle information outside the reference polygons (Openbaar Domein) - (default 1: SNAP_SINGLE_SIDE) + (default: SNAP_FULL_AREA_ALL_SIDE) threshold_overlap_percentage (int, optional): Threshold (%) to determine from which overlapping-percentage a reference-polygon has to be included when there aren't relevant intersections or relevant differences - (default 50%) - od_strategy (int, optional): Determines how the algorithm deals with parts - of the geometry that are not on the - reference (default 1: SNAP_SINGLE_SIDE) + (default 50%). + When setting this parameter to '-1' the original border for will be returned for cases where nor relevant intersections and relevant differences are found crs (str, optional): Coordinate Reference System (CRS) of the data. (default EPSG:31370) + multi_as_single_modus (boolean, optional): Modus to handle multipolygons (Default=True): + True: input-multipolygons will be split-up into single polygons and handled by the algorithm. After executing the algorithm, the results are merged together. + False: Multipolygons are directly processed by the algorithm + threshold_exclusion_percentage (int, optional): Percentage for excluding candidate reference-polygons when overlap(%) is smaller than the threshold(Default=0) + threshold_exclusion_area (int, optional):Area in m² for excluding candidate reference-polygons when overlap(m²) is smaller than the threshold (Default=0) + buffer_multiplication_factor (float, optional): Multiplication factor, used to buffer the thematic objects searching for reference borders (buffer= buffer_multiplication_factor*relevant_distance)(Default=1.01) + threshold_circle_ratio (float, optional): Threshold-value to exclude circles getting processed (perfect circle = 1) based on POLSPY-POPPER algorithm(Default=0.98) + correction_distance (float, optional): Distance used in a pos_neg_buffer to remove slivers (technical correction) (Default= 0.01 = 1cm ) + mitre_limit (int, optional):buffer-parameter - The mitre ratio is the ratio of the distance from the corner to the end of the mitred offset corner. + When two line segments meet at a sharp angle, a miter join will extend far beyond the original geometry. (and in the extreme case will be infinitely far.) To prevent unreasonable geometry, the mitre limit allows controlling the maximum length of the join corner. + Corners with a ratio which exceed the limit will be beveled(Default=10) area_limit (int, optional): Maximum area for processing. (default 100000) - + max_workers (int, optional): Amount of workers that is used in ThreadPoolExecutor (for parallel execution) when processing objects for multiple relevant distances. (default None). If set to -1, no parallel exececution is used. """ self.logger = Logger(feedback) - self.relevant_distance = relevant_distance + if relevant_distances is None and relevant_distance is not None: + relevant_distances = [relevant_distance] self.relevant_distances = relevant_distances self.od_strategy = od_strategy self.threshold_overlap_percentage = threshold_overlap_percentage + # Area in m² for excluding candidate reference-polygons when overlap(m²) is smaller than the + # threshold + self.threshold_exclusion_area = threshold_exclusion_area + # Percentage for excluding candidate reference-polygons when overlap(%) is smaller than the + # threshold + self.threshold_exclusion_percentage = threshold_exclusion_percentage self.area_limit = area_limit + self.max_workers = max_workers + # Multiplication-factor used in OD-strategy 2 (SNAP-BOTH SIDED) when calculating + # OD-area to take into account + self.buffer_multiplication_factor = buffer_multiplication_factor + # Threshold-value to exclude circles getting processed (perfect circle = 1) based on + # POLSPY-POPPER algorithm + self.threshold_circle_ratio = threshold_circle_ratio + # Distance used in a pos_neg_buffer to remove slivers (technical correction) + self.correction_distance = correction_distance + # Buffer parameters: + # Distance to limit a buffered corner (MITER-join-style parameter) + # Explanation and examples: + # https://shapely.readthedocs.io/en/stable/reference/shapely.buffer.html + # https://postgis.net/docs/ST_Buffer.html + self.mitre_limit = mitre_limit + # quad_segments = 8 (by default in shapely) # PROCESSING DEFAULTS # thematic # name of the identifier-field of the thematic data (id has to be unique) self.name_thematic_id = "theme_identifier" # dictionary to store all thematic geometries to handle - self.dict_thematic: dict[str, BaseGeometry] = {} + self.dict_thematic: dict[any, BaseGeometry] = {} # dictionary to store properties of the reference-features (optional) - self.dict_thematic_properties: dict[str, dict] = {} + self.dict_thematic_properties: dict[any, dict] = {} # Dict to store source-information of the thematic dictionary - self.dict_thematic_source: dict[str, str] = {} + self.dict_thematic_source: dict[any, str] = {} # dictionary to store all unioned thematic geometries self.thematic_union = None @@ -129,11 +173,11 @@ def __init__( # CAPAKEY for GRB-parcels) self.name_reference_id = "ref_identifier" # dictionary to store all reference geometries - self.dict_reference: dict[str, BaseGeometry] = {} + self.dict_reference: dict[any, BaseGeometry] = {} # dictionary to store properties of the reference-features (optional) - self.dict_reference_properties: dict[str, dict] = {} + self.dict_reference_properties: dict[any, dict] = {} # Dict to store source-information of the reference dictionary - self.dict_reference_source: dict[str, str] = {} + self.dict_reference_source: dict[any, str] = {} # to save a unioned geometry of all reference polygons; needed for calculation # in most OD-strategies self.reference_union = None @@ -141,9 +185,13 @@ def __init__( # results # output-dictionaries (all results of process()), grouped by theme_id and relevant_distance - self.dict_processresults: dict[str, dict[float, ProcessResult]] = {} + self.dict_processresults: dict[any, dict[float, ProcessResult]] = {} # dictionary with the 'predicted' results, grouped by theme_id and relevant_distance - self.dict_predictions: dict[str, dict[float, ProcessResult]] = {} + self.dict_predictions: dict[any, dict[float, ProcessResult]] = {} + # dictionary with the 'evaluated predicted' results, grouped by theme_id and relevant_distance + self.dict_evaluated_predictions: dict[any, dict[float, ProcessResult]] = {} + # dictionary with the 'evaluated predicted' properties, grouped by theme_id and relevant_distance + self.dict_evaluated_predictions_properties: dict[any, dict[float, {}]] = {} # Coordinate reference system # thematic geometries and reference geometries are assumed to be in the same CRS @@ -154,21 +202,28 @@ def __init__( self.CRS = crs # this parameter is used to treat multipolygon as single polygons. So polygons # with ID splitter are separately evaluated and merged on result. - self.multi_as_single_modus = True + self.multi_as_single_modus = multi_as_single_modus self.logger.feedback_info("Aligner initialized") - def buffer_distance(self): - return self.relevant_distance / 2 - ##########LOADERS########################## ########################################### def load_thematic_data(self, loader: Loader): + """ + Loads the thematic features into the aligner + :param loader: + :return: + """ self.dict_thematic, self.dict_thematic_properties, self.dict_thematic_source = ( loader.load_data() ) def load_reference_data(self, loader: Loader): + """ + Loads the reference features into the aligner, and prepares the reference-data for processing + :param loader: + :return: + """ ( self.dict_reference, self.dict_reference_properties, @@ -183,7 +238,7 @@ def process_geometry( self, input_geometry: BaseGeometry, relevant_distance: float = 1, - od_strategy=OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE, + od_strategy=OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE, threshold_overlap_percentage=50, ) -> ProcessResult: """ @@ -191,10 +246,16 @@ def process_geometry( Args: input_geometry (BaseGeometry): The input geometric object. - relevant_distance - od_strategy - threshold_overlap_percentage (int): The buffer distance (positive - or negative). + relevant_distance: The relevant distance (in meters) for processing + od_strategy (int, optional): The strategy to determine how to handle + information outside the reference polygons (Openbaar Domein) + (default: SNAP_FULL_AREA_ALL_SIDE) + threshold_overlap_percentage (int, optional): Threshold (%) to determine + from which overlapping-percentage a reference-polygon has to be included + when there aren't relevant intersections or relevant differences + (default 50%). + When setting this parameter to '-1' the original border for will be returned for cases where nor relevant intersections and relevant differences are found + Returns: ProcessResult : A dict containing the resulting geometries: @@ -207,21 +268,18 @@ def process_geometry( geometry * relevant_intersection (BaseGeometry): The relevant_intersection * relevant_difference (BaseGeometry): The relevant_difference - Notes: - - - Example: + * remark (str): remarks collected when processing the geoemetry """ if self.area_limit and input_geometry.area > self.area_limit: message = "The input geometry is too large to process." raise ValueError(message) self.logger.feedback_debug("process geometry") - self.relevant_distance = relevant_distance self.od_strategy = od_strategy self.threshold_overlap_percentage = threshold_overlap_percentage - + buffer_distance = relevant_distance / 2 # combine all parts of the input geometry to one polygon - input_geometry = unary_union(get_parts(input_geometry)) + input_geometry = safe_unary_union(get_parts(input_geometry)) # array with all relevant parts of a thematic geometry; initial empty Polygon ( @@ -229,7 +287,9 @@ def process_geometry( preresult, relevant_intersection_array, relevant_diff_array, - ) = self._calculate_intersection_between_geometry_and_od(input_geometry) + ) = self._calculate_intersection_between_geometry_and_od( + input_geometry, relevant_distance + ) # get a list of all ref_ids that are intersecting the thematic geometry ref_intersections = self.reference_items.take( self.reference_tree.query(geometry) @@ -248,8 +308,11 @@ def process_geometry( geom_intersection, geom_reference, False, - self.relevant_distance / 2, + buffer_distance, self.threshold_overlap_percentage, + self.threshold_exclusion_percentage, + self.threshold_exclusion_area, + self.mitre_limit, ) self.logger.feedback_debug("intersection calculated") preresult = self._add_multi_polygons_from_geom_to_array(geom, preresult) @@ -260,15 +323,17 @@ def process_geometry( relevant_diff, relevant_diff_array ) # UNION INTERMEDIATE LAYERS - relevant_intersection = unary_union(relevant_intersection_array) + relevant_intersection = safe_unary_union(relevant_intersection_array) if relevant_intersection is None or relevant_intersection.is_empty: relevant_intersection = Polygon() - relevant_diff = unary_union(relevant_diff_array) + relevant_diff = safe_unary_union(relevant_diff_array) if relevant_diff is None or relevant_diff.is_empty: relevant_diff = Polygon() # POSTPROCESSING - result_dict = self._postprocess_preresult(preresult, geometry) + result_dict = self._postprocess_preresult( + preresult, geometry, relevant_distance + ) result_dict["result_relevant_intersection"] = relevant_intersection result_dict["result_relevant_diff"] = relevant_diff @@ -276,19 +341,20 @@ def process_geometry( # make a unary union for each key value in the result dict for key in ProcessResult.__annotations__: geometry = result_dict.get(key, Polygon()) # noqa - if not geometry.is_empty: - geometry = unary_union(geometry) + if isinstance(geometry, BaseGeometry) and not geometry.is_empty: + geometry = safe_unary_union(geometry) result_dict[key] = geometry # noqa return result_dict def process( self, + dict_thematic=None, relevant_distances: Iterable[float] = None, relevant_distance=1, - od_strategy=OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE, + od_strategy=OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE, threshold_overlap_percentage=50, - ) -> dict[str, dict[float, ProcessResult]]: + ) -> dict[any, dict[float, ProcessResult]]: """ Calculates the resulting dictionaries for thematic data based on a series of relevant distances. @@ -296,10 +362,15 @@ def process( Args: relevant_distances (Iterable[float]): A series of relevant distances (in meters) to process - od_strategy (int, optional): The strategy for overlap detection. - Defaults to 1. - threshold_overlap_percentage (float, optional): The threshold percentage for - considering full overlap. Defaults to 50. + od_strategy (int, optional): The strategy to determine how to handle + information outside the reference polygons (Openbaar Domein) + (default: SNAP_FULL_AREA_ALL_SIDE) + threshold_overlap_percentage (int, optional): Threshold (%) to determine + from which overlapping-percentage a reference-polygon has to be included + when there aren't relevant intersections or relevant differences + (default 50%). + When setting this parameter to '-1' the original border for will be returned for cases where nor relevant intersections and relevant differences are found + Returns: dict: A dictionary, for every thematic ID a dictionary with the results for all distances @@ -314,38 +385,96 @@ def process( """ if relevant_distances is None: relevant_distances = [relevant_distance] - self.relevant_distance = relevant_distance self.relevant_distances = relevant_distances self.od_strategy = od_strategy self.threshold_overlap_percentage = threshold_overlap_percentage self.logger.feedback_debug("Process series" + str(self.relevant_distances)) dict_series = {} - dict_thematic = self.dict_thematic - + dict_series_queue = {} + futures = [] + if dict_thematic is None: + dict_thematic = self.dict_thematic + dict_multi_as_single = {} if self.multi_as_single_modus: - dict_thematic = multipolygons_to_singles(dict_thematic) - - for key, geometry in dict_thematic.items(): - self.logger.feedback_info( - f"thematic id {str(key)} processed with relevant distances (m) [{str(self.relevant_distances)}]" + dict_thematic, dict_multi_as_single = multipolygons_to_singles( + dict_thematic ) - dict_series[key] = {} - for relevant_distance in self.relevant_distances: - try: - self.relevant_distance = relevant_distance - processed_result = self.process_geometry( - geometry, - self.relevant_distance, - od_strategy, - threshold_overlap_percentage, + + if self.max_workers != -1: + with ThreadPoolExecutor( + max_workers=self.max_workers + ) as executor: # max_workers=5 + for key, geometry in dict_thematic.items(): + self.logger.feedback_info( + f"thematic id {str(key)} processed with relevant distances (m) [{str(self.relevant_distances)}]" ) - except ValueError as e: - self.logger.feedback_warning(str(e)) + dict_series[key] = {} + dict_series_queue[key] = {} + for relevant_distance in self.relevant_distances: + try: + future = executor.submit( + self.process_geometry, + geometry, + relevant_distance, + od_strategy, + threshold_overlap_percentage, + ) + futures.append(future) + dict_series_queue[key][relevant_distance] = future + except ValueError as e: + self.logger.feedback_warning( + "error for" + + f"thematic id {str(key)} processed with relevant distances (m) [{str(self.relevant_distances)}]" + ) + dict_series_queue[key][relevant_distance] = None + self.logger.feedback_warning(str(e)) + self.logger.feedback_debug("waiting all started RD calculations") + wait(futures) + for id_theme, dict_dist in dict_series_queue.items(): + for relevant_distance, future in dict_dist.items(): + dict_series[id_theme][relevant_distance] = future.result() + else: + for key, geometry in dict_thematic.items(): + self.logger.feedback_info( + f"thematic id {str(key)} processed with relevant distances (m) [{str(self.relevant_distances)}]" + ) + dict_series[key] = {} + for relevant_distance in self.relevant_distances: + try: + processed_result = self.process_geometry( + geometry, + relevant_distance, + od_strategy, + threshold_overlap_percentage, + ) + except ValueError as e: + self.logger.feedback_warning(str(e)) + processed_result = None - dict_series[key][self.relevant_distance] = processed_result + dict_series[key][relevant_distance] = processed_result if self.multi_as_single_modus: - dict_series = merge_process_results(dict_series) + dict_series = merge_process_results(dict_series, dict_multi_as_single) + + # Check if geom changes from polygon to multipolygon or vice versa + for theme_id, dict_dist_results in dict_series.items(): + original_geometry = self.dict_thematic[theme_id] + original_geometry_length = -1 + if original_geometry.geom_type == "Polygon": + original_geometry_length = 1 + elif original_geometry.geom_type == "MultiPolygon": + original_geometry_length = len(original_geometry.geoms) + for relevant_distance, process_result in dict_dist_results.items(): + resulting_geom = process_result["result"] + resulting_geometry_length = -1 + if resulting_geom.geom_type == "Polygon": + resulting_geometry_length = 1 + elif resulting_geom.geom_type == "MultiPolygon": + resulting_geometry_length = len(resulting_geom.geoms) + if original_geometry_length != resulting_geometry_length: + msg = "Difference in amount of polygons" + self.logger.feedback_debug(msg) + process_result["remark"] = process_result["remark"] + " | " + msg self.logger.feedback_info( "End of processing series: " + str(self.relevant_distances) @@ -354,46 +483,11 @@ def process( return self.dict_processresults - # def process( - # self, - # relevant_distance=1, - # od_strategy=OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE, - # threshold_overlap_percentage=50, - # ) -> dict[str, dict[float, ProcessResult]]: - # """ - # Aligns a thematic dictionary of geometries to the reference layer based on - # specified parameters. - method to align a thematic dictionary to the reference - # layer - # - # Args: - # relevant_distance (float, optional): The relevant distance (in meters) for - # processing. Defaults to 1. - # od_strategy (int, optional): The strategy for overlap detection. - # Defaults to 1. - # threshold_overlap_percentage (float, optional): The threshold percentage for - # considering full overlap. Defaults to 50. - # - # Returns: - # dict: A dict containing processed data for each thematic key: - # - result: Aligned thematic data. - # - result_diff: global differences between thematic data and reference - # data. - # - result_diff_plus: Positive differences. - # - result_diff_min: Negative differences. - # - relevant_intersection: relevant intersections. - # - relevant_diff: relevant differences. - # - # """ - # self.relevant_distance=relevant_distance - # self.dict_result = self.process(relevant_distances=[self.relevant_distance], - # od_strategy=od_strategy, - # threshold_overlap_percentage=threshold_overlap_percentage) - # return self.dict_result - def predictor( self, + dict_thematic=None, relevant_distances=np.arange(0, 300, 10, dtype=int) / 100, - od_strategy=OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE, + od_strategy=OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE, threshold_overlap_percentage=50, ): """ @@ -435,11 +529,24 @@ def predictor( element based on the element key (using `filter_resulting_series_by_key`). Args: + + relevant_distances (np.ndarray, optional): A series of relevant distances + (in meters) to process. : A NumPy array of distances to + be analyzed. + od_strategy (int, optional): The strategy to determine how to handle + information outside the reference polygons (Openbaar Domein) + (default: SNAP_FULL_AREA_ALL_SIDE) + threshold_overlap_percentage (int, optional): Threshold (%) to determine + from which overlapping-percentage a reference-polygon has to be included + when there aren't relevant intersections or relevant differences + (default 50%). + When setting this parameter to '-1' the original border for will be returned for cases where nor relevant intersections and relevant differences are found + relevant_distances (np.ndarray, optional): A NumPy array of distances to be analyzed. Defaults to np.arange(0.1, 5.05, 0.1). od_strategy (OpenbaarDomeinStrategy, optional): A strategy for handling open data in the processing (implementation specific). Defaults to - OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE. + OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE. threshold_overlap_percentage (int, optional): A percentage threshold for considering full overlap in the processing (implementation specific). Defaults to 50. @@ -455,17 +562,16 @@ def predictor( - Values: dicts containing results (likely specific to your implementation) from the distance series for the corresponding distance. - - Logs: - - Debug logs the thematic element key being processed. """ + + if dict_thematic is None: + dict_thematic = self.dict_thematic dict_predictions = defaultdict(dict) dict_series = self.process( relevant_distances=relevant_distances, od_strategy=od_strategy, threshold_overlap_percentage=threshold_overlap_percentage, ) - dict_thematic = self.dict_thematic diffs_dict = diffs_from_dict_series(dict_series, dict_thematic) @@ -497,7 +603,10 @@ def predictor( for rel_dist, processresults in dist_results.items(): predicted_geom = processresults["result"] if not _equal_geom_in_array( - predicted_geom, predicted_geoms_for_theme_id + predicted_geom, + predicted_geoms_for_theme_id, + self.correction_distance, + self.mitre_limit, ): dict_predictions_unique[theme_id][rel_dist] = processresults predicted_geoms_for_theme_id.append(processresults["result"]) @@ -514,100 +623,61 @@ def predictor( diffs_dict, ) - def compare( - self, - threshold_area=5, - threshold_percentage=1, - dict_unchanged=None, - ): + def evaluate(self, ids_to_evaluate=None, base_formula_field=FORMULA_FIELD_NAME): """ - Compares input-geometries (with formula) and evaluates these geometries: An attribute is added to evaluate and decide if new + + Compares and evaluate input-geometries (with formula). Attributes are added to evaluate and decide if new proposals can be used + affected: list with all IDs to evaluate. all other IDs will be unchanged. If None (default), all self.dict_thematic will be evaluated. """ - dict_series, dict_predictions, diffs = self.predictor(self.relevant_distances) - if dict_unchanged is None: - dict_unchanged = {} - theme_ids = list(dict_series.keys()) - dict_evaluated_result = {} + if ids_to_evaluate is None: + ids_to_evaluate = list(self.dict_thematic.keys()) + dict_affected = {} + dict_unaffected = {} + for id_theme, geom in self.dict_thematic.items(): + if id_theme in ids_to_evaluate: + dict_affected[id_theme] = geom + else: + dict_unaffected[id_theme] = geom + self.dict_thematic = dict_affected + # AFFECTED + dict_series, dict_affected_predictions, diffs = self.predictor( + dict_thematic=dict_affected, relevant_distances=self.relevant_distances + ) + dict_predictions_evaluated = {} prop_dictionary = {} - # Fill the dictionary-structure with empty values - for theme_id in theme_ids: - dict_evaluated_result[theme_id] = {} + for theme_id, dict_predictions_results in dict_affected_predictions.items(): + dict_predictions_evaluated[theme_id] = {} prop_dictionary[theme_id] = {} - for dist in dict_series[theme_id].keys(): + for dist in sorted(dict_predictions_results.keys()): prop_dictionary[theme_id][dist] = {} - for theme_id in dict_unchanged.keys(): - prop_dictionary[theme_id] = {} - - for theme_id, dict_results in dict_predictions.items(): - equality = False - for dist in sorted(dict_results.keys()): - if equality: - break - geomresult = dict_results[dist]["result"] - actual_formula = self.get_brdr_formula(geomresult) - prop_dictionary[theme_id][dist][FORMULA_FIELD_NAME] = json.dumps( - actual_formula + props = self._evaluate( + id_theme=theme_id, + geom_predicted=dict_predictions_results[dist]["result"], + base_formula_field=base_formula_field, ) - base_formula = None - if ( - theme_id in self.dict_thematic_properties - and FORMULA_FIELD_NAME in self.dict_thematic_properties[theme_id] - ): - base_formula = self.dict_thematic_properties[theme_id][ - FORMULA_FIELD_NAME - ] - equality, prop = _check_equality( - base_formula, - actual_formula, - threshold_area, - threshold_percentage, - ) - if equality: - dict_evaluated_result[theme_id][dist] = dict_predictions[theme_id][ - dist - ] - prop_dictionary[theme_id][dist][EVALUATION_FIELD_NAME] = prop - break - - evaluated_theme_ids = [ - theme_id for theme_id, value in dict_evaluated_result.items() if value != {} - ] - - # fill where no equality is found/ The biggest predicted distance is returned as - # proposal - for theme_id in theme_ids: - if theme_id not in evaluated_theme_ids: - if len(dict_predictions[theme_id].keys()) == 0: - result = dict_series[theme_id][0] - dict_evaluated_result[theme_id][0] = result - prop_dictionary[theme_id][0][FORMULA_FIELD_NAME] = json.dumps( - self.get_brdr_formula(result["result"]) - ) - prop_dictionary[theme_id][0][ - EVALUATION_FIELD_NAME - ] = Evaluation.NO_PREDICTION_5 - continue - # Add all predicted features so they can be manually checked - for dist in dict_predictions[theme_id].keys(): - predicted_resultset = dict_predictions[theme_id][dist] - dict_evaluated_result[theme_id][dist] = predicted_resultset - prop_dictionary[theme_id][dist][FORMULA_FIELD_NAME] = json.dumps( - self.get_brdr_formula(predicted_resultset["result"]) - ) - prop_dictionary[theme_id][dist][ - EVALUATION_FIELD_NAME - ] = Evaluation.TO_CHECK_4 - - for theme_id, geom in dict_unchanged.items(): - prop_dictionary[theme_id] = { - 0: { - "result": geom, - EVALUATION_FIELD_NAME: Evaluation.NO_CHANGE_6, - FORMULA_FIELD_NAME: json.dumps(self.get_brdr_formula(geom)), - } - } - return dict_evaluated_result, prop_dictionary + dict_predictions_evaluated[theme_id][dist] = dict_affected_predictions[ + theme_id + ][dist] + prop_dictionary[theme_id][dist] = props + # UNAFFECTED + relevant_distance = 0 + # dict_unaffected_series = self.process(dict_thematic=dict_unaffected,relevant_distances=[relevant_distance]) + # for theme_id, dict_unaffected_results in dict_unaffected_series.items(): + for theme_id, geom in dict_unaffected.items(): + dict_predictions_evaluated[theme_id] = {} + prop_dictionary[theme_id] = {relevant_distance: {}} + props = self._evaluate( + id_theme=theme_id, + geom_predicted=geom, + base_formula_field=base_formula_field, + ) + props[EVALUATION_FIELD_NAME] = Evaluation.NO_CHANGE_6 + dict_predictions_evaluated[theme_id][relevant_distance] = {"result": geom} + prop_dictionary[theme_id][relevant_distance] = props + self.dict_evaluated_predictions = dict_predictions_evaluated + self.dict_evaluated_predictions_properties = prop_dictionary + return dict_predictions_evaluated, prop_dictionary def get_brdr_formula(self, geometry: BaseGeometry, with_geom=False): """ @@ -621,19 +691,28 @@ def get_brdr_formula(self, geometry: BaseGeometry, with_geom=False): Returns: dict: A dictionary containing formula-related data: - - 'full': True if the intersection is the same as the reference - geometry, else False. - - 'area': Area of the intersection or reference geometry. - - 'percentage': Percentage of intersection area relative to the - reference geometry. - - 'geometry': GeoJSON representation of the intersection (if - with_geom is True). + - "alignment_date": datetime.now().strftime(DATE_FORMAT), + - "brdr_version": str(__version__), + - "reference_source": self.dict_reference_source, + - "full": True if the geometry exists out of all full reference-polygons, else False. + - "area": Area of the geometry. + - "reference_features": { + array of all the reference features the geometry is composed of: + - 'full': True if the intersection is the same as the reference + geometry, else False. + - 'area': Area of the intersection or reference geometry. + - 'percentage': Percentage of intersection area relative to the + reference geometry. + - 'geometry': GeoJSON representation of the intersection (if + with_geom is True).}, + - "reference_od": Discription of the OD-part of the geometry (= not covered by reference-features), """ dict_formula = { "alignment_date": datetime.now().strftime(DATE_FORMAT), "brdr_version": str(__version__), "reference_source": self.dict_reference_source, "full": True, + "area": round(geometry.area, 2), "reference_features": {}, "reference_od": None, } @@ -662,7 +741,8 @@ def get_brdr_formula(self, geometry: BaseGeometry, with_geom=False): key_ref in self.dict_reference_properties and VERSION_DATE in self.dict_reference_properties[key_ref] ): - version_date = self.dict_reference_properties[key_ref][VERSION_DATE] + str_version_date = self.dict_reference_properties[key_ref][VERSION_DATE] + version_date = datetime.strptime(str_version_date, DATE_FORMAT) if last_version_date is None and version_date is not None: last_version_date = version_date if version_date is not None and version_date > last_version_date: @@ -700,10 +780,12 @@ def get_brdr_formula(self, geometry: BaseGeometry, with_geom=False): dict_formula[LAST_VERSION_DATE] = last_version_date.strftime(DATE_FORMAT) geom_od = buffer_pos( buffer_neg( - safe_difference(geometry, make_valid(unary_union(intersected))), - CORR_DISTANCE, + safe_difference(geometry, safe_unary_union(intersected)), + self.correction_distance, + mitre_limit=self.mitre_limit, ), - CORR_DISTANCE, + self.correction_distance, + mitre_limit=self.mitre_limit, ) if geom_od is not None: area_od = round(geom_od.area, 2) @@ -718,17 +800,25 @@ def get_brdr_formula(self, geometry: BaseGeometry, with_geom=False): ########################################### def get_results_as_geojson( - self, resulttype=AlignerResultType.PROCESSRESULTS, formula=False + self, + resulttype=AlignerResultType.PROCESSRESULTS, + formula=False, + attributes=False, ): """ - get a geojson of a dictionary containing the resulting geometries for all - 'serial' relevant distances. If no dict_series is given, the dict_result returned. - Optional: The descriptive formula is added as an attribute to the result""" - + get a geojson of a dictionary containing the resulting geometries for all + 'serial' relevant distances. The resulttype can be chosen. + formula (boolean, Optional): The descriptive formula is added as an attribute to the result + attributes (boolean, Optional): The original attributes/properties are added to the result + """ + prop_dictionary = None if resulttype == AlignerResultType.PROCESSRESULTS: dict_series = self.dict_processresults elif resulttype == AlignerResultType.PREDICTIONS: dict_series = self.dict_predictions + elif resulttype == AlignerResultType.EVALUATED_PREDICTIONS: + dict_series = self.dict_evaluated_predictions + prop_dictionary = self.dict_evaluated_predictions_properties else: raise (ValueError, "AlignerResultType unknown") if dict_series is None or dict_series == {}: @@ -736,18 +826,24 @@ def get_results_as_geojson( "Empty results: No calculated results to export." ) return {} - - prop_dictionary = defaultdict(dict) + if prop_dictionary is None: + prop_dictionary = defaultdict(dict) for theme_id, results_dict in dict_series.items(): for relevant_distance, process_results in results_dict.items(): - if formula: + if relevant_distance not in prop_dictionary[theme_id]: + prop_dictionary[theme_id][relevant_distance] = {} + if attributes: + for attr, value in self.dict_thematic_properties[theme_id].items(): + prop_dictionary[theme_id][relevant_distance][attr] = value + if ( + formula + ): # and not (theme_id in prop_dictionary and relevant_distance in prop_dictionary[theme_id] and NEW_FORMULA_FIELD_NAME in prop_dictionary[theme_id][relevant_distance]): result = process_results["result"] formula = self.get_brdr_formula(result) - prop_dictionary[theme_id][relevant_distance] = { - FORMULA_FIELD_NAME: json.dumps(formula) - } - + prop_dictionary[theme_id][relevant_distance][FORMULA_FIELD_NAME] = ( + json.dumps(formula) + ) return get_series_geojson_dict( dict_series, crs=self.CRS, @@ -757,7 +853,7 @@ def get_results_as_geojson( def get_input_as_geojson(self, inputtype=AlignerInputType.REFERENCE): """ - get a geojson of the reference polygons + get a geojson of the input polygons (thematic or reference-polygons) """ if inputtype == AlignerInputType.THEMATIC: @@ -770,7 +866,6 @@ def get_input_as_geojson(self, inputtype=AlignerInputType.REFERENCE): property_id = self.name_reference_id else: raise (ValueError, "AlignerInputType unknown") - dict_properties if dict_to_geojson is None or dict_to_geojson == {}: self.logger.feedback_warning("Empty input: No input to export.") return {} @@ -786,7 +881,7 @@ def save_results( self, path, resulttype=AlignerResultType.PROCESSRESULTS, formula=True ): """ - Exports analysis results as GeoJSON files. + Exports analysis results (as geojson) to path. This function exports 6 GeoJSON files containing the analysis results to the specified `path`. @@ -821,18 +916,18 @@ def save_results( ) def get_thematic_union(self): + """ + returns a unary_unioned geometry from all the thematic geometries + :return: + """ if self.thematic_union is None: - self.thematic_union = make_valid( - unary_union(list(self.dict_thematic.values())) - ) + self.thematic_union = safe_unary_union(list(self.dict_thematic.values())) return self.thematic_union def _prepare_reference_data(self): """ Prepares reference data for spatial queries and analysis. - It performs the following tasks: - 1. **Optimizes spatial queries:** - Creates a Spatial Relationship Tree (STRtree) using `STRtree` for efficient spatial queries against the reference data in @@ -858,8 +953,17 @@ def _prepare_reference_data(self): self.reference_union = None return - def _calculate_intersection_between_geometry_and_od(self, geometry): + def _calculate_intersection_between_geometry_and_od( + self, geometry, relevant_distance + ): + """ + Calculates the intersecting parts between a thematic geometry and the openbaardomein( domain, not coverd by reference-polygons) + :param geometry: + :param relevant_distance: + :return: + """ # Calculate the intersection between thematic and Openbaar Domein + buffer_distance = relevant_distance / 2 relevant_intersection_array = [] relevant_difference_array = [] geom_thematic_od = Polygon() @@ -886,11 +990,17 @@ def _calculate_intersection_between_geometry_and_od(self, geometry): # geom of OD geom_od = safe_difference(geometry, self._get_reference_union()) # only the relevant parts of OD - geom_od_neg_pos = buffer_neg_pos(geom_od, self.buffer_distance()) + geom_od_neg_pos = buffer_neg_pos( + geom_od, + buffer_distance, + mitre_limit=self.mitre_limit, + ) # geom_thematic_od = safe_intersection(geom_od_neg_pos,geom_od)# resulting # thematic OD geom_od_neg_pos_buffered = buffer_pos( - geom_od_neg_pos, self.buffer_distance() + geom_od_neg_pos, + buffer_distance, + mitre_limit=self.mitre_limit, ) # include parts geom_thematic_od = safe_intersection( geom_od_neg_pos_buffered, geom_od @@ -904,7 +1014,7 @@ def _calculate_intersection_between_geometry_and_od(self, geometry): geom_thematic_od, relevant_difference_array, relevant_intersection_array, - ) = self._od_snap_all_side(geometry) + ) = self._od_snap_all_side(geometry, relevant_distance) elif self.od_strategy == OpenbaarDomeinStrategy.SNAP_FULL_AREA_SINGLE_SIDE: # Strategy useful for bigger areas. # integrates the entire inner area of the input geometry, @@ -913,7 +1023,7 @@ def _calculate_intersection_between_geometry_and_od(self, geometry): self.logger.feedback_debug( "OD-strategy Full-area-variant of OD-SNAP_SINGLE_SIDE" ) - geom_thematic_od = self._od_full_area(geometry) + geom_thematic_od = self._od_full_area(geometry, relevant_distance) elif self.od_strategy == OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE: # Strategy useful for bigger areas. # integrates the entire inner area of the input geometry, @@ -927,10 +1037,10 @@ def _calculate_intersection_between_geometry_and_od(self, geometry): geom_thematic_od, relevant_difference_array, relevant_intersection_array, - ) = self._od_snap_all_side(geometry) + ) = self._od_snap_all_side(geometry, relevant_distance) # This part is a copy of SNAP_FULL_AREA_SINGLE_SIDE geom_theme_od_min_clipped_plus_buffered_clipped = self._od_full_area( - geometry + geometry, relevant_distance ) # UNION the calculation of OD-SNAP_ALL_SIDE with FULL AREA of # OD-SNAP_FULL_AREA_SINGLE_SIDE @@ -948,7 +1058,11 @@ def _calculate_intersection_between_geometry_and_od(self, geometry): # geom of OD geom_od = safe_difference(geometry, self._get_reference_union()) # only the relevant parts of OD - geom_od_neg_pos = buffer_neg_pos(geom_od, self.buffer_distance()) + geom_od_neg_pos = buffer_neg_pos( + geom_od, + buffer_distance, + mitre_limit=self.mitre_limit, + ) # resulting thematic OD geom_thematic_od = safe_intersection(geom_od_neg_pos, geom_od) elif self.od_strategy == OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE_VARIANT_2: @@ -970,20 +1084,29 @@ def _calculate_intersection_between_geometry_and_od(self, geometry): relevant_difference_array, ) - def _od_full_area(self, geometry): + def _od_full_area(self, geometry, relevant_distance): + buffer_distance = relevant_distance / 2 geom_theme_od = safe_difference(geometry, self._get_reference_union()) geom_theme_min_buffered = buffer_neg( buffer_pos( - buffer_neg(geometry, self.relevant_distance), - self.buffer_distance(), + buffer_neg( + geometry, + relevant_distance, + mitre_limit=self.mitre_limit, + ), + buffer_distance, + mitre_limit=self.mitre_limit, ), - self.buffer_distance(), + buffer_distance, + mitre_limit=self.mitre_limit, ) geom_theme_od_clipped_min_buffered = safe_intersection( geom_theme_min_buffered, geom_theme_od ) geom_theme_od_min_clipped_plus_buffered = buffer_pos( - geom_theme_od_clipped_min_buffered, self.relevant_distance + geom_theme_od_clipped_min_buffered, + relevant_distance, + mitre_limit=self.mitre_limit, ) geom_theme_od_min_clipped_plus_buffered_clipped = safe_intersection( geom_theme_od_min_clipped_plus_buffered, geom_theme_od @@ -991,11 +1114,16 @@ def _od_full_area(self, geometry): geom_thematic_od = geom_theme_od_min_clipped_plus_buffered_clipped return geom_thematic_od - def _od_snap_all_side(self, geometry): + def _od_snap_all_side(self, geometry, relevant_distance): + buffer_distance = relevant_distance / 2 relevant_difference_array = [] relevant_intersection_array = [] geom_thematic_buffered = make_valid( - buffer_pos(geometry, BUFFER_MULTIPLICATION_FACTOR * self.relevant_distance) + buffer_pos( + geometry, + self.buffer_multiplication_factor * relevant_distance, + mitre_limit=self.mitre_limit, + ) ) clip_ref_thematic_buffered = safe_intersection( self._get_reference_union(), geom_thematic_buffered @@ -1016,8 +1144,11 @@ def _od_snap_all_side(self, geometry): geom_intersection, geom_reference, True, - self.relevant_distance / 2, + buffer_distance, self.threshold_overlap_percentage, + self.threshold_exclusion_percentage, + self.threshold_exclusion_area, + self.mitre_limit, ) relevant_intersection_array = self._add_multi_polygons_from_geom_to_array( geom_relevant_intersection, [] @@ -1028,13 +1159,17 @@ def _od_snap_all_side(self, geometry): return geom_thematic_od, relevant_difference_array, relevant_intersection_array def _get_reference_union(self): + """ + returns a unary_unioned geometry from all the referene geometries + :return: + """ if self.reference_union is None: - self.reference_union = make_valid( - unary_union(list(self.dict_reference.values())) - ) + self.reference_union = safe_unary_union(list(self.dict_reference.values())) return self.reference_union - def _postprocess_preresult(self, preresult, geom_thematic) -> ProcessResult: + def _postprocess_preresult( + self, preresult, geom_thematic, relevant_distance + ) -> ProcessResult: """ Postprocess the preresult with the following actions to create the final result *Corrections for areas that differ more than the relevant distance @@ -1057,10 +1192,13 @@ def _postprocess_preresult(self, preresult, geom_thematic) -> ProcessResult: output geometry * result_diff_min (BaseGeometry): The resulting negative difference output geometry + * remark (str): Remark when processing the geometry """ # Process array + remark = "" + buffer_distance = relevant_distance / 2 result = [] - geom_preresult = make_valid(unary_union(preresult)) + geom_preresult = safe_unary_union(preresult) geom_thematic = make_valid(geom_thematic) if not (geom_thematic is None or geom_thematic.is_empty): @@ -1069,24 +1207,31 @@ def _postprocess_preresult(self, preresult, geom_thematic) -> ProcessResult: # if a circle: (Polsby-popper score) if ( 4 * pi * (geom_thematic.area / (geom_thematic.length**2)) - > THRESHOLD_CIRCLE_RATIO + > self.threshold_circle_ratio ): - self.logger.feedback_debug( - "Circle: -->resulting geometry = original geometry" - ) - return {"result": geom_thematic} + remark = "Circle detected: -->resulting geometry = original geometry" + self.logger.feedback_debug(remark) + return {"result": geom_thematic, "remark": remark} # Correction for unchanged geometries if geom_preresult == geom_thematic: - return {"result": geom_thematic} + remark = "Unchanged geometry: -->resulting geometry = original geometry" + self.logger.feedback_debug(remark) + return {"result": geom_thematic, "remark": remark} # Corrections for areas that differ more than the relevant distance geom_thematic_dissolved = buffer_pos( buffer_neg( - buffer_pos(geom_preresult, CORR_DISTANCE), - 2 * CORR_DISTANCE, + buffer_pos( + geom_preresult, + self.correction_distance, + mitre_limit=self.mitre_limit, + ), + 2 * self.correction_distance, + mitre_limit=self.mitre_limit, ), - CORR_DISTANCE, + self.correction_distance, + mitre_limit=self.mitre_limit, ) # geom_symdiff = self._safe_symmetric_difference(geom_thematic, # geom_thematic_dissolved) @@ -1096,73 +1241,108 @@ def _postprocess_preresult(self, preresult, geom_thematic) -> ProcessResult: geom_thematic_dissolved, safe_intersection( geom_diff_delete, - buffer_neg_pos(geom_diff_delete, self.buffer_distance()), + buffer_neg_pos( + geom_diff_delete, + buffer_distance, + mitre_limit=self.mitre_limit, + ), ), ) geom_diff_removed_added = safe_union( geom_diff_removed, safe_intersection( geom_diff_add, - buffer_neg_pos(geom_diff_add, self.buffer_distance()), + buffer_neg_pos( + geom_diff_add, + buffer_distance, + mitre_limit=self.mitre_limit, + ), ), ) geom_thematic_preresult = buffer_pos( buffer_neg( - buffer_pos(geom_diff_removed_added, CORR_DISTANCE), - 2 * CORR_DISTANCE, + buffer_pos( + geom_diff_removed_added, + self.correction_distance, + mitre_limit=self.mitre_limit, + ), + 2 * self.correction_distance, + mitre_limit=self.mitre_limit, ), - CORR_DISTANCE, + self.correction_distance, + mitre_limit=self.mitre_limit, ) # Correction for Inner holes(donuts) / multipolygons # fill and remove gaps geom_thematic_cleaned_holes = fill_and_remove_gaps( - geom_thematic_preresult, self.buffer_distance() + geom_thematic_preresult, buffer_distance ) geom_thematic_result = buffer_pos( buffer_neg( - buffer_pos(geom_thematic_cleaned_holes, CORR_DISTANCE), - 2 * CORR_DISTANCE, + buffer_pos( + geom_thematic_cleaned_holes, + self.correction_distance, + mitre_limit=self.mitre_limit, + ), + 2 * self.correction_distance, + mitre_limit=self.mitre_limit, ), - CORR_DISTANCE, + self.correction_distance, + mitre_limit=self.mitre_limit, ) geom_thematic_result = make_valid(remove_repeated_points(geom_thematic_result)) # Correction for empty preresults if geom_thematic_result.is_empty or geom_thematic_result is None: - self.logger.feedback_warning( - "Empty result: -->resulting geometry = empty geometry" - ) - # geom_thematic_result = geom_thematic - geom_thematic_result = Polygon() + remark = "Calculated empty result: -->original geometry returned" + self.logger.feedback_warning(remark) + + geom_thematic_result = geom_thematic + # geom_thematic_result = Polygon() #If we return an empty geometry, the feature disappears, so we return the original geometry # group all initial multipolygons into a new resulting dictionary result.append(geom_thematic_result) # create all resulting geometries - geom_thematic_result = make_valid(unary_union(result)) + geom_thematic_result = safe_unary_union(result) # negative and positive buffer is added to the difference-calculations, to # remove 'very small' differences (smaller than the correction distance) geom_result_diff = buffer_pos( buffer_neg( - safe_symmetric_difference(geom_thematic_result, geom_thematic), - CORR_DISTANCE, + safe_symmetric_difference( + geom_thematic_result, + geom_thematic, + ), + self.correction_distance, + mitre_limit=self.mitre_limit, ), - CORR_DISTANCE, + self.correction_distance, + mitre_limit=self.mitre_limit, ) geom_result_diff_plus = buffer_pos( buffer_neg( - safe_difference(geom_thematic_result, geom_thematic), - CORR_DISTANCE, + safe_difference( + geom_thematic_result, + geom_thematic, + ), + self.correction_distance, + mitre_limit=self.mitre_limit, ), - CORR_DISTANCE, + self.correction_distance, + mitre_limit=self.mitre_limit, ) geom_result_diff_min = buffer_pos( buffer_neg( - safe_difference(geom_thematic, geom_thematic_result), - CORR_DISTANCE, + safe_difference( + geom_thematic, + geom_thematic_result, + ), + self.correction_distance, + mitre_limit=self.mitre_limit, ), - CORR_DISTANCE, + self.correction_distance, + mitre_limit=self.mitre_limit, ) # geom_result_diff_plus = safe_difference(geom_thematic_result, geom_thematic) # geom_result_diff_min = safe_difference(geom_thematic, geom_thematic_result) @@ -1172,7 +1352,126 @@ def _postprocess_preresult(self, preresult, geom_thematic) -> ProcessResult: "result_diff": geom_result_diff, "result_diff_plus": geom_result_diff_plus, "result_diff_min": geom_result_diff_min, + "remark": remark, + } + + def _evaluate( + self, id_theme, geom_predicted, base_formula_field=FORMULA_FIELD_NAME + ): + """ + function that evaluates a predicted geometry and returns a properties-dictionary + """ + threshold_od_percentage = 1 + properties = { + FORMULA_FIELD_NAME: "", + EVALUATION_FIELD_NAME: Evaluation.TO_CHECK_4, + FULL_BASE_FIELD_NAME: None, + FULL_ACTUAL_FIELD_NAME: None, + OD_ALIKE_FIELD_NAME: None, + EQUAL_REFERENCE_FEATURES_FIELD_NAME: None, + DIFF_PERCENTAGE_FIELD_NAME: None, + DIFF_AREA_FIELD_NAME: None, } + actual_formula = self.get_brdr_formula(geom_predicted) + properties[FORMULA_FIELD_NAME] = json.dumps(actual_formula) + base_formula = None + if ( + id_theme in self.dict_thematic_properties + and base_formula_field in self.dict_thematic_properties[id_theme] + ): + base_formula = json.loads( + self.dict_thematic_properties[id_theme][base_formula_field] + ) + + if base_formula is None or actual_formula is None: + properties[EVALUATION_FIELD_NAME] = Evaluation.NO_PREDICTION_5 + return properties + properties[FULL_BASE_FIELD_NAME] = base_formula["full"] + properties[FULL_ACTUAL_FIELD_NAME] = actual_formula["full"] + od_alike = False + if ( + base_formula["reference_od"] is None + and actual_formula["reference_od"] is None + ): + od_alike = True + elif ( + base_formula["reference_od"] is None + or actual_formula["reference_od"] is None + ): + od_alike = False + elif ( + abs( + base_formula["reference_od"]["area"] + - actual_formula["reference_od"]["area"] + ) + * 100 + / base_formula["reference_od"]["area"] + ) < threshold_od_percentage: + od_alike = True + properties[OD_ALIKE_FIELD_NAME] = od_alike + + equal_reference_features = True + if ( + base_formula["reference_features"].keys() + == actual_formula["reference_features"].keys() + ): + equal_reference_features = True + max_diff_area_reference_feature = 0 + max_diff_percentage_reference_feature = 0 + for key in base_formula["reference_features"].keys(): + if ( + base_formula["reference_features"][key]["full"] + != actual_formula["reference_features"][key]["full"] + ): + equal_reference_features = False + + diff_area_reference_feature = abs( + base_formula["reference_features"][key]["area"] + - actual_formula["reference_features"][key]["area"] + ) + diff_percentage_reference_feature = ( + abs( + base_formula["reference_features"][key]["area"] + - actual_formula["reference_features"][key]["area"] + ) + * 100 + / base_formula["reference_features"][key]["area"] + ) + if diff_area_reference_feature > max_diff_area_reference_feature: + max_diff_area_reference_feature = diff_area_reference_feature + if ( + diff_percentage_reference_feature + > max_diff_percentage_reference_feature + ): + max_diff_percentage_reference_feature = ( + diff_percentage_reference_feature + ) + properties[EQUAL_REFERENCE_FEATURES_FIELD_NAME] = equal_reference_features + properties[DIFF_AREA_FIELD_NAME] = max_diff_area_reference_feature + properties[DIFF_PERCENTAGE_FIELD_NAME] = ( + max_diff_percentage_reference_feature + ) + # EVALUATION + if ( + equal_reference_features + and od_alike + and base_formula["full"] + and actual_formula["full"] + ): + properties[EVALUATION_FIELD_NAME] = Evaluation.EQUALITY_EQUAL_FORMULA_FULL_1 + elif ( + equal_reference_features + and od_alike + and base_formula["full"] == actual_formula["full"] + ): + properties[EVALUATION_FIELD_NAME] = Evaluation.EQUALITY_EQUAL_FORMULA_2 + elif base_formula["full"] and actual_formula["full"] and od_alike: + properties[EVALUATION_FIELD_NAME] = Evaluation.EQUALITY_FULL_3 + # elif base_formula["full"] == actual_formula["full"] and od_alike:#TODO evaluate when not-full-parcels? + # properties[EVALUATION_FIELD_NAME] = Evaluation.EQUALITY_NON_FULL + else: + properties[EVALUATION_FIELD_NAME] = Evaluation.TO_CHECK_4 + return properties @staticmethod def _add_multi_polygons_from_geom_to_array(geom: BaseGeometry, array): @@ -1209,8 +1508,9 @@ def _calculate_geom_by_intersection_and_reference( is_openbaar_domein, buffer_distance, threshold_overlap_percentage, - threshold_exclusion_percentage=THRESHOLD_EXCLUSION_PERCENTAGE, - threshold_exclusion_area=THRESHOLD_EXCLUSION_AREA, + threshold_exclusion_percentage, + threshold_exclusion_area, + mitre_limit, ): """ Calculates the geometry based on intersection and reference geometries. @@ -1260,8 +1560,16 @@ def _calculate_geom_by_intersection_and_reference( return Polygon(), Polygon(), Polygon() geom_difference = safe_difference(geom_reference, geom_intersection) - geom_relevant_intersection = buffer_neg(geom_intersection, buffer_distance) - geom_relevant_difference = buffer_neg(geom_difference, buffer_distance) + geom_relevant_intersection = buffer_neg( + geom_intersection, + buffer_distance, + mitre_limit=mitre_limit, + ) + geom_relevant_difference = buffer_neg( + geom_difference, + buffer_distance, + mitre_limit=mitre_limit, + ) if ( not geom_relevant_intersection.is_empty and not geom_relevant_difference.is_empty @@ -1273,15 +1581,24 @@ def _calculate_geom_by_intersection_and_reference( geom_reference, safe_intersection( geom_difference, - buffer_neg_pos(geom_difference, buffer_distance), + buffer_neg_pos( + geom_difference, + buffer_distance, + mitre_limit=mitre_limit, + ), ), ), ) geom = safe_intersection( geom_x, buffer_pos( - buffer_neg_pos(geom_x, buffer_distance), + buffer_neg_pos( + geom_x, + buffer_distance, + mitre_limit=mitre_limit, + ), buffer_distance, + mitre_limit=mitre_limit, ), ) # when calculating for OD, we create a 'virtual parcel'. When calculating this @@ -1290,7 +1607,7 @@ def _calculate_geom_by_intersection_and_reference( # in the result. The function below tries to exclude these non-logical parts. # see eo_id 206363 with relevant distance=0.2m and SNAP_ALL_SIDE if is_openbaar_domein: - geom = _get_relevant_polygons_from_geom(geom, buffer_distance) + geom = _get_relevant_polygons_from_geom(geom, buffer_distance, mitre_limit) elif not geom_relevant_intersection.is_empty and geom_relevant_difference.is_empty: geom = geom_reference elif geom_relevant_intersection.is_empty and not geom_relevant_difference.is_empty: @@ -1310,7 +1627,9 @@ def _calculate_geom_by_intersection_and_reference( return geom, geom_relevant_intersection, geom_relevant_difference -def _get_relevant_polygons_from_geom(geometry: BaseGeometry, buffer_distance: float): +def _get_relevant_polygons_from_geom( + geometry: BaseGeometry, buffer_distance: float, mitre_limit +): """ Get only the relevant parts (polygon) from a geometry. Points, Lines and Polygons smaller than relevant distance are excluded from the @@ -1320,7 +1639,7 @@ def _get_relevant_polygons_from_geom(geometry: BaseGeometry, buffer_distance: fl # If the input geometry is empty or None, do nothing. return geometry else: - geometry = make_valid(unary_union(geometry)) + geometry = safe_unary_union(geometry) # Create a GeometryCollection from the input geometry. geometry_collection = GeometryCollection(geometry) array = [] @@ -1328,13 +1647,17 @@ def _get_relevant_polygons_from_geom(geometry: BaseGeometry, buffer_distance: fl # Ensure each sub-geometry is valid. g = make_valid(g) if str(g.geom_type) in ["Polygon", "MultiPolygon"]: - relevant_geom = buffer_neg(g, buffer_distance) + relevant_geom = buffer_neg( + g, + buffer_distance, + mitre_limit=mitre_limit, + ) if relevant_geom is not None and not relevant_geom.is_empty: array.append(g) - return make_valid(unary_union(array)) + return safe_unary_union(array) -def _equal_geom_in_array(geom, geom_array): +def _equal_geom_in_array(geom, geom_array, correction_distance, mitre_limit): """ Check if a predicted geometry is equal to other predicted geometries in a list. Equality is defined as there is the symmetrical difference is smaller than the CORRECTION DISTANCE @@ -1342,7 +1665,11 @@ def _equal_geom_in_array(geom, geom_array): """ for g in geom_array: # if safe_equals(geom,g): - if buffer_neg(safe_symmetric_difference(geom, g), CORR_DISTANCE).is_empty: + if buffer_neg( + safe_symmetric_difference(geom, g), + correction_distance, + mitre_limit=mitre_limit, + ).is_empty: return True return False @@ -1375,8 +1702,8 @@ def _check_equality( == actual_formula["reference_features"].keys() and od_alike ): - if base_formula["full"] and base_formula["full"]: - return True, Evaluation.EQUALITY_FORMULA_GEOM_1 + if base_formula["full"] and actual_formula["full"]: + return True, Evaluation.EQUALITY_EQUAL_FORMULA_FULL_1 equal_reference_features = True for key in base_formula["reference_features"].keys(): @@ -1406,7 +1733,7 @@ def _check_equality( ): equal_reference_features = False if equal_reference_features: - return True, Evaluation.EQUALITY_FORMULA_2 - if base_formula["full"] and base_formula["full"] and od_alike: - return True, Evaluation.EQUALITY_GEOM_3 + return True, Evaluation.EQUALITY_EQUAL_FORMULA_2 + if base_formula["full"] and actual_formula["full"] and od_alike: + return True, Evaluation.EQUALITY_FULL_3 return False, Evaluation.NO_PREDICTION_5 diff --git a/brdr/constants.py b/brdr/constants.py index f43e20f..4fbdfb1 100644 --- a/brdr/constants.py +++ b/brdr/constants.py @@ -1,68 +1,45 @@ -# Thresholds -# Area in m² for excluding candidate reference when overlap(m²) is smaller than the -# threshold -THRESHOLD_EXCLUSION_AREA = 0 -# Percentage for excluding candidate reference when overlap(%) is smaller than the -# threshold -THRESHOLD_EXCLUSION_PERCENTAGE = 0 - -# Buffer parameters: -# Explanation and examples: -# https://shapely.readthedocs.io/en/stable/reference/shapely.buffer.html -# https://postgis.net/docs/ST_Buffer.html -# Distance to limit a buffered corner (MITER-join-style parameter) -MITRE_LIMIT = 10 -# Used in buffer-operations to define a quarter circle -QUAD_SEGMENTS = 5 - -# Correction-parameters (technical) - -# Multiplication-factor used in OD-strategy 2 (SNAP-BOTH SIDED) when calculating -# OD-area to take into account -BUFFER_MULTIPLICATION_FACTOR = 1.01 -# Threshold-value to exclude circles getting processed (perfect circle = 1) based on -# POLSPY-POPPER algorithm -THRESHOLD_CIRCLE_RATIO = 0.98 -# Distance used in a pos_neg_buffer to remove slivers (technical correction) -CORR_DISTANCE = 0.01 - # Download-settings: when extracting features by URL -# max buffer around thematic geometry to download reference parcels -MAX_REFERENCE_BUFFER = 10 # Limit used when extracting features by URL, using the feature API (f.e. from GRB) DOWNLOAD_LIMIT = 10000 - # default CRS: DEFAULT_CRS = "EPSG:31370" # BelgianLambert72 # MULTI_SINGLE_ID_SEPARATOR #separator to split multipolygon_ids to single polygons MULTI_SINGLE_ID_SEPARATOR = "*$*" - -FORMULA_FIELD_NAME = "brdr_formula" -EVALUATION_FIELD_NAME = "brdr_evaluation" -NR_CALCULATION_FIELD_NAME = "brdr_nr_calculations" -RELEVANT_DISTANCE_FIELD_NAME = "brdr_relevant_distance" +PREFIX_FIELDNAME = "brdr_" +BASE_FORMULA_FIELD_NAME = ( + PREFIX_FIELDNAME + "base_formula" +) # for use in grb_actualisation +FORMULA_FIELD_NAME = PREFIX_FIELDNAME + "formula" +EVALUATION_FIELD_NAME = PREFIX_FIELDNAME + "evaluation" +DIFF_PERCENTAGE_FIELD_NAME = PREFIX_FIELDNAME + "diff_percentage" +DIFF_AREA_FIELD_NAME = PREFIX_FIELDNAME + "diff_area" +FULL_BASE_FIELD_NAME = PREFIX_FIELDNAME + "full_base" +FULL_ACTUAL_FIELD_NAME = PREFIX_FIELDNAME + "full_actual" +EQUAL_REFERENCE_FEATURES_FIELD_NAME = PREFIX_FIELDNAME + "equal_reference_features" +OD_ALIKE_FIELD_NAME = PREFIX_FIELDNAME + "od_alike" + +NR_CALCULATION_FIELD_NAME = PREFIX_FIELDNAME + "nr_calculations" +RELEVANT_DISTANCE_FIELD_NAME = PREFIX_FIELDNAME + "relevant_distance" +REMARK_FIELD_NAME = PREFIX_FIELDNAME + "remark" LAST_VERSION_DATE = "last_version_date" VERSION_DATE = "version_date" - DATE_FORMAT = "%Y-%m-%d" + # GRB_CONSTANTS +# max buffer (m) around thematic geometry to download reference parcels +GRB_MAX_REFERENCE_BUFFER = 10 # URL of the OGC feature API of actual GRB to extract collections GRB_FEATURE_URL = "https://geo.api.vlaanderen.be/GRB/ogc/features/collections" - # URL of the OGC feature API of GRB fiscal parcels (situation of 1st of January) to # extract collections GRB_FISCAL_PARCELS_URL = "https://geo.api.vlaanderen.be/Adpf/ogc/features/collections" - # Property-name of version_date GRB_VERSION_DATE = "VERSDATUM" - # Property-name of id of GRB-parcels GRB_PARCEL_ID = "CAPAKEY" - # Property-name of id of GRB-parcels GRB_BUILDING_ID = "OIDN" - # Property-name of id of GRB-parcels GRB_KNW_ID = "OIDN" diff --git a/brdr/enums.py b/brdr/enums.py index 030a622..d5802b4 100644 --- a/brdr/enums.py +++ b/brdr/enums.py @@ -45,6 +45,7 @@ class AlignerResultType(str, Enum): """ PREDICTIONS = "predictions" + EVALUATED_PREDICTIONS = "evaluated_predictions" PROCESSRESULTS = "processresults" @@ -95,17 +96,17 @@ class Evaluation(str, Enum): """ Enum to evaluate an automatically updated geometry: - * EQUALITY_FORMULA_GEOM_1 = "equality_formula_geom_1" - * EQUALITY_FORMULA_2 = "equality_formula_2" - * EQUALITY_GEOM_3 = "equality_geom_3" - * TO_CHECK_4 = "to_check_4" - * NO_PREDICTION_5 = "no_prediction_5" - * NO_CHANGE_6 = "no_change_6" + EQUALITY_EQUAL_FORMULA_FULL_1 = "equality_equal_formula_full_1" + EQUALITY_EQUAL_FORMULA_2 = "equality_equal_formula_2" + EQUALITY_FULL_3 = "equality_full_3" + TO_CHECK_4 = "to_check_4" + NO_PREDICTION_5 = "no_prediction_5" + NO_CHANGE_6 = "no_change_6" """ - EQUALITY_FORMULA_GEOM_1 = "equality_formula_geom_1" - EQUALITY_FORMULA_2 = "equality_formula_2" - EQUALITY_GEOM_3 = "equality_geom_3" + EQUALITY_EQUAL_FORMULA_FULL_1 = "equality_equal_formula_full_1" + EQUALITY_EQUAL_FORMULA_2 = "equality_equal_formula_2" + EQUALITY_FULL_3 = "equality_full_3" TO_CHECK_4 = "to_check_4" NO_PREDICTION_5 = "no_prediction_5" NO_CHANGE_6 = "no_change_6" diff --git a/brdr/geometry_utils.py b/brdr/geometry_utils.py index bca3996..89b1aaa 100644 --- a/brdr/geometry_utils.py +++ b/brdr/geometry_utils.py @@ -13,18 +13,17 @@ from shapely import get_parts from shapely import intersection from shapely import is_empty +from shapely import make_valid from shapely import polygons from shapely import symmetric_difference from shapely import to_wkt +from shapely import unary_union from shapely import union from shapely.geometry.base import BaseGeometry from shapely.prepared import prep -from brdr.constants import MITRE_LIMIT -from brdr.constants import QUAD_SEGMENTS - -def buffer_neg_pos(geometry, buffer_value): +def buffer_neg_pos(geometry, buffer_value, mitre_limit=5): """ Computes two buffers accordingly: one with a negative buffer value and another with a positive buffer value. This function can be used the check where relevant areas @@ -57,18 +56,18 @@ def buffer_neg_pos(geometry, buffer_value): buffer( geometry, -buffer_value, - quad_segs=QUAD_SEGMENTS, + # quad_segs=QUAD_SEGMENTS, join_style="mitre", - mitre_limit=MITRE_LIMIT, + mitre_limit=mitre_limit, ), buffer_value, - quad_segs=QUAD_SEGMENTS, + # quad_segs=QUAD_SEGMENTS, join_style="mitre", - mitre_limit=MITRE_LIMIT, + mitre_limit=mitre_limit, ) -def buffer_neg(geometry, buffer_value): +def buffer_neg(geometry, buffer_value, mitre_limit=5): """ Computes the negative buffer of a given geometric object. @@ -95,13 +94,13 @@ def buffer_neg(geometry, buffer_value): return buffer( geometry, -buffer_value, - quad_segs=QUAD_SEGMENTS, + # quad_segs=QUAD_SEGMENTS, join_style="mitre", - mitre_limit=MITRE_LIMIT, + mitre_limit=mitre_limit, ) -def buffer_pos(geometry, buffer_value): +def buffer_pos(geometry, buffer_value, mitre_limit=5): """ Computes the positive buffer of a given geometric object. @@ -128,9 +127,9 @@ def buffer_pos(geometry, buffer_value): return buffer( geometry, buffer_value, - quad_segs=QUAD_SEGMENTS, + # quad_segs=QUAD_SEGMENTS, join_style="mitre", - mitre_limit=MITRE_LIMIT, + mitre_limit=mitre_limit, ) @@ -508,6 +507,10 @@ def fill_and_remove_gaps(input_geometry, buffer_value): return cleaned_geometry +def safe_unary_union(geometries): + return make_valid(unary_union(geometries)) + + def get_bbox(geometry): """ Get the BBOX (string) of a shapely geometry diff --git a/brdr/grb.py b/brdr/grb.py index 1e846d1..56a34c1 100644 --- a/brdr/grb.py +++ b/brdr/grb.py @@ -5,9 +5,8 @@ from datetime import datetime import numpy as np -from shapely import intersects, Polygon +from shapely import intersects from shapely.geometry import shape -from shapely.geometry.base import BaseGeometry from brdr.aligner import Aligner from brdr.constants import ( @@ -16,17 +15,18 @@ DATE_FORMAT, VERSION_DATE, FORMULA_FIELD_NAME, + BASE_FORMULA_FIELD_NAME, ) from brdr.constants import DOWNLOAD_LIMIT from brdr.constants import GRB_BUILDING_ID from brdr.constants import GRB_FEATURE_URL from brdr.constants import GRB_FISCAL_PARCELS_URL from brdr.constants import GRB_KNW_ID +from brdr.constants import GRB_MAX_REFERENCE_BUFFER from brdr.constants import GRB_PARCEL_ID from brdr.constants import GRB_VERSION_DATE -from brdr.constants import MAX_REFERENCE_BUFFER -from brdr.enums import GRBType -from brdr.geometry_utils import buffer_pos, safe_intersection +from brdr.enums import GRBType, AlignerResultType +from brdr.geometry_utils import buffer_pos, safe_intersection, safe_unary_union from brdr.geometry_utils import create_donut from brdr.geometry_utils import features_by_geometric_operation from brdr.geometry_utils import get_bbox @@ -35,7 +35,6 @@ from brdr.utils import geojson_to_dicts from brdr.utils import get_collection from brdr.utils import get_collection_by_partition -from brdr.utils import get_series_geojson_dict log = logging.getLogger(__name__) @@ -78,20 +77,22 @@ def is_grb_changed( return False -def get_geoms_affected_by_grb_change( - aligner, +def get_affected_by_grb_change( + dict_thematic, grb_type=GRBType.ADP, date_start=date.today(), date_end=date.today(), one_by_one=False, border_distance=0, + geometry_thematic_union=None, + crs=DEFAULT_CRS, ): """ - Get a dictionary of thematic geometries that are affected bij GRB-changes in a + Get a list of affected and unaffected IDs by GRB-changes in a specific timespan Args: - aligner: Aligner instance + dict_thematic: dictionary if thematicID & Geometry grb_type: Type of GRB: parcels, buildings,... date_start: start-date to check changes in GRB date_end: end-date to check changes in GRB @@ -106,12 +107,9 @@ def get_geoms_affected_by_grb_change( dictionary of affected geometries """ - dict_thematic = aligner.dict_thematic - # if aligner.multi_as_single_modus: - # dict_thematic = merge_dict(dict_thematic) - crs = aligner.CRS - affected_dict: dict[str, BaseGeometry] = {} - unchanged_dict: dict[str, BaseGeometry] = {} + + affected = [] + unaffected = [] if border_distance > 0: for key in dict_thematic.keys(): dict_thematic[key] = create_donut(dict_thematic[key], border_distance) @@ -119,15 +117,16 @@ def get_geoms_affected_by_grb_change( for key in dict_thematic: geom = dict_thematic[key] if is_grb_changed(geom, grb_type, date_start, date_end): - affected_dict[key] = geom + affected.append(key) else: - unchanged_dict[key] = geom - return affected_dict, unchanged_dict + unaffected.append(key) + return affected, unaffected else: # Temporal filter on VERDATUM - geometry = aligner.get_thematic_union() + if geometry_thematic_union is None: + geometry_thematic_union = safe_unary_union(list(dict_thematic.values())) coll_changed_grb, name_reference_id = get_collection_grb_actual( - geometry, + geometry_thematic_union, grb_type=grb_type, partition=1000, date_start=date_start, @@ -140,7 +139,7 @@ def get_geoms_affected_by_grb_change( if len(dict_changed_grb) == 0: logging.info("No detected changes") - return affected_dict, dict_thematic # empty affected dict + return affected, list(dict_thematic.keys()) # empty affected dict logging.info("Changed parcels in timespan: " + str(len(dict_changed_grb))) thematic_intersections = features_by_geometric_operation( list(dict_thematic.values()), @@ -150,11 +149,12 @@ def get_geoms_affected_by_grb_change( ) logging.info("Number of filtered features: " + str(len(thematic_intersections))) for key, geom in dict_thematic.items(): - if key in thematic_intersections: - affected_dict[key] = geom - else: - unchanged_dict[key] = geom - return affected_dict, unchanged_dict + ( + affected.append(key) + if key in thematic_intersections + else unaffected.append(key) + ) + return affected, unaffected def get_last_version_date( @@ -345,13 +345,22 @@ def get_collection_grb_parcels_by_date( def update_to_actual_grb( featurecollection, id_theme_fieldname, - formula_field=FORMULA_FIELD_NAME, + base_formula_field=FORMULA_FIELD_NAME, max_distance_for_actualisation=2, feedback=None, + attributes=True, ): """ Function to update a thematic featurecollection to the most actual version of GRB. Important to notice that the featurecollection needs a 'formula' for the base-alignment. + + :param featurecollection: Thematic featurecollection + :param id_theme_fieldname: property-fieldname that states which property has to be used as unique ID + :param base_formula_field: Name of the property-field that holds the original/base formula of the geometry, that has to be compared with the actual formula. + :param max_distance_for_actualisation: Maximum relevant distance that is used to search and evaluate resulting geometries. All relevant distance between 0 and this max_distance are used to search, with a interval of 0.1m. + :param feedback: (default None): a QGIS feedback can be added to push all the logging to QGIS + :param attributes: (boolean, default=True): States of all original attributes has to be added to the result + :return: featurecollection """ logger = Logger(feedback) # Load featurecollection into a shapely_dict: @@ -361,32 +370,27 @@ def update_to_actual_grb( last_version_date = datetime.now().date() for feature in featurecollection["features"]: id_theme = feature["properties"][id_theme_fieldname] - try: - geom = shape(feature["geometry"]) - except Exception: - geom = Polygon() - logger.feedback_debug("id theme: " + id_theme) - logger.feedback_debug("geometry (wkt): " + geom.wkt) + geom = shape(feature["geometry"]) + # logger.feedback_debug("id theme: " + id_theme) + # logger.feedback_debug("geometry (wkt): " + geom.wkt) dict_thematic[id_theme] = geom + dict_thematic_props[id_theme] = feature["properties"] try: - dict_thematic_props[id_theme] = { - FORMULA_FIELD_NAME: json.loads(feature["properties"][formula_field]) - } - logger.feedback_debug("formula: " + str(dict_thematic_props[id_theme])) + base_formula_string = feature["properties"][base_formula_field] + dict_thematic_props[id_theme][BASE_FORMULA_FIELD_NAME] = base_formula_string + base_formula = json.loads(base_formula_string) + + logger.feedback_debug("formula: " + str(base_formula)) except Exception: raise Exception("Formula -attribute-field (json) cannot be loaded") try: logger.feedback_debug(str(dict_thematic_props[id_theme])) if ( - LAST_VERSION_DATE in dict_thematic_props[id_theme][FORMULA_FIELD_NAME] - and dict_thematic_props[id_theme][FORMULA_FIELD_NAME][LAST_VERSION_DATE] - is not None - and dict_thematic_props[id_theme][FORMULA_FIELD_NAME][LAST_VERSION_DATE] - != "" + LAST_VERSION_DATE in base_formula + and base_formula[LAST_VERSION_DATE] is not None + and base_formula[LAST_VERSION_DATE] != "" ): - str_lvd = dict_thematic_props[id_theme][FORMULA_FIELD_NAME][ - LAST_VERSION_DATE - ] + str_lvd = base_formula[LAST_VERSION_DATE] lvd = datetime.strptime(str_lvd, DATE_FORMAT).date() if lvd < last_version_date: last_version_date = lvd @@ -399,29 +403,30 @@ def update_to_actual_grb( base_aligner_result.load_thematic_data(DictLoader(dict_thematic)) base_aligner_result.name_thematic_id = id_theme_fieldname - dict_affected, dict_unchanged = get_geoms_affected_by_grb_change( - base_aligner_result, + affected, unaffected = get_affected_by_grb_change( + dict_thematic=base_aligner_result.dict_thematic, grb_type=GRBType.ADP, date_start=datetime_start, date_end=datetime_end, one_by_one=False, + geometry_thematic_union=base_aligner_result.get_thematic_union(), + crs=base_aligner_result.CRS, ) logger.feedback_info( - "Number of possible affected OE-thematic during timespan: " - + str(len(dict_affected)) + "Number of possible affected OE-thematic during timespan: " + str(len(affected)) ) - if len(dict_affected) == 0: + if len(affected) == 0: logger.feedback_info( "No change detected in referencelayer during timespan. Script is finished" ) return {} logger.feedback_debug(str(datetime_start)) - logger.feedback_debug(str(formula_field)) + logger.feedback_debug(str(base_formula_field)) # Initiate a Aligner to reference thematic features to the actual borders - actual_aligner = Aligner(feedback=feedback) + actual_aligner = Aligner(feedback=feedback, max_workers=None) actual_aligner.load_thematic_data( - DictLoader(data_dict=dict_affected, data_dict_properties=dict_thematic_props) + DictLoader(data_dict=dict_thematic, data_dict_properties=dict_thematic_props) ) actual_aligner.load_reference_data( GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner) @@ -430,15 +435,14 @@ def update_to_actual_grb( actual_aligner.relevant_distances = ( np.arange(0, max_distance_for_actualisation * 100, 10, dtype=int) / 100 ) - dict_evaluated, prop_dictionary = actual_aligner.compare( - threshold_area=5, threshold_percentage=1, dict_unchanged=dict_unchanged + dict_evaluated, prop_dictionary = actual_aligner.evaluate( + ids_to_evaluate=affected, base_formula_field=BASE_FORMULA_FIELD_NAME ) - return get_series_geojson_dict( - dict_evaluated, - crs=actual_aligner.CRS, - id_field=actual_aligner.name_thematic_id, - series_prop_dict=prop_dictionary, + return actual_aligner.get_results_as_geojson( + resulttype=AlignerResultType.EVALUATED_PREDICTIONS, + formula=True, + attributes=attributes, ) @@ -454,7 +458,9 @@ def __init__(self, grb_type: GRBType, aligner, partition: int = 1000): def load_data(self): if not self.aligner.dict_thematic: raise ValueError("Thematic data not loaded") - geom_union = buffer_pos(self.aligner.get_thematic_union(), MAX_REFERENCE_BUFFER) + geom_union = buffer_pos( + self.aligner.get_thematic_union(), GRB_MAX_REFERENCE_BUFFER + ) collection, id_property = get_collection_grb_actual( grb_type=self.grb_type, geometry=geom_union, @@ -483,7 +489,9 @@ def __init__(self, year: str, aligner, partition=1000): def load_data(self): if not self.aligner.dict_thematic: raise ValueError("Thematic data not loaded") - geom_union = buffer_pos(self.aligner.get_thematic_union(), MAX_REFERENCE_BUFFER) + geom_union = buffer_pos( + self.aligner.get_thematic_union(), GRB_MAX_REFERENCE_BUFFER + ) collection = get_collection_grb_fiscal_parcels( year=self.year, geometry=geom_union, @@ -497,7 +505,9 @@ def load_data(self): class GRBSpecificDateParcelLoader(GeoJsonLoader): def __init__(self, date, aligner, partition=1000): - logging.warning("experimental loader; use with care!!!") + logging.warning( + "Loader for GRB parcel-situation on specific date (experimental); Use it with care!!!" + ) try: date = datetime.strptime(date, DATE_FORMAT).date() if date.year >= datetime.now().year: @@ -519,7 +529,9 @@ def __init__(self, date, aligner, partition=1000): def load_data(self): if not self.aligner.dict_thematic: raise ValueError("Thematic data not loaded") - geom_union = buffer_pos(self.aligner.get_thematic_union(), MAX_REFERENCE_BUFFER) + geom_union = buffer_pos( + self.aligner.get_thematic_union(), GRB_MAX_REFERENCE_BUFFER + ) collection = get_collection_grb_parcels_by_date( date=self.date, geometry=geom_union, diff --git a/brdr/loader.py b/brdr/loader.py index 5f67a88..dc00c22 100644 --- a/brdr/loader.py +++ b/brdr/loader.py @@ -14,28 +14,31 @@ class Loader(ABC): def __init__(self): - self.data_dict: dict[str, BaseGeometry] = {} - self.data_dict_properties: dict[str, dict] = {} - self.data_dict_source: dict[str, str] = {} - self.versiondate_info: dict[str, str] = None + self.data_dict: dict[any, BaseGeometry] = {} + self.data_dict_properties: dict[any, dict] = {} + self.data_dict_source: dict[any, str] = {} + self.versiondate_info: dict[any, str] = None def load_data(self): self.data_dict = {x: make_valid(self.data_dict[x]) for x in self.data_dict} if self.versiondate_info is not None: for key in self.data_dict_properties.keys(): try: - self.data_dict_properties[key][VERSION_DATE] = datetime.strptime( + date = datetime.strptime( self.data_dict_properties[key][self.versiondate_info["name"]], self.versiondate_info["format"], ) except: # Catch, to try extracting only the date with default -date format if specific format does not work - self.data_dict_properties[key][VERSION_DATE] = datetime.strptime( + date = datetime.strptime( self.data_dict_properties[key][self.versiondate_info["name"]][ :10 ], DATE_FORMAT, ) + self.data_dict_properties[key][VERSION_DATE] = datetime.strftime( + date, DATE_FORMAT + ) return self.data_dict, self.data_dict_properties, self.data_dict_source diff --git a/brdr/typings.py b/brdr/typings.py index 22556f0..5ad2c05 100644 --- a/brdr/typings.py +++ b/brdr/typings.py @@ -37,3 +37,4 @@ class ProcessResult(TypedDict, total=False): result_diff_min: BaseGeometry result_relevant_intersection: BaseGeometry result_relevant_diff: BaseGeometry + remark: str diff --git a/brdr/utils.py b/brdr/utils.py index 05b47a7..1406e5c 100644 --- a/brdr/utils.py +++ b/brdr/utils.py @@ -3,14 +3,8 @@ import numpy as np import requests -from geojson import Feature -from geojson import FeatureCollection -from geojson import dump -from shapely import GeometryCollection -from shapely import make_valid -from shapely import node -from shapely import polygonize -from shapely import unary_union +from geojson import Feature, FeatureCollection, dump +from shapely import GeometryCollection, make_valid, node, polygonize, unary_union from shapely.geometry import shape from shapely.geometry.base import BaseGeometry @@ -20,6 +14,7 @@ DOWNLOAD_LIMIT, RELEVANT_DISTANCE_FIELD_NAME, NR_CALCULATION_FIELD_NAME, + REMARK_FIELD_NAME, ) from brdr.enums import DiffMetric from brdr.geometry_utils import get_partitions, get_bbox @@ -27,14 +22,24 @@ def get_series_geojson_dict( - series_dict: dict[str, dict[float, ProcessResult]], + series_dict: dict[any, dict[float, ProcessResult]], crs: str, id_field: str, - series_prop_dict: dict[str, dict[float, any]] = None, + series_prop_dict: dict[any, dict[float, any]] = None, geom_attributes=True, ): """ - Convert a series of process results to a GeoJSON feature collections. + Convert a series of process results to a GeoJSON feature collection. + + Args: + series_dict (dict): Dictionary containing process results. + crs (str): Coordinate reference system. + id_field (str): Field name for the ID. + series_prop_dict (dict, optional): Dictionary containing series properties. + geom_attributes (bool, optional): Whether to include geometry attributes. + + Returns: + dict: Dictionary of GeoJSON feature collections. """ features_list_dict = {} @@ -46,8 +51,12 @@ def get_series_geojson_dict( properties[id_field] = theme_id properties[NR_CALCULATION_FIELD_NAME] = nr_calculations properties[RELEVANT_DISTANCE_FIELD_NAME] = relative_distance + if "remark" in process_result: + properties[REMARK_FIELD_NAME] = process_result["remark"] for results_type, geom in process_result.items(): + if not isinstance(geom, BaseGeometry): + continue if results_type not in features_list_dict: features_list_dict[results_type] = [] @@ -71,13 +80,12 @@ def _feature_from_geom( Args: geom (BaseGeometry): The geometry to convert. - properties (dict): The properties to include in the feature. - geom_attributes (bool): Whether to include geometry attributes (default True). + properties (dict, optional): The properties to include in the feature. + geom_attributes (bool, optional): Whether to include geometry attributes. Returns: Feature: The GeoJSON feature. """ - properties = dict(properties or {}) if geom_attributes: area = geom.area @@ -90,8 +98,17 @@ def _feature_from_geom( def geojson_from_dict(dictionary, crs, id_field, prop_dict=None, geom_attributes=True): """ - get a geojson (featurecollection) from a dictionary of ids(keys) and geometries - (values) + Get a GeoJSON (FeatureCollection) from a dictionary of IDs (keys) and geometries (values). + + Args: + dictionary (dict): Dictionary of geometries. + crs (str): Coordinate reference system. + id_field (str): Field name for the ID. + prop_dict (dict, optional): Dictionary of properties. + geom_attributes (bool, optional): Whether to include geometry attributes. + + Returns: + FeatureCollection: The GeoJSON FeatureCollection. """ features = [] for key, geom in dictionary.items(): @@ -104,6 +121,13 @@ def geojson_from_dict(dictionary, crs, id_field, prop_dict=None, geom_attributes def write_geojson(path_to_file, geojson): + """ + Write a GeoJSON object to a file. + + Args: + path_to_file (str): Path to the output file. + geojson (FeatureCollection): The GeoJSON object to write. + """ parent = os.path.dirname(path_to_file) os.makedirs(parent, exist_ok=True) with open(path_to_file, "w") as f: @@ -112,36 +136,19 @@ def write_geojson(path_to_file, geojson): def multipolygons_to_singles(dict_geoms): """ - Converts a dictionary of shapely-geometries to a dictionary containing only single - polygons. - - This function iterates through a dictionary where values are Shapely-geometries - and performs the following: - - * **Polygons:** Preserves the key and geometry from the original dictionary. - * **MultiPolygons with one polygon:** Preserves the key and extracts the single - polygon. - * **MultiPolygons with multiple polygons:** - * Creates new keys for each polygon by appending a suffix (_index) to the - original key. - * Assigns the individual polygons from the MultiPolygon to the newly created - keys. + Convert a dictionary of Shapely geometries to a dictionary containing only single polygons. Args: - dict_geoms (dict): A dictionary where keys are identifiers and values are - GeoJSON geometries. + dict_geoms (dict): Dictionary of geometries. Returns: - dict: A new dictionary containing only single polygons (as Polygon geometries). - Keys are created based on the logic described above. - - Notes: - * Geometries that are not Polygons or MultiPolygons are excluded with a warning - message printed. + tuple: A tuple containing: + - dict: Dictionary of single polygons. + - dict: Dictionary mapping new keys to original keys. """ resulting_dict_geoms = {} - for key in dict_geoms: - geom = dict_geoms[key] + dict_multi_as_single = {} + for key, geom in dict_geoms.items(): if str(geom.geom_type) == "Polygon": resulting_dict_geoms[key] = geom elif str(geom.geom_type) == "MultiPolygon": @@ -152,41 +159,23 @@ def multipolygons_to_singles(dict_geoms): i = 0 for p in polygons: new_key = str(key) + MULTI_SINGLE_ID_SEPARATOR + str(i) + dict_multi_as_single[new_key] = key resulting_dict_geoms[new_key] = p i = i + 1 else: logging.debug("geom excluded: " + str(geom) + " for key: " + str(key)) - return resulting_dict_geoms + return resulting_dict_geoms, dict_multi_as_single def polygonize_reference_data(dict_ref): """ - Creates a new dictionary with non-overlapping polygons based on a reference data - dictionary. - - This function is designed to handle situations where the original reference data - dictionary might contain: - - * Overlapping polygons: It creates new, non-overlapping polygons by combining all - reference borders. - * Multiple overlapping references: This function is useful when combining references - like parcels and buildings that might overlap. - - **Important:** The original reference IDs are lost in the process of creating new - non-overlapping polygons. New unique keys are assigned instead. + Create a new dictionary with non-overlapping polygons based on a reference data dictionary. Args: - dict_ref (dict): A dictionary where keys are identifiers and values are Shapely - geometries (assumed to be Polygons or MultiPolygons). + dict_ref (dict): Dictionary of reference geometries. Returns: - dict: A new dictionary containing non-overlapping polygons derived from the - original reference data. - Keys are unique strings (reference IDs are lost). - - Notes: - * Geometries that are not Polygons or MultiPolygons are excluded with a warning - message printed. + dict: Dictionary of non-overlapping polygons. """ arr_ref = [] for key in dict_ref: @@ -207,27 +196,16 @@ def polygonize_reference_data(dict_ref): def get_breakpoints_zerostreak(x, y): """ - Determine the extremes and zero_streaks of a graph based on the derivative, and - return: - * the breakpoints: extremes (breakpoints) of graph where 'change' occurs - * the zero_streaks: ranges where the derivative is zero, ranges of relevant_distance - where 'no-change' occurs + Determine the extremes and zero_streaks of a graph based on the derivative. - Parameters: - x (numpy.ndarray): The x values of the graph. - derivative (numpy.ndarray): The y values of the graph. + Args: + x (numpy.ndarray): The x values of the graph. + y (numpy.ndarray): The y values of the graph. Returns: - extremes: A list of tuples for breakpoints: - * relevant distance where extreme occurs - * extreme value - * minimum or maximum - zero_streaks: A list of tuples for zero_streaks: - * relevant distance where zero_streak starts - * relevant distance where zero_streak ends - * center of start- and end- zero_streak - * counter of #relevant_distances where zero-streak holds on - * extreme value for zero_streak + tuple: A tuple containing: + - list: List of breakpoints (extremes). + - list: List of zero_streaks. """ derivative = _numerical_derivative(x, y) # plt.plot(x, y, label="y") @@ -311,58 +289,20 @@ def _numerical_derivative(x, y): def diffs_from_dict_series( - dict_series: dict[str, dict[float, ProcessResult]], - dict_thematic: dict[str, BaseGeometry], + dict_series: dict[any, dict[float, ProcessResult]], + dict_thematic: dict[any, BaseGeometry], diff_metric: DiffMetric = DiffMetric.CHANGES_AREA, ): """ - Calculates a dictionary containing difference metrics for thematic elements based on - a distance series. + Calculates a dictionary containing difference metrics for thematic elements based on a distance series. - This function analyzes the changes in thematic elements (represented by - thematic_ids in `dict_thematic`) across different distances provided in the - `dict_series`. It calculates a difference metric for each thematic element at - each distance and returns a dictionary summarizing these differences. - - Args: - dict_series (dict): A dictionary where thematic_ids are distances and - values are tuples of two dictionaries. - - The first dictionary in the tuple represents thematic - element areas for a specific distance. - - The second dictionary represents the difference in - areas from the original thematic data for a specific - distance. - dict_thematic (dict): A dictionary where thematic_ids are thematic element - identifiers and values are GeoJSON geometry objects representing the - original thematic data. - diff_metric (DiffMetric): The metric used to determine the difference between - the thematic and reference data. + Parameters: + dict_series (dict): A dictionary where keys are thematic IDs and values are dictionaries mapping relative distances to ProcessResult objects. + dict_thematic (dict): A dictionary where keys are thematic IDs and values are BaseGeometry objects representing the original geometries. + diff_metric (DiffMetric, optional): The metric to use for calculating differences. Default is DiffMetric.CHANGES_AREA. Returns: - dict: A dictionary containing difference metrics for each thematic element - (`key`) across different distances. - The structure is as follows: - { - 'thematic_key1': { - distance1: difference_metric1, - distance2: difference_metric2, - ... - }, - 'thematic_key2': { - distance1: difference_metric1, - distance2: difference_metric2, - ... - }, - ... - } - - - `difference_metric`: This value depends on the chosen calculation for - thematic element change. The docstring provides examples like area - difference, percentage change, and absolute difference. - - Raises: - KeyError: If a thematic element key is missing from the results in - `dict_series`. + dict: A dictionary where keys are thematic IDs and values are dictionaries mapping relative distances to calculated difference metrics. """ diffs = {} # all the relevant distances used to calculate the series @@ -445,6 +385,18 @@ def get_collection(ref_url, limit): def geojson_to_dicts(collection, id_property): + """ + Converts a GeoJSON collection into dictionaries of geometries and properties. + + Parameters: + collection (dict): The GeoJSON collection to convert. + id_property (str): The property name to use as the key for the dictionaries. + + Returns: + tuple: Two dictionaries: + - data_dict (dict): A dictionary where keys are the id_property values and values are the geometries. + - data_dict_properties (dict): A dictionary where keys are the id_property values and values are the properties. + """ data_dict = {} data_dict_properties = {} if collection is None or "features" not in collection: @@ -460,6 +412,19 @@ def geojson_to_dicts(collection, id_property): def get_collection_by_partition( url, geometry, partition=1000, limit=DOWNLOAD_LIMIT, crs=DEFAULT_CRS ): + """ + Retrieves a collection of geographic data by partitioning the input geometry. + + Parameters: + url (str): The base URL for the data source. + geometry (object): The geometric area to partition and retrieve data for. If None, retrieves data for the entire area. + partition (int, optional): The number of partitions to divide the geometry into. Default is 1000. If less than 1, no partitioning is done. + limit (int, optional): The maximum number of items to retrieve. Default is DOWNLOAD_LIMIT. + crs (str, optional): The coordinate reference system to use. Default is DEFAULT_CRS. + + Returns: + dict: A collection of geographic data, potentially partitioned by the input geometry. + """ collection = {} if geometry is None: collection = get_collection( @@ -483,6 +448,17 @@ def get_collection_by_partition( def _add_bbox_to_url(url, crs=DEFAULT_CRS, bbox=None): + """ + Adds a bounding box (bbox) parameter to the URL for geographic data requests. + + Parameters: + url (str): The base URL for the data source. + crs (str, optional): The coordinate reference system to use. Default is DEFAULT_CRS. + bbox (str, optional): The bounding box coordinates to add to the URL. If None, no bbox is added. + + Returns: + str: The updated URL with the bbox parameter included, if provided. + """ # Load the Base reference data if bbox is not None: url = url + "&bbox-crs=" + crs + "&bbox=" + bbox @@ -490,34 +466,51 @@ def _add_bbox_to_url(url, crs=DEFAULT_CRS, bbox=None): def merge_process_results( - result_dict: dict[str, dict[float, ProcessResult]] -) -> dict[str, dict[float, ProcessResult]]: + result_dict: dict[any, dict[float, ProcessResult]], dict_multi_as_single: dict +) -> dict[any, dict[float, ProcessResult]]: """ - Merges geometries in a dictionary from multiple themes into a single theme. + Merges processresults in a dictionary from multiple themeIDs into a single themeID. Args: result_dict (dict): A dictionary where keys are theme IDs and values are process results - Returns: dict: A new dictionary with merged geometries, where keys are global - theme IDs and values are merged geometries. + Returns: dict: A new dictionary with merged geometries and remarks (processresults), where keys are global + theme IDs and values are merged geometries and remarks. """ - grouped_results: dict[str, dict[float, ProcessResult]] = {} + grouped_results: dict[any, dict[float, ProcessResult]] = {} for id_theme, dict_results in result_dict.items(): - id_theme_global = id_theme.split(MULTI_SINGLE_ID_SEPARATOR)[0] + if id_theme in dict_multi_as_single.keys(): + id_theme_global = dict_multi_as_single[id_theme] + else: + id_theme_global = id_theme if id_theme_global not in grouped_results: grouped_results[id_theme_global] = dict_results else: for rel_dist, process_result in dict_results.items(): for key in process_result: - geom: BaseGeometry = process_result[key] # noqa - if geom.is_empty or geom is None: + value = process_result[key] # noqa + if isinstance(value, str) and value != "": + existing_remark: str = grouped_results[id_theme_global][ + rel_dist + ][ + key + ] # noqa + grouped_results[id_theme_global][rel_dist][key] = ( + existing_remark + " | " + str(value) + ) continue - existing: BaseGeometry = grouped_results[id_theme_global][rel_dist][ - key - ] # noqa - grouped_results[id_theme_global][rel_dist][key] = unary_union( - [existing, geom] - ) # noqa + elif isinstance(value, BaseGeometry): + geom = value + if geom.is_empty or geom is None: + continue + existing: BaseGeometry = grouped_results[id_theme_global][ + rel_dist + ][ + key + ] # noqa + grouped_results[id_theme_global][rel_dist][key] = unary_union( + [existing, geom] + ) # noqa return grouped_results diff --git a/examples/__init__.py b/examples/__init__.py index 64baa2f..5a7f239 100644 --- a/examples/__init__.py +++ b/examples/__init__.py @@ -89,7 +89,7 @@ def _make_map(ax, processresult, thematic_dict, reference_dict): def show_map( - dict_results: dict[str, dict[float, ProcessResult]], + dict_results: dict[any, dict[float, ProcessResult]], dict_thematic, dict_reference, ): diff --git a/examples/example_aligners.py b/examples/example_aligners.py deleted file mode 100644 index a9523e5..0000000 --- a/examples/example_aligners.py +++ /dev/null @@ -1,58 +0,0 @@ -from brdr.aligner import Aligner -from brdr.enums import OpenbaarDomeinStrategy, GRBType -from brdr.grb import GRBActualLoader -from brdr.loader import GeoJsonFileLoader -from brdr.utils import diffs_from_dict_series -from examples import plot_series -from examples import show_map - -if __name__ == "__main__": - # Initiate brdr - aligner = Aligner() - # Load thematic data - aligner.load_thematic_data( - GeoJsonFileLoader("../tests/testdata/themelayer_referenced.geojson", "id_theme") - ) - - # Use GRB adp-parcels as reference polygons adp= administratieve percelen - aligner.load_reference_data( - GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=aligner) - ) - - # Example how to use the Aligner - rel_dist = 10 - dict_results = aligner.process( - relevant_distance=rel_dist, - od_strategy=OpenbaarDomeinStrategy.SNAP_FULL_AREA_SINGLE_SIDE, - ) - aligner.save_results("output/") - show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) - - rel_dist = 6 - dict_results = aligner.process( - relevant_distance=rel_dist, od_strategy=OpenbaarDomeinStrategy.SNAP_ALL_SIDE - ) - - aligner.save_results("output/") - show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) - # for key in r: - # x.get_formula(r[key]) - - # Example how to use a series (for histogram) - series = [0.1, 0.2, 0.3, 0.4, 0.5, 1, 2] - dict_series = aligner.process(series, 2, 50) - resulting_areas = diffs_from_dict_series(dict_series, aligner.dict_thematic) - plot_series(series, resulting_areas) - - # Example how to use the Aligner with threshold_overlap_percentage=-1 (original - # border will be used for cases where relevant zones cannot be used for - # determination) - rel_dist = 6 - dict_results = aligner.process( - relevant_distance=rel_dist, - od_strategy=OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE, - threshold_overlap_percentage=-1, - ) - - aligner.save_results("output/") - show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) diff --git a/examples/example_ao.py b/examples/example_ao.py deleted file mode 100644 index 0f2366e..0000000 --- a/examples/example_ao.py +++ /dev/null @@ -1,35 +0,0 @@ -import numpy as np - -from brdr.aligner import Aligner -from brdr.enums import GRBType -from brdr.grb import GRBActualLoader -from brdr.oe import OnroerendErfgoedLoader -from examples import show_map, plot_series - -if __name__ == "__main__": - # EXAMPLE to test the algorithm for erfgoedobject with relevant distance 0.2m and - # od_strategy SNAP_ALL_SIDE - - # Initiate brdr - aligner = Aligner() - # Load thematic data & reference data - aanduidingsobjecten = [117798, 116800, 117881] - - loader = OnroerendErfgoedLoader(aanduidingsobjecten) - aligner.load_thematic_data(loader) - loader = GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=aligner) - aligner.load_reference_data(loader) - - series = np.arange(0, 500, 20, dtype=int) / 100 - # predict which relevant distances are interesting to propose as resulting geometry - dict_series, dict_predictions, diffs = aligner.predictor( - relevant_distances=series, od_strategy=2, threshold_overlap_percentage=50 - ) - for key in dict_predictions.keys(): - diff = {key: diffs[key]} - plot_series(series, diff) - show_map( - {key: dict_predictions[key]}, - {key: aligner.dict_thematic[key]}, - aligner.dict_reference, - ) diff --git a/examples/example_combined_borders_adp_gbg.py b/examples/example_combined_borders_adp_gbg.py index c6ee219..146f555 100644 --- a/examples/example_combined_borders_adp_gbg.py +++ b/examples/example_combined_borders_adp_gbg.py @@ -5,17 +5,18 @@ from brdr.utils import polygonize_reference_data, geojson_to_dicts from examples import show_map, print_brdr_formula -# example to test what happens if we combine borders -# (so thematic data can use both polygons) - -# If we just combine sets of polygons (fe parcels and buildings) these polygons will -# overlap, and gives some unwanted/unexpected results. The reference data can be -# preprocessed with 'shapely-polygonize', to create a new set of non-overlapping -# polygons. These non-overlapping polygons can be used as reference-data to align the -# theme-data. - if __name__ == "__main__": + """ + # example to test what happens if we combine borders + # (so thematic data can use both polygons) + + # If we just combine sets of polygons (fe parcels and buildings) these polygons will + # overlap, and gives some unwanted/unexpected results. The reference data can be + # preprocessed with 'shapely-polygonize', to create a new set of non-overlapping + # polygons. These non-overlapping polygons can be used as reference-data to align the + # theme-data. + """ # Initiate brdr aligner = Aligner() diff --git a/examples/example_dictloader_properties.py b/examples/example_dictloader_properties.py new file mode 100644 index 0000000..2c67583 --- /dev/null +++ b/examples/example_dictloader_properties.py @@ -0,0 +1,45 @@ +from brdr.aligner import Aligner +from brdr.enums import OpenbaarDomeinStrategy, AlignerResultType +from brdr.geometry_utils import geom_from_wkt +from brdr.loader import DictLoader + +if __name__ == "__main__": + """ + Example to load dat with a DictLoader, and also adding the properties to the result + """ + # CREATE AN ALIGNER + aligner = Aligner(crs="EPSG:31370", multi_as_single_modus=True) + # ADD A THEMATIC POLYGON TO THEMATIC DICTIONARY and LOAD into Aligner + id = 1 + thematic_dict = {id: geom_from_wkt("POLYGON ((0 0, 0 9, 5 10, 10 0, 0 0))")} + # Add properties + thematic_dict_properties = { + id: {"propA": 1, "propB": 1.1, "propC": "dit is tekst", "propD": None} + } + loader = DictLoader( + data_dict=thematic_dict, data_dict_properties=thematic_dict_properties + ) + aligner.load_thematic_data(loader) + # ADD A REFERENCE POLYGON TO REFERENCE DICTIONARY and LOAD into Aligner + reference_dict = {100: geom_from_wkt("POLYGON ((0 1, 0 10,8 10,10 1,0 1))")} + loader = DictLoader(reference_dict) + aligner.load_reference_data(loader) + # EXECUTE THE ALIGNMENT + relevant_distance = 1 + process_result = aligner.process( + relevant_distance=relevant_distance, + od_strategy=OpenbaarDomeinStrategy.SNAP_SINGLE_SIDE, + threshold_overlap_percentage=50, + ) + # PRINT RESULTS IN WKT + print("result: " + process_result[id][relevant_distance]["result"].wkt) + print( + "added area: " + process_result[id][relevant_distance]["result_diff_plus"].wkt + ) + print( + "removed area: " + process_result[id][relevant_distance]["result_diff_min"].wkt + ) + fcs = aligner.get_results_as_geojson( + resulttype=AlignerResultType.PROCESSRESULTS, formula=True, attributes=True + ) + print(fcs["result"]) diff --git a/examples/example_eo.py b/examples/example_eo.py deleted file mode 100644 index dbfd77a..0000000 --- a/examples/example_eo.py +++ /dev/null @@ -1,50 +0,0 @@ -import numpy as np - -from brdr.aligner import Aligner -from brdr.enums import GRBType, AlignerResultType -from brdr.grb import GRBActualLoader -from brdr.oe import OnroerendErfgoedLoader, OEType -from brdr.utils import write_geojson -from examples import show_map, plot_series - -if __name__ == "__main__": - # EXAMPLE to test the algorithm for erfgoedobject with relevant distance 0.2m and - # od_strategy SNAP_ALL_SIDE - - # Initiate brdr - aligner = Aligner() - # Load thematic data & reference data - # dict_theme = get_oe_dict_by_ids([206363], oetype='erfgoedobjecten') - - erfgoedobjecten = [ - # 206407, - # 206403, - # 206372, - # 206369, - # 206377, - # 206371, - # 206370, - # 206368, - 206786 - ] - loader = OnroerendErfgoedLoader(objectids=erfgoedobjecten, oetype=OEType.EO) - aligner.load_thematic_data(loader) - aligner.load_reference_data(GRBActualLoader(aligner=aligner, grb_type=GRBType.ADP)) - - series = np.arange(0, 200, 20, dtype=int) / 100 - # predict which relevant distances are interesting to propose as resulting geometry - dict_series, dict_predictions, diffs = aligner.predictor( - relevant_distances=series, od_strategy=2, threshold_overlap_percentage=50 - ) - fcs = aligner.get_results_as_geojson(resulttype=AlignerResultType.PREDICTIONS) - write_geojson("output/predicted.geojson", fcs["result"]) - write_geojson("output/predicted_diff.geojson", fcs["result_diff"]) - - for key in dict_predictions.keys(): - diff = {key: diffs[key]} - plot_series(series, diff) - show_map( - {key: dict_predictions[key]}, - {key: aligner.dict_thematic[key]}, - aligner.dict_reference, - ) diff --git a/examples/example_evaluate.py b/examples/example_evaluate.py index 5b399c1..520f711 100644 --- a/examples/example_evaluate.py +++ b/examples/example_evaluate.py @@ -1,83 +1,92 @@ +import json from datetime import date import numpy as np -from shapely import from_wkt from brdr.aligner import Aligner -from brdr.constants import FORMULA_FIELD_NAME, EVALUATION_FIELD_NAME -from brdr.enums import GRBType +from brdr.constants import EVALUATION_FIELD_NAME +from brdr.enums import GRBType, AlignerResultType from brdr.grb import GRBActualLoader from brdr.grb import GRBFiscalParcelLoader -from brdr.grb import get_geoms_affected_by_grb_change +from brdr.grb import get_affected_by_grb_change from brdr.loader import DictLoader, GeoJsonFileLoader -from brdr.utils import get_series_geojson_dict -thematic_dict = { - "theme_id_1": from_wkt( - "Polygon ((174072.91453437806922011 179188.47430499014444649, 174121.17416846146807075 179179.98909460185677744, 174116.93156326730968431 179156.47799081765697338, 174110.56765547610120848 179152.58893605635967106, 174069.37903004963300191 179159.30639428040012717, 174069.37903004963300191 179159.30639428040012717, 174070.97000699743512087 179169.7361320493509993, 174072.91453437806922011 179188.47430499014444649))" +# Press the green button in the gutter to run the script. +if __name__ == "__main__": + """ + EXAMPLE of the 'evaluate()-function of 'brdr': This function evaluates thematic objects with a former brdr_formula and compares them with an actual formula; and adds evaluation-properties to the result + """ + # initiate a base Aligner, to align thematic objects on an older version of the parcels (year 2022) + base_aligner = Aligner() + # Load thematic data + loader = GeoJsonFileLoader("themelayer.geojson", "theme_identifier") + base_aligner.load_thematic_data(loader) + base_year = "2022" + name_formula = "base_formula" + # Load reference data + base_aligner.load_reference_data( + GRBFiscalParcelLoader(year=base_year, aligner=base_aligner) ) -} -base_aligner = Aligner() -loader = GeoJsonFileLoader("themelayer.geojson", "theme_identifier") -base_aligner.load_thematic_data(loader) -base_year = "2022" -base_aligner.load_reference_data( - GRBFiscalParcelLoader(year=base_year, aligner=base_aligner) -) -relevant_distance = 2 -base_process_result = base_aligner.process(relevant_distance=relevant_distance) -thematic_dict_formula = {} -thematic_dict_result = {} -for key in base_process_result: - thematic_dict_result[key] = base_process_result[key][relevant_distance]["result"] - thematic_dict_formula[key] = { - FORMULA_FIELD_NAME: base_aligner.get_brdr_formula(thematic_dict_result[key]) - } - print(key + ": " + thematic_dict_result[key].wkt) - print(key + ": " + str(thematic_dict_formula[key])) -base_aligner_result = Aligner() -base_aligner_result.load_thematic_data(DictLoader(thematic_dict_result)) -dict_affected, dict_unchanged = get_geoms_affected_by_grb_change( - base_aligner_result, - grb_type=GRBType.ADP, - date_start=date(2022, 1, 1), - date_end=date.today(), - one_by_one=False, -) -if dict_affected == {}: - print("No affected dicts") - exit() -for key, value in dict_affected.items(): - print(key + ": " + value.wkt) -actual_aligner = Aligner() -loader = DictLoader(dict_affected) -actual_aligner.load_thematic_data( - DictLoader(data_dict=dict_affected, data_dict_properties=thematic_dict_formula) -) -actual_aligner.load_reference_data( - GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner) -) -actual_aligner.relevant_distances = np.arange(0, 200, 10, dtype=int) / 100 -dict_evaluated, prop_dictionary = actual_aligner.compare( - # thematic_dict_formula=thematic_dict_formula, - threshold_area=5, - threshold_percentage=1, - dict_unchanged=dict_unchanged, -) + relevant_distance = 2 + # Align the thematic object on the parcelborders of 2022, to simulate a base-situation + base_process_result = base_aligner.process(relevant_distance=2) -fc = get_series_geojson_dict( - dict_evaluated, - crs=actual_aligner.CRS, - id_field=actual_aligner.name_thematic_id, - series_prop_dict=prop_dictionary, -) -print(fc["result"]) -fcs = actual_aligner.get_results_as_geojson(formula=True) -print(fcs["result"]) + # Collect the base-situation (base-geometries and the brdr_formula from that moment + thematic_dict_formula = {} + thematic_dict_result = {} + for key in base_process_result: + thematic_dict_result[key] = base_process_result[key][relevant_distance][ + "result" + ] + thematic_dict_formula[key] = { + name_formula: json.dumps( + base_aligner.get_brdr_formula(thematic_dict_result[key]) + ) + } + print(key + ": " + thematic_dict_result[key].wkt) + print(key + ": " + str(thematic_dict_formula[key])) -for feature in fc["result"]["features"]: - print( - feature["properties"][actual_aligner.name_thematic_id] - + ": " - + feature["properties"][EVALUATION_FIELD_NAME] + # (OPTIONAL) Check for changes in the period 2022-now of the reference-parcels (GRB/Flanders-specific function) + affected, unaffected = get_affected_by_grb_change( + dict_thematic=thematic_dict_result, + grb_type=GRBType.ADP, + date_start=date(2022, 1, 1), + date_end=date.today(), + one_by_one=False, ) + if len(affected) == 0: + print("No affected dicts") + exit() + print("Affected_IDs: " + str(affected)) + + # Start an aligner to align thematic objects on the actual parcels + actual_aligner = Aligner(relevant_distances=np.arange(0, 200, 10, dtype=int) / 100) + # Load the thematic objects (aligned on 2022) and also give the brdr_formula from 2022 as property + actual_aligner.load_thematic_data( + DictLoader( + data_dict=thematic_dict_result, data_dict_properties=thematic_dict_formula + ) + ) + # Load reference data; the actual parcels + actual_aligner.load_reference_data( + GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner) + ) + # Use the EVALUATE-function + dict_evaluated, prop_dictionary = actual_aligner.evaluate( + ids_to_evaluate=affected, base_formula_field=name_formula + ) + + # SHOW the EVALUATED results + fc = actual_aligner.get_results_as_geojson( + resulttype=AlignerResultType.EVALUATED_PREDICTIONS, + formula=True, + attributes=True, + ) + print(fc["result"]) + + for feature in fc["result"]["features"]: + print( + feature["properties"][actual_aligner.name_thematic_id] + + ": " + + feature["properties"][EVALUATION_FIELD_NAME] + ) diff --git a/examples/example_evaluate_ao.py b/examples/example_evaluate_ao.py deleted file mode 100644 index 6025736..0000000 --- a/examples/example_evaluate_ao.py +++ /dev/null @@ -1,67 +0,0 @@ -from datetime import date - -import numpy as np - -from brdr.aligner import Aligner -from brdr.constants import EVALUATION_FIELD_NAME, FORMULA_FIELD_NAME -from brdr.enums import GRBType -from brdr.grb import ( - get_geoms_affected_by_grb_change, - GRBFiscalParcelLoader, - GRBActualLoader, -) -from brdr.loader import DictLoader -from brdr.oe import OnroerendErfgoedLoader -from brdr.utils import get_series_geojson_dict - -base_aligner = Aligner() -loader = OnroerendErfgoedLoader([120288]) -base_aligner.load_thematic_data(loader) -base_year = "2022" -base_aligner.load_reference_data( - GRBFiscalParcelLoader(year=base_year, aligner=base_aligner) -) -relevant_distance = 3 -base_process_result = base_aligner.process(relevant_distance=relevant_distance) -thematic_dict_formula = {} -thematic_dict_result = {} -for key in base_process_result: - thematic_dict_result[key] = base_process_result[key][relevant_distance]["result"] - thematic_dict_formula[key] = { - FORMULA_FIELD_NAME: base_aligner.get_brdr_formula(thematic_dict_result[key]) - } -base_aligner_result = Aligner() -base_aligner_result.load_thematic_data(DictLoader(thematic_dict_result)) -dict_affected, dict_unchanged = get_geoms_affected_by_grb_change( - base_aligner_result, - grb_type=GRBType.ADP, - date_start=date(2022, 1, 1), - date_end=date.today(), - one_by_one=False, -) -if dict_affected == {}: - print("No affected dicts") - exit() - -actual_aligner = Aligner() -actual_aligner.load_thematic_data( - DictLoader(data_dict=dict_affected, data_dict_properties=thematic_dict_formula) -) -loader = GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner) -actual_aligner.load_reference_data(loader) -series = np.arange(0, 300, 10, dtype=int) / 100 - -dict_evaluated, prop_dictionary = actual_aligner.compare( - threshold_area=5, - threshold_percentage=1, - dict_unchanged=dict_unchanged, -) - -fc = get_series_geojson_dict( - dict_evaluated, - crs=actual_aligner.CRS, - id_field=actual_aligner.name_thematic_id, - series_prop_dict=prop_dictionary, -) -for feature in fc["result"]["features"]: - print(feature["properties"][EVALUATION_FIELD_NAME]) diff --git a/examples/example_aligner.py b/examples/example_geojsonloader.py similarity index 87% rename from examples/example_aligner.py rename to examples/example_geojsonloader.py index ed08939..4ae7adf 100644 --- a/examples/example_aligner.py +++ b/examples/example_geojsonloader.py @@ -1,13 +1,16 @@ from brdr.aligner import Aligner -from brdr.enums import OpenbaarDomeinStrategy, GRBType +from brdr.enums import GRBType from brdr.grb import GRBActualLoader from brdr.loader import GeoJsonLoader -from examples import show_map +from examples import show_map, print_brdr_formula if __name__ == "__main__": + """ + #EXAMPLE of a Geojson, aligned by 'brdr' + """ + # Initiate brdr aligner = Aligner() - # Load thematic data thematic_json = { "type": "FeatureCollection", @@ -45,14 +48,14 @@ loader = GeoJsonLoader(_input=thematic_json, id_property="theme_identifier") aligner.load_thematic_data(loader) + # Load reference data: The actual GRB-parcels loader = GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=aligner) aligner.load_reference_data(loader) # Example how to use the Aligner - rel_dist = 6 - dict_results = aligner.process( - relevant_distance=rel_dist, - od_strategy=OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE, - ) + dict_results = aligner.process(relevant_distance=6) + + # show results aligner.save_results("output/") show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) + print_brdr_formula(dict_results, aligner) diff --git a/examples/example_grbspecificloader.py b/examples/example_grbspecificloader.py index 93d2c20..1fb7cbb 100644 --- a/examples/example_grbspecificloader.py +++ b/examples/example_grbspecificloader.py @@ -1,16 +1,33 @@ +import os + from shapely import from_wkt from brdr.aligner import Aligner +from brdr.enums import AlignerInputType from brdr.grb import GRBSpecificDateParcelLoader from brdr.loader import DictLoader +from brdr.utils import write_geojson + +if __name__ == "__main__": + aligner = Aligner() + thematic_dict = { + "theme_id_1": from_wkt( + "Polygon ((172283.76869662097305991 174272.85233648214489222, 172276.89871930953813717 174278.68436246179044247, 172274.71383684969623573 174280.57171753142029047, 172274.63047763772192411 174280.64478165470063686, 172272.45265833073062822 174282.52660570573061705, 172269.33533191855531186 174285.22093996312469244, 172265.55258252174826339 174288.49089696351438761, 172258.77032718938426115 174294.22654021997004747, 172258.63259260458289646 174294.342757155187428, 172254.93673790179309435 174288.79932878911495209, 172248.71360730109154247 174279.61860501393675804, 172248.96566232520854101 174279.43056782521307468, 172255.25363882273086347 174274.73737183399498463, 172257.08298882702365518 174273.37133203260600567, 172259.32325354730710387 174271.69890458136796951, 172261.65807284769834951 174269.9690355472266674, 172266.35596220899606124 174266.4871726930141449, 172273.34350050613284111 174261.30863015633076429, 172289.60360219911672175 174249.35944479051977396, 172293.30328181147342548 174246.59864199347794056, 172297.34760522318538278 174253.10583685990422964, 172289.53060952731175348 174259.6846851697191596, 172292.86485871637705714 174265.19099397677928209, 172283.76869662097305991 174272.85233648214489222))" + ) + } + # EXAMPLE to use a GRBSpecificLoader, that retrieves the parcels for a specific data, based on the 2 fiscal parcel-situations of the year before and after + # Based on the date, the referencelayer will be different + date = "2023-05-03" + date = "2023-08-03" + loader = DictLoader(thematic_dict) + aligner.load_thematic_data(loader) + loader = GRBSpecificDateParcelLoader(date=date, aligner=aligner) + aligner.load_reference_data(loader) + + # aligner.process() + # aligner.save_results(path = "output/") -aligner = Aligner() -thematic_dict = { - "theme_id_1": from_wkt( - "Polygon ((172283.76869662097305991 174272.85233648214489222, 172276.89871930953813717 174278.68436246179044247, 172274.71383684969623573 174280.57171753142029047, 172274.63047763772192411 174280.64478165470063686, 172272.45265833073062822 174282.52660570573061705, 172269.33533191855531186 174285.22093996312469244, 172265.55258252174826339 174288.49089696351438761, 172258.77032718938426115 174294.22654021997004747, 172258.63259260458289646 174294.342757155187428, 172254.93673790179309435 174288.79932878911495209, 172248.71360730109154247 174279.61860501393675804, 172248.96566232520854101 174279.43056782521307468, 172255.25363882273086347 174274.73737183399498463, 172257.08298882702365518 174273.37133203260600567, 172259.32325354730710387 174271.69890458136796951, 172261.65807284769834951 174269.9690355472266674, 172266.35596220899606124 174266.4871726930141449, 172273.34350050613284111 174261.30863015633076429, 172289.60360219911672175 174249.35944479051977396, 172293.30328181147342548 174246.59864199347794056, 172297.34760522318538278 174253.10583685990422964, 172289.53060952731175348 174259.6846851697191596, 172292.86485871637705714 174265.19099397677928209, 172283.76869662097305991 174272.85233648214489222))" + fc = aligner.get_input_as_geojson( + inputtype=AlignerInputType.REFERENCE, ) -} -loader = DictLoader(thematic_dict) -aligner.load_thematic_data(loader) -loader = GRBSpecificDateParcelLoader(date="2023-05-03", aligner=aligner) -aligner.load_reference_data(loader) + write_geojson(os.path.join("output/", "grb_adp_" + date + ".geojson"), fc) diff --git a/examples/example_local_data.py b/examples/example_local_data.py deleted file mode 100644 index 23599fe..0000000 --- a/examples/example_local_data.py +++ /dev/null @@ -1,23 +0,0 @@ -from brdr.aligner import Aligner -from brdr.enums import OpenbaarDomeinStrategy -from brdr.loader import GeoJsonFileLoader - -if __name__ == "__main__": - # Initiate brdr - aligner = Aligner() - # Load local thematic data and reference data - loader = GeoJsonFileLoader( - "../tests/testdata/themelayer_referenced.geojson", "id_theme" - ) - aligner.load_thematic_data(loader) - loader = GeoJsonFileLoader("../tests/testdata/reference_leuven.geojson", "capakey") - aligner.load_reference_data(loader) - # Example how to use the Aligner - rel_dist = 1 - dict_results = aligner.process( - relevant_distance=rel_dist, - od_strategy=OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE, - ) - - aligner.save_results("output/") - # show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) diff --git a/examples/example_multi_to_single.py b/examples/example_multi_to_single.py deleted file mode 100644 index 456f2b8..0000000 --- a/examples/example_multi_to_single.py +++ /dev/null @@ -1,47 +0,0 @@ -from brdr.aligner import Aligner -from brdr.enums import GRBType -from brdr.grb import GRBActualLoader -from brdr.oe import OnroerendErfgoedLoader -from examples import print_brdr_formula -from examples import show_map - -# EXAMPLE of "multi_as_single_modus" - -# Initiate brdr -aligner = Aligner() -# WITHOUT MULTI_TO_SINGLE -aligner.multi_as_single_modus = False -# Load thematic data & reference data -# Get a specific feature of OE that exists out of a Multipolygon -loader = OnroerendErfgoedLoader([110082]) -aligner.load_thematic_data(loader) -aligner.load_reference_data( - GRBActualLoader(aligner=aligner, grb_type=GRBType.GBG, partition=1000) -) - -rel_dist = 20 -dict_results = aligner.process(relevant_distance=rel_dist, od_strategy=4) -aligner.save_results("output/") -show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) - -print_brdr_formula(dict_results, aligner) - -# WITH MULTI_TO_SINGLE - -# Initiate brdr -aligner = Aligner() -aligner.multi_as_single_modus = True -# Load thematic data & reference data -# Get a specific feature of OE that exists out of a Multipolygon -loader = OnroerendErfgoedLoader([110082]) -aligner.load_thematic_data(loader) -aligner.load_reference_data( - GRBActualLoader(aligner=aligner, grb_type=GRBType.GBG, partition=1000) -) - -rel_dist = 20 -dict_results = aligner.process(relevant_distance=rel_dist, od_strategy=4) -aligner.save_results("output/") -show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) - -print_brdr_formula(dict_results, aligner) diff --git a/examples/example_multi_to_single_modus.py b/examples/example_multi_to_single_modus.py new file mode 100644 index 0000000..16f4231 --- /dev/null +++ b/examples/example_multi_to_single_modus.py @@ -0,0 +1,53 @@ +from brdr.aligner import Aligner +from brdr.enums import GRBType, OpenbaarDomeinStrategy +from brdr.grb import GRBActualLoader +from brdr.oe import OnroerendErfgoedLoader, OEType +from examples import print_brdr_formula +from examples import show_map + +if __name__ == "__main__": + """ + # This example shows the usage of the setting 'multi_as_single_modus' + # True: (default): All polygons inside a MultiPolygon will be processed seperataly by the algorithm, and merged after processing. + # False: Multipolygon will be processed directly by the algorithm + + """ + # Example (ErfgoedObject): https://inventaris.onroerenderfgoed.be/erfgoedobjecten/305858 + loader = OnroerendErfgoedLoader([305858], oetype=OEType.EO) + relevant_distance = 5 # rd is taken very high to show the difference + od_strategy = OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE + threshold_circle_ratio = 0.75 # default it is 0.98, but because it are not fully circles in this example we put this on 0.75 + + # EXAMPLE of "multi_as_single_modus"=FALSE + print("EXAMPLE with 'multi_as_single_modus'=False") + aligner = Aligner( + multi_as_single_modus=False, + relevant_distance=relevant_distance, + od_strategy=od_strategy, + threshold_circle_ratio=threshold_circle_ratio, + ) + aligner.load_thematic_data(loader) + aligner.load_reference_data( + GRBActualLoader(aligner=aligner, grb_type=GRBType.ADP, partition=1000) + ) + dict_results = aligner.process() + aligner.save_results("output/") + print_brdr_formula(dict_results, aligner) + show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) + + # WITH "multi_as_single_modus"=True + print("EXAMPLE with 'multi_as_single_modus'=True") + aligner = Aligner( + multi_as_single_modus=True, + relevant_distance=relevant_distance, + od_strategy=od_strategy, + threshold_circle_ratio=threshold_circle_ratio, + ) + aligner.load_thematic_data(loader) + aligner.load_reference_data( + GRBActualLoader(aligner=aligner, grb_type=GRBType.ADP, partition=1000) + ) + dict_results = aligner.process() + aligner.save_results("output/") + print_brdr_formula(dict_results, aligner) + show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) diff --git a/examples/example_multipolygon.py b/examples/example_multipolygon.py deleted file mode 100644 index 0c3ed81..0000000 --- a/examples/example_multipolygon.py +++ /dev/null @@ -1,36 +0,0 @@ -# Initiate brdr -from brdr.aligner import Aligner -from brdr.enums import GRBType, AlignerResultType -from brdr.grb import GRBActualLoader -from brdr.loader import DictLoader, GeoJsonFileLoader -from brdr.utils import multipolygons_to_singles -from brdr.utils import write_geojson - -aligner0 = Aligner() - -# Load thematic data - -aligner0.load_thematic_data( - GeoJsonFileLoader("../tests/testdata/multipolygon.geojson", "theme_identifier") -) -aligner0.dict_thematic = multipolygons_to_singles(aligner0.dict_thematic) -aligner0.load_thematic_data( - DictLoader( - aligner0.dict_thematic, - ) -) -# gebruik de actuele adp-percelen adp= administratieve percelen -aligner = Aligner() -aligner.load_thematic_data(DictLoader(aligner0.dict_thematic)) - -aligner.load_reference_data( - GRBActualLoader(aligner=aligner, grb_type=GRBType.ADP, partition=1000) -) - -dict_series, dict_predictions, diffs = aligner.predictor() -fcs = aligner.get_results_as_geojson( - resulttype=AlignerResultType.PREDICTIONS, formula=True -) -aligner.save_results("output/") -write_geojson("output/predicted.geojson", fcs["result"]) -write_geojson("output/predicted_diff.geojson", fcs["result_diff"]) diff --git a/examples/example_131635.py b/examples/example_onroerenderfgoed.py similarity index 55% rename from examples/example_131635.py rename to examples/example_onroerenderfgoed.py index 7051514..c862786 100644 --- a/examples/example_131635.py +++ b/examples/example_onroerenderfgoed.py @@ -1,30 +1,26 @@ from brdr.aligner import Aligner from brdr.enums import GRBType from brdr.grb import GRBActualLoader -from brdr.oe import OnroerendErfgoedLoader +from brdr.oe import OnroerendErfgoedLoader, OEType from examples import print_brdr_formula from examples import show_map if __name__ == "__main__": - # TODO - # EXAMPLE for a thematic Polygon (aanduid_id 131635) + # EXAMPLE for a thematic Polygon from Onroerend Erfgoed (https://inventaris.onroerenderfgoed.be/aanduidingsobjecten/131635) # Initiate brdr aligner = Aligner() - # Load thematic data & reference data - loader = OnroerendErfgoedLoader([131635]) + # Load thematic data from Onroerend Erfgoed + loader = OnroerendErfgoedLoader(objectids=[131635], oetype=OEType.AO) aligner.load_thematic_data(loader) + # Load reference data: The actual GRB-parcels loader = GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=aligner) aligner.load_reference_data(loader) - ref_geojson = aligner.get_input_as_geojson() - # RESULTS - rel_dist = 2 - dict_results = aligner.process(relevant_distance=rel_dist, od_strategy=4) - fcs = aligner.get_results_as_geojson() - print(fcs["result"]) - # put resulting tuple in a dictionary - aligner.save_results("output/", formula=True) + # PROCESS + dict_results = aligner.process(relevant_distance=2) + # GET/SHOW results + aligner.save_results("output/", formula=True) show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) print_brdr_formula(dict_results, aligner) diff --git a/examples/example_parcel_change_detector.py b/examples/example_parcel_change_detector.py index b419396..375d915 100644 --- a/examples/example_parcel_change_detector.py +++ b/examples/example_parcel_change_detector.py @@ -1,19 +1,27 @@ -import logging +import os +from datetime import datetime from brdr.aligner import Aligner -from brdr.constants import EVALUATION_FIELD_NAME, RELEVANT_DISTANCE_FIELD_NAME +from brdr.constants import ( + EVALUATION_FIELD_NAME, + RELEVANT_DISTANCE_FIELD_NAME, + FORMULA_FIELD_NAME, +) from brdr.grb import GRBFiscalParcelLoader from brdr.grb import update_to_actual_grb from brdr.oe import OnroerendErfgoedLoader +from brdr.utils import write_geojson -# This code shows an example how the aligner can be used inside a flow of -# parcel change detection: -# * it can be used to do a first alignment of the original features -# (in this case, based on parcel version adpF2023 (1st January 2023) -# * it can be used to do a new alignment on the actual version of the parcels adp -# * it can be used to convert the geometries to a formula, to compare and -# evaluate if equality is detected after alignement - +if __name__ == "__main__": + """ + # This code shows an example how the aligner can be used inside a flow of + # parcel change detection: + # * it can be used to do a first alignment of the original features + # (in this case, based on parcel version adpF2022 (1st January 2022) + # * it can be used to do a new alignment on the actual version of the parcels adp + # * it can be used to convert the geometries to a formula, to compare and + # evaluate if equality is detected after alignement + """ counter_excluded = 0 # PARAMS # ========= @@ -21,10 +29,6 @@ limit = 10000 bbox = [172800, 170900, 173000, 171100] bbox = [172000, 172000, 174000, 174000] -# bbox = "170000,170000,175000,174900" -# bbox = "100000,195000,105000,195900" -# bbox = "150000,210000,155000,214900" -# bbox = "173500,173500,174000,174000" # example "aanduid_id" = 34195 base_year = "2022" # relevant distance that is used to align the original geometries to the # reference-polygons of the base-year @@ -39,18 +43,19 @@ # Initiate an Aligner to create a themeset that is base-referenced on a specific # base_year base_aligner = Aligner() +print("start loading OE-objects") # Load the thematic data to evaluate loader = OnroerendErfgoedLoader(bbox=bbox, partition=0) base_aligner.load_thematic_data(loader) -logging.info( +print( "Number of OE-thematic features loaded into base-aligner: " + str(len(base_aligner.dict_thematic)) ) base_aligner.load_reference_data( - GRBFiscalParcelLoader(year=base_year, aligner=base_aligner) + GRBFiscalParcelLoader(year=base_year, aligner=base_aligner, partition=1000) ) - +print("Reference-data loaded") # Exclude objects bigger than specified area keys_to_exclude = [] nr_features = len(base_aligner.dict_thematic) @@ -58,13 +63,13 @@ if base_aligner.dict_thematic[key].area > excluded_area: keys_to_exclude.append(key) counter_excluded = counter_excluded + 1 - logging.info( - "geometrie excluded; bigger than " + str(excluded_area) + ": " + key - ) + print("geometrie excluded; bigger than " + str(excluded_area) + ": " + key) for x in keys_to_exclude: del base_aligner.dict_thematic[x] # # Align the features to the base-GRB +print("Process base objects") +starttime = datetime.now() base_process_result = base_aligner.process(relevant_distance=base_correction) # get resulting aligned features on Adpfxxxx, with formula processresults = base_aligner.get_results_as_geojson(formula=True) @@ -74,24 +79,36 @@ featurecollection_base_result = processresults["result"] # Update Featurecollection to actual version +print("Actualise base objects") fcs = update_to_actual_grb( featurecollection_base_result, base_aligner.name_thematic_id, + base_formula_field=FORMULA_FIELD_NAME, max_distance_for_actualisation=max_distance_for_actualisation, ) +write_geojson( + os.path.join("output/", "parcel_change_detector_with.geojson"), fcs["result"] +) + counter_equality = 0 counter_equality_by_alignment = 0 counter_difference = 0 +counter_no_change = 0 +# TODO: counter_difference collects al the 'TO_CHECK's' but these are multiple proposals, so clean up the stats +# TODO: Move this as general output from the updater? for feature in fcs["result"]["features"]: if EVALUATION_FIELD_NAME in feature["properties"].keys(): ev = feature["properties"][EVALUATION_FIELD_NAME] + print(ev) rd = feature["properties"][RELEVANT_DISTANCE_FIELD_NAME] if ev.startswith("equal") and rd == 0: counter_equality = counter_equality + 1 elif ev.startswith("equal") and rd > 0: counter_equality_by_alignment = counter_equality_by_alignment + 1 + elif ev.startswith("no_change"): + counter_no_change = counter_no_change + 1 else: counter_difference = counter_difference + 1 @@ -102,8 +119,13 @@ + str(counter_equality) + "//Equality by alignment: " + str(counter_equality_by_alignment) + + "//No change: " + + str(counter_no_change) + "//Difference: " + str(counter_difference) + "//Excluded: " + str(counter_excluded) ) +endtime = datetime.now() +seconds = (endtime - starttime).total_seconds() +print("duration: " + str(seconds)) diff --git a/examples/example_parcel_vs_building.py b/examples/example_parcel_vs_building.py index 7d3cba1..ffc07c7 100644 --- a/examples/example_parcel_vs_building.py +++ b/examples/example_parcel_vs_building.py @@ -7,9 +7,11 @@ from brdr.utils import diffs_from_dict_series from examples import plot_series -# example to check if we can notice if it is better to align to a building instead of a -# parcel if __name__ == "__main__": + """ + # example to check if we can notice if it is better to align to a building instead of a + # parcel + """ # Initiate brdr aligner_x = Aligner() # Load thematic data & reference data (parcels) @@ -35,9 +37,13 @@ # Example how to use a series (for histogram) series = np.arange(0, 300, 10, dtype=int) / 100 - x_dict_series = aligner_x.process(series, 4, 50) + x_dict_series = aligner_x.process( + relevant_distances=series, od_strategy=4, threshold_overlap_percentage=50 + ) x_resulting_areas = diffs_from_dict_series(x_dict_series, aligner_x.dict_thematic) - y_dict_series = aligner_y.process(series, 4, 50) + y_dict_series = aligner_y.process( + relevant_distances=series, od_strategy=4, threshold_overlap_percentage=50 + ) y_resulting_areas = diffs_from_dict_series(y_dict_series, aligner_y.dict_thematic) # plot_diffs(series,x_resulting_areas) # plot_diffs(series,y_resulting_areas) diff --git a/examples/example_predictor.py b/examples/example_predictor.py index e7d60ac..276cc04 100644 --- a/examples/example_predictor.py +++ b/examples/example_predictor.py @@ -1,14 +1,15 @@ import numpy as np from brdr.aligner import Aligner -from brdr.enums import GRBType, AlignerResultType +from brdr.enums import GRBType, AlignerResultType, OpenbaarDomeinStrategy from brdr.grb import GRBActualLoader from brdr.loader import GeoJsonFileLoader +from examples import show_map, plot_series # Press the green button in the gutter to run the script. if __name__ == "__main__": """ - example to use the predictor-function to automatically predict which resulting + EXAMPLE to use the predictor-function to automatically predict which resulting geometries are interesting to look at (based on detection of breakpoints and relevant distances of 'no-change') """ @@ -19,21 +20,28 @@ "../tests/testdata/test_wanted_changes.geojson", "theme_id" ) aligner.load_thematic_data(loader) + # Load reference data loader = GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=aligner) aligner.load_reference_data(loader) + # PREDICT the 'stable' relevant distances, for a series of relevant distances series = np.arange(0, 300, 10, dtype=int) / 100 # predict which relevant distances are interesting to propose as resulting geometry dict_series, dict_predictions, diffs = aligner.predictor( - relevant_distances=series, od_strategy=4, threshold_overlap_percentage=50 + relevant_distances=series, + od_strategy=OpenbaarDomeinStrategy.SNAP_FULL_AREA_ALL_SIDE, + threshold_overlap_percentage=50, ) + + # SHOW results of the predictions fcs = aligner.get_results_as_geojson( resulttype=AlignerResultType.PREDICTIONS, formula=False ) print(fcs["result"]) - # for key in dict_predictions: - # show_map( - # {key:dict_predictions[key]}, - # {key: aligner.dict_thematic[key]}, - # aligner.dict_reference, - # ) + for key in dict_predictions: + plot_series(series, {key: diffs[key]}) + show_map( + {key: dict_predictions[key]}, + {key: aligner.dict_thematic[key]}, + aligner.dict_reference, + ) diff --git a/examples/example_predictor_double_prediction.py b/examples/example_predictor_double_prediction.py deleted file mode 100644 index 9b2ad5e..0000000 --- a/examples/example_predictor_double_prediction.py +++ /dev/null @@ -1,42 +0,0 @@ -import numpy as np -from shapely import from_wkt - -from brdr.aligner import Aligner -from brdr.enums import GRBType -from brdr.grb import GRBActualLoader -from brdr.loader import DictLoader - -# Press the green button in the gutter to run the script. -if __name__ == "__main__": - """ - example to use the predictor-function to automatically predict which resulting - geometries are interesting to look at (based on detection of breakpoints and - relevant distances of 'no-change') - """ - # Initiate an Aligner - aligner = Aligner() - # Load thematic data & reference data - loader = DictLoader( - { - "id1": from_wkt( - "MultiPolygon Z (((138430.4033999964594841 194082.86080000177025795 0, 138422.19659999758005142 194080.36510000005364418 0, 138419.01550000160932541 194079.34930000081658363 0, 138412.59849999845027924 194077.14139999821782112 0, 138403.65579999983310699 194074.06430000066757202 0, 138402.19910000264644623 194077.67480000108480453 0, 138401.83420000225305557 194078.57939999923110008 0, 138400.89329999685287476 194080.91140000149607658 0, 138400.31650000065565109 194080.67880000174045563 0, 138399.27300000190734863 194083.37680000066757202 0, 138405.93310000002384186 194085.95410000160336494 0, 138413.51049999892711639 194088.80620000138878822 0, 138427.25680000334978104 194094.29969999939203262 0, 138430.4033999964594841 194082.86080000177025795 0)))" - ) - } - ) - aligner.load_thematic_data(loader) - loader = GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=aligner) - aligner.load_reference_data(loader) - - series = np.arange(0, 800, 10, dtype=int) / 100 - # predict which relevant distances are interesting to propose as resulting geometry - dict_series, dict_predictions, diffs = aligner.predictor( - relevant_distances=series, od_strategy=4, threshold_overlap_percentage=50 - ) - fcs = aligner.get_results_as_geojson(formula=False) - print(fcs["result"]) - # for key in dict_predictions: - # show_map( - # {key:dict_predictions[key]}, - # {key: aligner.dict_thematic[key]}, - # aligner.dict_reference, - # ) diff --git a/examples/example_process_relevant_distances.py b/examples/example_process_relevant_distances.py new file mode 100644 index 0000000..1c7a8fe --- /dev/null +++ b/examples/example_process_relevant_distances.py @@ -0,0 +1,64 @@ +from brdr.aligner import Aligner +from brdr.enums import OpenbaarDomeinStrategy, GRBType +from brdr.grb import GRBActualLoader +from brdr.loader import GeoJsonLoader +from brdr.utils import diffs_from_dict_series +from examples import plot_series +from examples import show_map + +if __name__ == "__main__": + # EXAMPLE to process a series of relevant distances + # Initiate brdr + aligner = Aligner() + # Load thematic data + thematic_json = { + "type": "FeatureCollection", + "name": "test", + "crs": {"type": "name", "properties": {"name": "urn:ogc:def:crs:EPSG::31370"}}, + "features": [ + { + "type": "Feature", + "properties": {"fid": 1100, "id": 1100, "theme_identifier": "1100"}, + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [170020.885142877610633, 171986.324472956912359], + [170078.339491307124263, 172031.344329243671382], + [170049.567976467020344, 172070.009247593494365], + [170058.413533725659363, 172089.43287940043956], + [170071.570170061604585, 172102.403589786874363], + [170061.212614970601862, 172153.235667688539252], + [170008.670597244432429, 172137.344562214449979], + [169986.296310421457747, 172121.231194059771951], + [169979.658902874827618, 172095.676061166130239], + [169980.509304589155363, 172078.495172578957863], + [169985.424311356124235, 172065.162689057324314], + [169971.609697333740769, 172057.093288242496783], + [170020.885142877610633, 171986.324472956912359], + ] + ] + ], + }, + } + ], + } + + loader = GeoJsonLoader(_input=thematic_json, id_property="theme_identifier") + aligner.load_thematic_data(loader) + # Load reference data: The actual GRB-parcels + aligner.load_reference_data( + GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=aligner) + ) + # PROCESS a series of relevant distances + relevant_distances = [0.5, 1, 3, 6] + dict_results = aligner.process( + relevant_distances=relevant_distances, + od_strategy=OpenbaarDomeinStrategy.SNAP_ALL_SIDE, + threshold_overlap_percentage=50, + ) + # SHOW results: map and plotted changes + show_map(dict_results, aligner.dict_thematic, aligner.dict_reference) + resulting_areas = diffs_from_dict_series(dict_results, aligner.dict_thematic) + plot_series(relevant_distances, resulting_areas) diff --git a/examples/example_refactor_dict_series.py b/examples/example_refactor_dict_series.py deleted file mode 100644 index 511a6a8..0000000 --- a/examples/example_refactor_dict_series.py +++ /dev/null @@ -1,26 +0,0 @@ -from brdr.aligner import Aligner -from brdr.enums import GRBType -from brdr.grb import GRBActualLoader -from brdr.oe import OnroerendErfgoedLoader - - -# EXAMPLE to test the algorithm for erfgoedobject with relevant distance 0.2m and -# od_strategy SNAP_ALL_SIDE - -# Initiate brdr -aligner = Aligner() -# Load thematic data & reference data -aanduidingsobjecten = [117798, 116800, 117881] - -loader = OnroerendErfgoedLoader(aanduidingsobjecten) -aligner.load_thematic_data(loader) -loader = GRBActualLoader(grb_type=GRBType.ADP, partition=1000, aligner=aligner) -aligner.load_reference_data(loader) - -test = aligner.process() -test = aligner.process([1, 2, 3]) -test = aligner.predictor() -fcs = aligner.get_results_as_geojson(formula=True) -print(test) -print(fcs) -print(fcs["result"]) diff --git a/examples/example_speedtest.py b/examples/example_speedtest.py index ff460f1..d1b6da8 100644 --- a/examples/example_speedtest.py +++ b/examples/example_speedtest.py @@ -4,38 +4,49 @@ from brdr.aligner import Aligner from brdr.loader import GeoJsonFileLoader -# Initiate brdr -aligner = Aligner(relevant_distance=2) -aligner.multi_as_single_modus = True -# Load local thematic data and reference data -# loader = GeoJsonFileLoader( -# "../tests/testdata/theme.geojson", "theme_identifier" -# ) -loader = GeoJsonFileLoader( - "../tests/testdata/themelayer_not_referenced.geojson", "theme_identifier" -) -aligner.load_thematic_data(loader) -loader = GeoJsonFileLoader("../tests/testdata/reference_leuven.geojson", "capakey") -aligner.load_reference_data(loader) - -times = [] -for iter in range(1, 3): - starttime = datetime.now() - - # Example how to use the Aligner - aligner.predictor() - fcs = aligner.get_results_as_geojson(formula=True) - endtime = datetime.now() - seconds = (endtime - starttime).total_seconds() - times.append(seconds) - print(seconds) -print("duration: " + str(times)) - -print("Min: " + str(min(times))) -print("Max: " + str(max(times))) -print("Mean: " + str(statistics.mean(times))) -print("Median: " + str(statistics.median(times))) -print("Stdv: " + str(statistics.stdev(times))) + +def main(): + """ + EXAMPLE of a test to measure the speed of the aligner + :return: + """ + # Initiate brdr + aligner = Aligner(relevant_distance=2, max_workers=None) + iterations = 10 + aligner.multi_as_single_modus = True + # Load local thematic data and reference data + # loader = GeoJsonFileLoader( + # "../tests/testdata/theme.geojson", "theme_identifier" + # ) + loader = GeoJsonFileLoader( + "../tests/testdata/themelayer_not_referenced.geojson", "theme_identifier" + ) + aligner.load_thematic_data(loader) + loader = GeoJsonFileLoader("../tests/testdata/reference_leuven.geojson", "capakey") + aligner.load_reference_data(loader) + + times = [] + total_starttime = datetime.now() + for iter in range(1, iterations + 1): + starttime = datetime.now() + + # Example how to use the Aligner + aligner.predictor() + fcs = aligner.get_results_as_geojson(formula=True) + endtime = datetime.now() + seconds = (endtime - starttime).total_seconds() + times.append(seconds) + print(seconds) + total_endtime = datetime.now() + total_seconds = (total_endtime - total_starttime).total_seconds() + print("Total time: " + str(total_seconds)) + print("duration: " + str(times)) + print("Min: " + str(min(times))) + print("Max: " + str(max(times))) + print("Mean: " + str(statistics.mean(times))) + print("Median: " + str(statistics.median(times))) + print("Stdv: " + str(statistics.stdev(times))) + # #BEFORE REFACTORING dict_series # duration: [25.652311, 27.894154, 19.641618, 19.929254, 44.754033, 25.218422, 23.167992, 18.649832, 22.899336, 52.108296] @@ -52,3 +63,6 @@ # Mean: 17.981391 # Median: 17.8996155 # Stdv: 1.504459449440969 + +if __name__ == "__main__": + main() diff --git a/examples/example_update_to_actual_grb.py b/examples/example_update_to_actual_grb.py index 9f97bf0..1940e14 100644 --- a/examples/example_update_to_actual_grb.py +++ b/examples/example_update_to_actual_grb.py @@ -1,76 +1,87 @@ from brdr.aligner import Aligner -from brdr.constants import EVALUATION_FIELD_NAME +from brdr.constants import EVALUATION_FIELD_NAME, FORMULA_FIELD_NAME from brdr.grb import GRBFiscalParcelLoader from brdr.grb import update_to_actual_grb from brdr.loader import GeoJsonLoader -# Create a featurecollection (aligned on 2022), to use for the 'update_to_actual_grb' -base_year = "2022" -base_aligner = Aligner() -name_thematic_id = "theme_identifier" -loader = GeoJsonLoader( - _input={ - "type": "FeatureCollection", - "name": "extract", - "crs": {"type": "name", "properties": {"name": "urn:ogc:def:crs:EPSG::31370"}}, - "features": [ - { - "type": "Feature", - "properties": { - "nr_calculations": 1, - "ID": "206285", - "relevant_distance": 2.0, - "area": 503.67736346047076, - "perimeter": 125.74541473322422, - "shape_index": 0.24965468741597097, - }, - "geometry": { - "type": "MultiPolygon", - "coordinates": [ - [ + +if __name__ == "__main__": + """ + EXAMPLE of the use of GRB (flanders-specific) function: Update_to_actual_grb + """ + # Create a featurecollection (aligned on 2022), to use for the 'update_to_actual_grb' + base_year = "2022" + base_aligner = Aligner() + name_thematic_id = "theme_identifier" + loader = GeoJsonLoader( + _input={ + "type": "FeatureCollection", + "name": "extract", + "crs": { + "type": "name", + "properties": {"name": "urn:ogc:def:crs:EPSG::31370"}, + }, + "features": [ + { + "type": "Feature", + "properties": { + "testattribute": "test", + "nr_calculations": 1, + "ID": "206285", + "relevant_distance": 2.0, + "area": 503.67736346047076, + "perimeter": 125.74541473322422, + "shape_index": 0.24965468741597097, + }, + "geometry": { + "type": "MultiPolygon", + "coordinates": [ [ - [138539.326299999986077, 193994.138199999986682], - [138529.3663, 193995.566400000010617], - [138522.0997, 193996.6084], - [138514.984399999986636, 193997.6287], - [138505.8261, 193996.615], - [138498.8406, 193996.4314], - [138492.9442, 193996.289500000013504], - [138491.224599999986822, 193996.2481], - [138491.4111, 194004.814699999988079], - [138514.368500000011409, 194005.1297], - [138520.2585, 194004.5753], - [138520.3946, 194005.5833], - [138520.542599999986123, 194009.731999999989057], - [138541.4173, 194007.7292], - [138539.326299999986077, 193994.138199999986682], + [ + [138539.326299999986077, 193994.138199999986682], + [138529.3663, 193995.566400000010617], + [138522.0997, 193996.6084], + [138514.984399999986636, 193997.6287], + [138505.8261, 193996.615], + [138498.8406, 193996.4314], + [138492.9442, 193996.289500000013504], + [138491.224599999986822, 193996.2481], + [138491.4111, 194004.814699999988079], + [138514.368500000011409, 194005.1297], + [138520.2585, 194004.5753], + [138520.3946, 194005.5833], + [138520.542599999986123, 194009.731999999989057], + [138541.4173, 194007.7292], + [138539.326299999986077, 193994.138199999986682], + ] ] - ] - ], - }, - } - ], - }, - id_property="ID", -) -base_aligner.load_thematic_data(loader) -base_aligner.load_reference_data( - GRBFiscalParcelLoader(year=base_year, aligner=base_aligner) -) -base_process_result = base_aligner.process(relevant_distance=2) -fcs = base_aligner.get_results_as_geojson(formula=True) -featurecollection_base_result = fcs["result"] -print(featurecollection_base_result) -# Update Featurecollection to actual version -featurecollection = update_to_actual_grb( - featurecollection_base_result, base_aligner.name_thematic_id -) -# Print results -for feature in featurecollection["result"]["features"]: - print( - feature["properties"][name_thematic_id] - + ": " - + feature["properties"][EVALUATION_FIELD_NAME] + ], + }, + } + ], + }, + id_property="ID", + ) + base_aligner.load_thematic_data(loader) + base_aligner.load_reference_data( + GRBFiscalParcelLoader(year=base_year, aligner=base_aligner) + ) + base_process_result = base_aligner.process(relevant_distance=2) + fcs = base_aligner.get_results_as_geojson(formula=True, attributes=True) + featurecollection_base_result = fcs["result"] + print(featurecollection_base_result) + # Update Featurecollection to actual version + featurecollection = update_to_actual_grb( + featurecollection_base_result, + base_aligner.name_thematic_id, + base_formula_field=FORMULA_FIELD_NAME, ) -geojson = featurecollection["result"] -print(geojson) + # Print results + for feature in featurecollection["result"]["features"]: + print( + feature["properties"][name_thematic_id] + + ": " + + feature["properties"][EVALUATION_FIELD_NAME] + ) + geojson = featurecollection["result"] + print(geojson) diff --git a/pyproject.toml b/pyproject.toml index 56a919b..11cbb85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "brdr" -version = "0.3.0" +version = "0.4.0" description = "BRDR - a Python library to assist in realigning (multi-)polygons (OGC Simple Features) to reference borders " readme = { file = "README.md", content-type = "text/markdown" } license = { file = "LICENSE" } @@ -35,7 +35,14 @@ Repository = "https://github.com/OnroerendErfgoed/brdr" Issues = "https://github.com/OnroerendErfgoed/brdr/issues" [project.optional-dependencies] +test = [ + "pytest-cov==5.0.0", + "pytest==8.1.1", + "responses==0.25.3", + "toml==0.10.2", +] dev = [ + "brdr[test]", "black==24.4.0", "flake8==7.0.0", "geopandas==0.14.3", @@ -43,10 +50,6 @@ dev = [ "matplotlib==3.8.4", "mypy==1.9.0", "pip-tools==7.4.1", - "pytest-cov==5.0.0", - "pytest==8.1.1", - "responses==0.25.3", - "toml==0.10.2", "types-requests==2.31.0.20240406", ] @@ -59,3 +62,18 @@ packages = ["brdr"] [tool.black] target-version = ['py39', 'py310', 'py311', 'py312'] + +# Config for hatch environments +[tool.hatch.envs.dev] +features = [ + 'dev', +] + +# Config for hatch test +[tool.hatch.envs.hatch-test] +features = [ + 'test', +] + +[[tool.hatch.envs.hatch-test.matrix]] +python = ["3.9", "3.10", '3.11', '3.12'] diff --git a/tests/test_aligner.py b/tests/test_aligner.py index 2dc2f7d..82e212b 100644 --- a/tests/test_aligner.py +++ b/tests/test_aligner.py @@ -1,3 +1,4 @@ +import json import os import unittest from datetime import date @@ -7,6 +8,7 @@ from shapely import from_wkt from shapely.geometry import Polygon from shapely.geometry import shape +from shapely.predicates import equals from brdr.aligner import Aligner from brdr.constants import FORMULA_FIELD_NAME @@ -17,7 +19,7 @@ from brdr.grb import ( GRBActualLoader, GRBFiscalParcelLoader, - get_geoms_affected_by_grb_change, + get_affected_by_grb_change, ) from brdr.loader import GeoJsonLoader, DictLoader from brdr.typings import FeatureCollection, ProcessResult @@ -240,7 +242,7 @@ def test_all_od_strategies(self): od_strategy=od_strategy, threshold_overlap_percentage=50, ) - self.assertEqual(len(process_result["theme_id_1"][relevant_distance]), 6) + self.assertEqual(len(process_result["theme_id_1"][relevant_distance]), 7) def test_process_interior_ring(self): thematic_dict = { @@ -269,8 +271,8 @@ def test_process_interior_ring(self): self.assertEqual(len(result_dict), len(thematic_dict)) def test_process_circle(self): - # TODO geometry = Point(0, 0).buffer(3) + # geometry = MultiPolygon([geometry]) thematic_dict = {"key": geometry} self.sample_aligner.load_thematic_data(DictLoader(thematic_dict)) # LOAD REFERENCE DICTIONARY @@ -336,7 +338,7 @@ def test_get_reference_as_geojson(self): self.sample_aligner.get_input_as_geojson() def test_fully_aligned_input(self): - aligned_shape = from_wkt("POLYGON ((0 0, 0 9, 5 10, 10 0, 0 0))") + aligned_shape = from_wkt("MULTIPOLYGON (((0 0, 0 9, 5 10, 10 0, 0 0)))") loader = DictLoader({"theme_id_1": aligned_shape}) self.sample_aligner.load_thematic_data( DictLoader({"theme_id_1": aligned_shape}) @@ -344,14 +346,12 @@ def test_fully_aligned_input(self): self.sample_aligner.load_reference_data(DictLoader({"ref_id_1": aligned_shape})) relevant_distance = 1 result = self.sample_aligner.process(relevant_distance=relevant_distance) - assert result["theme_id_1"][relevant_distance].get("result") == aligned_shape - assert result["theme_id_1"][relevant_distance].get("result_diff") == Polygon() - assert ( - result["theme_id_1"][relevant_distance].get("result_diff_min") == Polygon() - ) - assert ( - result["theme_id_1"][relevant_distance].get("result_diff_plus") == Polygon() + assert equals( + result["theme_id_1"][relevant_distance].get("result"), aligned_shape ) + assert result["theme_id_1"][relevant_distance].get("result_diff").is_empty + assert result["theme_id_1"][relevant_distance].get("result_diff_min").is_empty + assert result["theme_id_1"][relevant_distance].get("result_diff_plus").is_empty def test_evaluate(self): thematic_dict = { @@ -378,42 +378,63 @@ def test_evaluate(self): "result" ] thematic_dict_formula[key] = { - FORMULA_FIELD_NAME: base_aligner.get_brdr_formula( - thematic_dict_result[key] + FORMULA_FIELD_NAME: json.dumps( + base_aligner.get_brdr_formula(thematic_dict_result[key]) ) } - aligner_result = Aligner() - aligner_result.load_thematic_data(DictLoader(thematic_dict_result)) - dict_affected, dict_unchanged = get_geoms_affected_by_grb_change( - aligner=aligner_result, + print(key + ": " + thematic_dict_result[key].wkt) + print(key + ": " + str(thematic_dict_formula[key])) + base_aligner_result = Aligner() + base_aligner_result.load_thematic_data(DictLoader(thematic_dict_result)) + affected, unaffected = get_affected_by_grb_change( + dict_thematic=thematic_dict_result, grb_type=GRBType.ADP, date_start=date(2022, 1, 1), date_end=date.today(), one_by_one=False, ) - + if len(affected) == 0: + print("No affected dicts") + exit() + print("Affected_IDs: " + str(affected)) actual_aligner = Aligner() actual_aligner.load_thematic_data( DictLoader( - data_dict=dict_affected, data_dict_properties=thematic_dict_formula + data_dict=thematic_dict_result, + data_dict_properties=thematic_dict_formula, + ) + ) + actual_aligner.load_reference_data( + GRBActualLoader( + grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner ) ) - loader = GRBActualLoader( - grb_type=GRBType.ADP, partition=1000, aligner=actual_aligner + actual_aligner.relevant_distances = np.arange(0, 200, 10, dtype=int) / 100 + dict_evaluated, prop_dictionary = actual_aligner.evaluate( + ids_to_evaluate=affected, base_formula_field=FORMULA_FIELD_NAME ) - actual_aligner.load_reference_data(loader) - series = np.arange(0, 200, 10, dtype=int) / 100 - dict_evaluated, prop_dictionary = actual_aligner.compare( - threshold_area=5, - threshold_percentage=1, - ) fc = get_series_geojson_dict( dict_evaluated, crs=actual_aligner.CRS, id_field=actual_aligner.name_thematic_id, series_prop_dict=prop_dictionary, ) + print(fc["result"]) + fcs = actual_aligner.get_results_as_geojson(formula=True) + + def test_remark_for_poly_multipoly(self): + shape = from_wkt( + "MultiPolygon(((48893.03662109375 214362.93756103515625, 48890.8258056640625 214368.482666015625, 48890.7159423828125 214368.44110107421875, 48887.6488037109375 214367.2845458984375, 48886.3800048828125 214368.68017578125, 48885.1068115234375 214370.08062744140625, 48884.3330078125 214369.782470703125, 48882.563720703125 214369.10064697265625, 48882.1116943359375 214370.1346435546875, 48878.5626220703125 214368.70196533203125, 48877.839111328125 214368.40997314453125, 48877.2352294921875 214369.79376220703125, 48876.7911376953125 214369.60687255859375, 48875.0850830078125 214373.62353515625, 48875.478759765625 214373.8182373046875, 48881.5286865234375 214376.81109619140625, 48885.10546875 214372.36151123046875, 48887.0050048828125 214370.08538818359375, 48888.4698486328125 214368.330078125, 48890.366943359375 214369.2685546875, 48901.0638427734375 214374.56024169921875, 48905.0159912109375 214369.61175537109375, 48904.472900390625 214367.53851318359375, 48893.03662109375 214362.93756103515625)))" + ) + self.sample_aligner.load_thematic_data(DictLoader({"theme_id_1": shape})) + self.sample_aligner.load_reference_data( + GRBActualLoader( + grb_type=GRBType.ADP, partition=1000, aligner=self.sample_aligner + ) + ) + self.sample_aligner.process(relevant_distances=[2]) + assert self.sample_aligner.dict_processresults["theme_id_1"][2]["remark"] != "" def test_fully_aligned_geojson_output(self): aligned_shape = from_wkt( diff --git a/tests/test_examples.py b/tests/test_examples.py index 3bd1f02..1289651 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -191,7 +191,9 @@ def test_example_wanted_changes(self): # Example how to use a series (for histogram) series = np.arange(0, 300, 10, dtype=int) / 100 - dict_series = aligner.process(series, 4, 50) + dict_series = aligner.process( + relevant_distances=series, od_strategy=4, threshold_overlap_percentage=50 + ) resulting_areas = diffs_from_dict_series(dict_series, aligner.dict_thematic) for key in resulting_areas: if len(resulting_areas[key]) == len(series): diff --git a/tests/test_grb.py b/tests/test_grb.py index adc156f..98e911d 100644 --- a/tests/test_grb.py +++ b/tests/test_grb.py @@ -9,7 +9,7 @@ from brdr.grb import ( get_last_version_date, is_grb_changed, - get_geoms_affected_by_grb_change, + get_affected_by_grb_change, GRBSpecificDateParcelLoader, update_to_actual_grb, ) @@ -68,25 +68,25 @@ def test_get_geoms_affected_by_grb_change_outerborder(self): } aligner = Aligner() aligner.load_thematic_data(DictLoader(thematic_dict)) - dict_affected, dict_unchanged = get_geoms_affected_by_grb_change( - aligner=aligner, + affected, unaffected = get_affected_by_grb_change( + dict_thematic=thematic_dict, grb_type=GRBType.ADP, date_start=date.today() - timedelta(days=30), date_end=date.today(), one_by_one=False, border_distance=0, ) - assert len(dict_affected.keys()) > 0 + assert len(affected) > 0 - dict_affected, dict_unchanged = get_geoms_affected_by_grb_change( - aligner=aligner, + affected, unaffected = get_affected_by_grb_change( + dict_thematic=thematic_dict, grb_type=GRBType.ADP, date_start=date.today() - timedelta(days=30), date_end=date.today(), one_by_one=False, border_distance=10, ) - assert len(dict_affected.keys()) == 0 + assert len(affected) == 0 def test_get_geoms_affected_by_grb_change(self): thematic_dict = { @@ -101,23 +101,23 @@ def test_get_geoms_affected_by_grb_change(self): } aligner = Aligner() aligner.load_thematic_data(DictLoader(thematic_dict)) - dict_affected, dict_unchanged = get_geoms_affected_by_grb_change( - aligner=aligner, + affected, unaffected = get_affected_by_grb_change( + dict_thematic=thematic_dict, grb_type=GRBType.ADP, date_start=date.today() - timedelta(days=1), date_end=date.today(), one_by_one=True, ) - assert len(dict_affected.keys()) == 0 + assert len(affected) == 0 - dict_affected, dict_unchanged = get_geoms_affected_by_grb_change( - aligner=aligner, + affected, unaffected = get_affected_by_grb_change( + dict_thematic=thematic_dict, grb_type=GRBType.ADP, date_start=date.today() - timedelta(days=1000), date_end=date.today(), one_by_one=True, ) - assert len(dict_affected.keys()) == 0 + assert len(affected) == 0 thematic_dict2 = { "theme_id_2": from_wkt( "MultiPolygon (((174180.20077791667426936 171966.14649116666987538, " @@ -130,14 +130,14 @@ def test_get_geoms_affected_by_grb_change(self): } aligner2 = Aligner() aligner2.load_thematic_data(DictLoader(thematic_dict2)) - dict_affected, dict_unchanged = get_geoms_affected_by_grb_change( - aligner=aligner2, + affected, unaffected = get_affected_by_grb_change( + dict_thematic=thematic_dict2, grb_type=GRBType.ADP, date_start=date.today() - timedelta(days=1000), date_end=date.today(), one_by_one=True, ) - assert len(dict_affected.keys()) > 0 + assert len(affected) > 0 def test_get_geoms_affected_by_grb_change_bulk(self): thematic_dict = { @@ -152,23 +152,23 @@ def test_get_geoms_affected_by_grb_change_bulk(self): } aligner = Aligner() aligner.load_thematic_data(DictLoader(thematic_dict)) - dict_affected, dict_unchanged = get_geoms_affected_by_grb_change( - aligner=aligner, + affected, unaffected = get_affected_by_grb_change( + dict_thematic=thematic_dict, grb_type=GRBType.ADP, date_start=date.today() - timedelta(days=1), date_end=date.today(), one_by_one=False, ) - assert len(dict_affected.keys()) == 0 + assert len(affected) == 0 - dict_affected, dict_unchanged = get_geoms_affected_by_grb_change( - aligner=aligner, + affected, unaffected = get_affected_by_grb_change( + dict_thematic=thematic_dict, grb_type=GRBType.ADP, date_start=date.today() - timedelta(days=1000), date_end=date.today(), one_by_one=False, ) - assert len(dict_affected.keys()) > 0 + assert len(affected) > 0 def test_grbspecificdateparcelloader(self): thematic_dict = { @@ -218,7 +218,7 @@ def test_update_to_actual_grb(self): }, "properties": { "area": 503.67736346047064, - "brdr_formula": '{"alignment_date": "2024-09-19", "brdr_version": "0.2.1", "reference_source": {"source": "Adpf", "version_date": "2022-01-01"}, "full": true, "reference_features": {"12034A0181/00K000": {"full": true, "area": 503.68, "percentage": 100, "version_date": "2019-08-30"}}, "reference_od": null, "last_version_date": "2019-08-30"}', + "brdr_base_formula": '{"alignment_date": "2024-09-19", "brdr_version": "0.2.1", "reference_source": {"source": "Adpf", "version_date": "2022-01-01"}, "full": true, "reference_features": {"12034A0181/00K000": {"full": true, "area": 503.68, "percentage": 100, "version_date": "2019-08-30"}}, "reference_od": null, "last_version_date": "2019-08-30"}', "nr_calculations": 1, "perimeter": 125.74541473322422, "relevant_distance": 2, @@ -233,7 +233,9 @@ def test_update_to_actual_grb(self): # Update Featurecollection to actual version featurecollection = update_to_actual_grb( - featurecollection_base_result, name_thematic_id + featurecollection_base_result, + name_thematic_id, + base_formula_field="brdr_base_formula", ) # Print results for feature in featurecollection["result"]["features"]: diff --git a/tests/test_integration.py b/tests/test_integration.py index 0a77a56..c5489c0 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -55,7 +55,11 @@ def test_webservice_brdr(self): series = np.arange(0, 61, 1, dtype=float) / 10 - dict_series = aligner.process(series, openbaardomein_strategy, 50) + dict_series = aligner.process( + relevant_distances=series, + od_strategy=openbaardomein_strategy, + threshold_overlap_percentage=50, + ) dict_diffs = diffs_from_dict_series( dict_series, aligner.dict_thematic, DiffMetric.CHANGES_AREA ) diff --git a/tests/test_utils.py b/tests/test_utils.py index 59a06da..0e7e175 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -42,26 +42,26 @@ def test_get_breakpoints_zerostreak_no_zerostreaks(self): def test_multipolygons_to_singles_empty_dict(self): data = {} - result = multipolygons_to_singles(data) + result, dict_multi_as_single = multipolygons_to_singles(data) self.assertEqual(result, {}) def test_multipolygons_to_singles_with_point(self): geometry1 = shapely.geometry.Polygon([(0, 0), (1, 0), (1, 1), (0, 1)]) geometry2 = shapely.geometry.Point(0, 0) data = {"test_id1": geometry1, "test_id2": geometry2} - result = multipolygons_to_singles(data) + result, dict_multi_as_single = multipolygons_to_singles(data) self.assertEqual(result, {"test_id1": geometry1}) def test_multipolygons_to_singles_single_polygon(self): geometry = shapely.geometry.Polygon([(0, 0), (1, 0), (1, 1), (0, 1)]) data = {"test_id": geometry} - result = multipolygons_to_singles(data) + result, dict_multi_as_single = multipolygons_to_singles(data) self.assertEqual(result, data) def test_multipolygons_to_singles_multipolygon_single_poly(self): geometry = shapely.geometry.Polygon([(0, 0), (1, 0), (1, 1), (0, 1)]) data = {"test_id": shapely.geometry.MultiPolygon([geometry])} - result = multipolygons_to_singles(data) + result, dict_multi_as_single = multipolygons_to_singles(data) self.assertEqual(result, {"test_id": geometry}) def test_polygonize_reference_data_no_overlap(self): @@ -169,6 +169,7 @@ def test_merge_process_results(self): key_1 = "key" + MULTI_SINGLE_ID_SEPARATOR + "1" key_2 = "key" + MULTI_SINGLE_ID_SEPARATOR + "2" key_3 = "key_3" + dict_multi_as_single = {key_1: "key", key_2: "key"} process_result_1 = ProcessResult() process_result_1["result"] = Polygon([(0, 0), (10, 0), (10, 10), (0, 10)]) process_result_2 = ProcessResult() @@ -180,5 +181,7 @@ def test_merge_process_results(self): key_2: {0: process_result_2}, key_3: {0: process_result_3}, } - merged_testdict = merge_process_results(testdict) + merged_testdict = merge_process_results( + result_dict=testdict, dict_multi_as_single=dict_multi_as_single + ) assert len(merged_testdict.keys()) == 2