From 1f2cc55748e6eb7e5fdaf423273c65b975c903a2 Mon Sep 17 00:00:00 2001 From: David de Meij Date: Thu, 8 Sep 2022 14:40:04 +0200 Subject: [PATCH 01/13] Add check on missing sift matches and check if width or height in coordinates are smaller than zero (and exclude those tiles if so). --- s2p/__init__.py | 16 ++++++++++++++++ s2p/initialization.py | 3 +++ s2p/pointing_accuracy.py | 3 +++ 3 files changed, 22 insertions(+) diff --git a/s2p/__init__.py b/s2p/__init__.py index 80dce4e1..ddf83d02 100644 --- a/s2p/__init__.py +++ b/s2p/__init__.py @@ -46,6 +46,21 @@ from s2p import visualisation +def check_missing_sift(tiles_pairs): + missing_sift = [] + with open(os.path.join(cfg["out_dir"], "missing_sift.txt"), "w") as f: + for tile, i in tiles_pairs: + out_dir = os.path.join(tile['dir'], 'pair_{}'.format(i)) + path = os.path.join(out_dir, 'sift_matches.txt') + if not os.path.exists(path): + missing_sift.append(path) + f.write(path + "\n") + if len(missing_sift) > 0: + print(" --- ") + print(f"WARNING: missing {len(missing_sift)}/{len(tiles_pairs)} " + "SIFT matches, this may deteriorate output quality") + print(" --- ") + def pointing_correction(tile, i): """ Compute the translation that corrects the pointing error on a pair of tiles. @@ -576,6 +591,7 @@ def main(user_cfg, start_from=0): print('1) correcting pointing locally...') parallel.launch_calls(pointing_correction, tiles_pairs, nb_workers, timeout=timeout) + check_missing_sift(tiles_pairs) # global-pointing step: if start_from <= 2: diff --git a/s2p/initialization.py b/s2p/initialization.py index 392dc093..16391934 100644 --- a/s2p/initialization.py +++ b/s2p/initialization.py @@ -298,6 +298,9 @@ def is_this_tile_useful(x, y, w, h, images_sizes): mask (np.array): tile validity mask. Set to None if the tile is discarded """ # check if the tile is partly contained in at least one other image + if w <= 0 or h <= 0: + return False, None + rpc = cfg['images'][0]['rpcm'] for img, size in zip(cfg['images'][1:], images_sizes[1:]): coords = rpc_utils.corresponding_roi(rpc, img['rpcm'], x, y, w, h) diff --git a/s2p/pointing_accuracy.py b/s2p/pointing_accuracy.py index cca73149..37ce2450 100644 --- a/s2p/pointing_accuracy.py +++ b/s2p/pointing_accuracy.py @@ -121,6 +121,9 @@ def compute_correction(img1, img2, rpc1, rpc2, x, y, w, h, order to correct the pointing error, and the list of sift matches used to compute this correction. """ + if w <= 0 or h <= 0: + raise ValueError(f"width or height <= 0 for:\n{img1}\n{img2}\nx={x}, y={y}, w={w}, h={h}. Try a different" + f"tilesize or different ROI.") m = sift.matches_on_rpc_roi(img1, img2, rpc1, rpc2, x, y, w, h, method, sift_thresh, epipolar_threshold) From db65ef97301a22e1e609982505462941afde28a8 Mon Sep 17 00:00:00 2001 From: David de Meij Date: Thu, 8 Sep 2022 14:42:09 +0200 Subject: [PATCH 02/13] bump to v1.3.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e2ccbcef..bc40aaab 100644 --- a/setup.py +++ b/setup.py @@ -60,7 +60,7 @@ def finalize_options(self): } setup(name="s2p", - version="1.3", + version="1.3.1", description="Satellite Stereo Pipeline.", long_description=readme(), long_description_content_type='text/markdown', From 949fb1a5dbc705633f1f2fb17d79bbe6e0ac4a26 Mon Sep 17 00:00:00 2001 From: David de Meij Date: Thu, 8 Sep 2022 17:40:42 +0200 Subject: [PATCH 03/13] Fix visualization sift points --- s2p/visualisation.py | 48 +++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/s2p/visualisation.py b/s2p/visualisation.py index 8dcd261e..e77941a5 100644 --- a/s2p/visualisation.py +++ b/s2p/visualisation.py @@ -8,6 +8,7 @@ from s2p import common from s2p import rpc_utils +from rasterio.windows import Window def plot_line(im, x1, y1, x2, y2, colour): """ @@ -48,41 +49,42 @@ def plot_line(im, x1, y1, x2, y2, colour): return im -def plot_matches_low_level(img1, img2, matches, outfile): +def plot_matches_low_level(crop1, crop2, matches, outfile): """ Displays two images side by side with matches highlighted Args: - img1, img2 (np.array): two input images + crop1, crop2 (np.array): two input images matches: 2D numpy array of size 4xN containing a list of matches (a list of pairs of points, each pair being represented by x1, y1, x2, y2) outfile (str): path where to write the resulting image, to be displayed """ # transform single channel to 3-channels - if img1.ndim < 3: - img1 = np.dstack([img1] * 3) - if img2.ndim < 3: - img2= np.dstack([img2] * 3) + if crop1.shape[0] < 3: + crop1 = np.concatenate([crop1] * 3, axis=0) + if crop2.shape[0] < 3: + crop2 = np.concatenate([crop2] * 3, axis=0) # if images have more than 3 channels, keep only the first 3 - if img1.shape[2] > 3: - img1 = img1[:, :, 0:3] - if img2.shape[2] > 3: - img2 = img2[:, :, 0:3] + if crop1.shape[0] > 3: + crop1 = crop1[:3, :, :] + if crop2.shape[0] > 3: + crop2 = crop2[:3, :, :] # build the output image - h1, w1 = img1.shape[:2] - h2, w2 = img2.shape[:2] + h1, w1 = crop1.shape[1:] + h2, w2 = crop2.shape[1:] w = w1 + w2 h = max(h1, h2) - out = np.zeros((h, w, 3), np.uint8) - out[:h1, :w1] = img1 - out[:h2, w1:w] = img2 + out = np.zeros((3, h, w), np.uint8) + out[:, :h1, :w1] = crop1 + out[:, :h2, w1:w] = crop2 + out = np.transpose(out, [1, 2, 0]) # define colors, according to min/max intensity values - out_min = min(np.nanmin(img1), np.nanmin(img2)) - out_max = max(np.nanmax(img1), np.nanmax(img2)) + out_min = min(np.nanmin(crop1), np.nanmin(crop2)) + out_max = max(np.nanmax(crop1), np.nanmax(crop2)) green = [out_min, out_max, out_min] blue = [out_min, out_min, out_max] @@ -105,7 +107,7 @@ def plot_matches_low_level(img1, img2, matches, outfile): common.rasterio_write(outfile, out) -def plot_matches(im1, im2, rpc1, rpc2, matches, outfile, x, y, w, h): +def plot_matches(img1, img2, rpc1, rpc2, matches, outfile, x, y, w, h): """ Plot keypoint matches on images corresponding ROIs. @@ -129,11 +131,11 @@ def plot_matches(im1, im2, rpc1, rpc2, matches, outfile, x, y, w, h): x1, y1, w1, h1 = x, y, w, h x2, y2, w2, h2 = map(int, rpc_utils.corresponding_roi(rpc1, rpc2, x1, y1, w1, h1)) - # do the crops - with rasterio.open(im1, "r") as f: - crop1 = f.read(window=((y1, y1 + h1), (x1, x1 + w1))) - with rasterio.open(im2, "r") as f: - crop2 = f.read(window=((y2, y2 + h2), (x2, x2 + w2))) + # read the crops + with rasterio.open(img1, "r") as f: + crop1 = f.read(window=Window(x1, y1, w1, h1)) + with rasterio.open(img2, "r") as f: + crop2 = f.read(window=Window(x2, y2, w2, h2)) crop1 = common.linear_stretching_and_quantization_8bit(crop1) crop2 = common.linear_stretching_and_quantization_8bit(crop2) From e6cae7abdeed9c103b3d32f4a28462b18e96e5b2 Mon Sep 17 00:00:00 2001 From: David de Meij Date: Thu, 8 Sep 2022 17:41:39 +0200 Subject: [PATCH 04/13] bump to v1.3.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bc40aaab..6d5c9ffe 100644 --- a/setup.py +++ b/setup.py @@ -60,7 +60,7 @@ def finalize_options(self): } setup(name="s2p", - version="1.3.1", + version="1.3.2", description="Satellite Stereo Pipeline.", long_description=readme(), long_description_content_type='text/markdown', From 562f85cf4d64afa0a990beb92ae47fbd88c06922 Mon Sep 17 00:00:00 2001 From: David de Meij Date: Thu, 8 Sep 2022 17:57:14 +0200 Subject: [PATCH 05/13] Better default for nb_stereo_workers --- s2p/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/s2p/__init__.py b/s2p/__init__.py index ddf83d02..1eb37057 100644 --- a/s2p/__init__.py +++ b/s2p/__init__.py @@ -604,17 +604,19 @@ def main(user_cfg, start_from=0): print('3) rectifying tiles...') parallel.launch_calls(rectification_pair, tiles_pairs, nb_workers, timeout=timeout) - tiles = [t for t in tiles if t is not None] + # matching step: if start_from <= 4: - print('4) running stereo matching...') if cfg['max_processes_stereo_matching'] is not None: nb_workers_stereo = cfg['max_processes_stereo_matching'] else: - # Set the number of stereo workers to 2/3 of the number of cores by default - nb_workers_stereo = min(1, int(2 * (nb_workers / 3))) + # Set the number of stereo workers to 2/3 of the number of cores by default, divided + # by a certain amount depending on the tile_size this should be a generally safe number of workers. + divider = (cfg['tile_size'] / 800.0) * (cfg['tilesize'] / 800.0) + nb_workers_stereo = min(nb_workers, max(1, int((2 * (nb_workers / 3)) / divider))) try: + print(f'4) running stereo matching using {nb_workers_stereo} workers...') parallel.launch_calls(stereo_matching, tiles_pairs, nb_workers_stereo, timeout=timeout) except subprocess.CalledProcessError as e: From 9530d0de0e788e3d1f74768d110431a5c3221b9e Mon Sep 17 00:00:00 2001 From: David de Meij Date: Sat, 10 Sep 2022 16:06:53 +0200 Subject: [PATCH 06/13] fix typo --- s2p/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s2p/__init__.py b/s2p/__init__.py index 1eb37057..dced523b 100644 --- a/s2p/__init__.py +++ b/s2p/__init__.py @@ -613,7 +613,7 @@ def main(user_cfg, start_from=0): else: # Set the number of stereo workers to 2/3 of the number of cores by default, divided # by a certain amount depending on the tile_size this should be a generally safe number of workers. - divider = (cfg['tile_size'] / 800.0) * (cfg['tilesize'] / 800.0) + divider = (cfg['tile_size'] / 800.0) * (cfg['tile_size'] / 800.0) nb_workers_stereo = min(nb_workers, max(1, int((2 * (nb_workers / 3)) / divider))) try: print(f'4) running stereo matching using {nb_workers_stereo} workers...') From d00191455414f12a88744eeb02ddaf6e9ad97fa8 Mon Sep 17 00:00:00 2001 From: David de Meij Date: Thu, 15 Sep 2022 11:47:26 +0200 Subject: [PATCH 07/13] Use less stereo workers by default to avoid OOM errors. --- s2p/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s2p/__init__.py b/s2p/__init__.py index dced523b..b79c86ec 100644 --- a/s2p/__init__.py +++ b/s2p/__init__.py @@ -614,7 +614,7 @@ def main(user_cfg, start_from=0): # Set the number of stereo workers to 2/3 of the number of cores by default, divided # by a certain amount depending on the tile_size this should be a generally safe number of workers. divider = (cfg['tile_size'] / 800.0) * (cfg['tile_size'] / 800.0) - nb_workers_stereo = min(nb_workers, max(1, int((2 * (nb_workers / 3)) / divider))) + nb_workers_stereo = min(nb_workers, max(1, int(nb_workers / 2.0) / divider)) try: print(f'4) running stereo matching using {nb_workers_stereo} workers...') parallel.launch_calls(stereo_matching, tiles_pairs, nb_workers_stereo, From eff61bea148b4c5dc2f19741b82ad5e6d46f7300 Mon Sep 17 00:00:00 2001 From: David de Meij Date: Thu, 15 Sep 2022 15:23:31 +0200 Subject: [PATCH 08/13] Make sure nb_workers_stereo is int --- s2p/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s2p/__init__.py b/s2p/__init__.py index b79c86ec..e0f226aa 100644 --- a/s2p/__init__.py +++ b/s2p/__init__.py @@ -614,7 +614,7 @@ def main(user_cfg, start_from=0): # Set the number of stereo workers to 2/3 of the number of cores by default, divided # by a certain amount depending on the tile_size this should be a generally safe number of workers. divider = (cfg['tile_size'] / 800.0) * (cfg['tile_size'] / 800.0) - nb_workers_stereo = min(nb_workers, max(1, int(nb_workers / 2.0) / divider)) + nb_workers_stereo = int(min(nb_workers, max(1, int(nb_workers / 2.0) / divider))) try: print(f'4) running stereo matching using {nb_workers_stereo} workers...') parallel.launch_calls(stereo_matching, tiles_pairs, nb_workers_stereo, From f70d4bb91b4a0da96976a037bfe4f45c1bdfdaf4 Mon Sep 17 00:00:00 2001 From: David de Meij Date: Mon, 19 Sep 2022 16:27:38 +0200 Subject: [PATCH 09/13] Better estimate the appropriate number of stereo workers --- s2p/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/s2p/__init__.py b/s2p/__init__.py index e0f226aa..f3bf3b87 100644 --- a/s2p/__init__.py +++ b/s2p/__init__.py @@ -611,10 +611,12 @@ def main(user_cfg, start_from=0): if cfg['max_processes_stereo_matching'] is not None: nb_workers_stereo = cfg['max_processes_stereo_matching'] else: - # Set the number of stereo workers to 2/3 of the number of cores by default, divided - # by a certain amount depending on the tile_size this should be a generally safe number of workers. + # Set the number of stereo workers to the number of workers divided + # by a certain amount depending on the tile_size and number of tiles + # this should be a generally safe number of workers. divider = (cfg['tile_size'] / 800.0) * (cfg['tile_size'] / 800.0) - nb_workers_stereo = int(min(nb_workers, max(1, int(nb_workers / 2.0) / divider))) + divider *= (len(tiles_pairs) / 500.0) + nb_workers_stereo = int(min(nb_workers, max(1, int(nb_workers / divider)))) try: print(f'4) running stereo matching using {nb_workers_stereo} workers...') parallel.launch_calls(stereo_matching, tiles_pairs, nb_workers_stereo, From 8c7fbbf81abd0644735c73b13c48e9e1f2f60bc7 Mon Sep 17 00:00:00 2001 From: David de Meij Date: Mon, 19 Sep 2022 16:28:15 +0200 Subject: [PATCH 10/13] Avoid plotting to many matches in plot_matches_low_level --- s2p/visualisation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/s2p/visualisation.py b/s2p/visualisation.py index e77941a5..7c6b1c98 100644 --- a/s2p/visualisation.py +++ b/s2p/visualisation.py @@ -49,7 +49,7 @@ def plot_line(im, x1, y1, x2, y2, colour): return im -def plot_matches_low_level(crop1, crop2, matches, outfile): +def plot_matches_low_level(crop1, crop2, matches, outfile, max_matches=100): """ Displays two images side by side with matches highlighted @@ -88,8 +88,8 @@ def plot_matches_low_level(crop1, crop2, matches, outfile): green = [out_min, out_max, out_min] blue = [out_min, out_min, out_max] - # plot the matches - for i in range(len(matches)): + # plot the matches (not more than max_matches) + for i in range(min(max_matches, len(matches))): x1 = matches[i, 0] y1 = matches[i, 1] x2 = matches[i, 2] + w1 From 9a42413d7fccc04a9eaa769d823dbf1b93820e31 Mon Sep 17 00:00:00 2001 From: David de Meij Date: Mon, 19 Sep 2022 16:28:52 +0200 Subject: [PATCH 11/13] Bump to v1.3.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6d5c9ffe..84383cc8 100644 --- a/setup.py +++ b/setup.py @@ -60,7 +60,7 @@ def finalize_options(self): } setup(name="s2p", - version="1.3.2", + version="1.3.3", description="Satellite Stereo Pipeline.", long_description=readme(), long_description_content_type='text/markdown', From b4730aae5af3c1d80cec848d6be5d16cf55aa4bc Mon Sep 17 00:00:00 2001 From: David de Meij Date: Mon, 19 Sep 2022 19:15:01 +0200 Subject: [PATCH 12/13] Use better number of stereo workers --- s2p/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/s2p/__init__.py b/s2p/__init__.py index f3bf3b87..7453fab5 100644 --- a/s2p/__init__.py +++ b/s2p/__init__.py @@ -565,6 +565,7 @@ def main(user_cfg, start_from=0): nb_workers = multiprocessing.cpu_count() # nb of available cores if cfg['max_processes'] is not None: nb_workers = cfg['max_processes'] + print(f"Running s2p using {nb_workers} workers.") tw, th = initialization.adjust_tile_size() tiles_txt = os.path.join(cfg['out_dir'], 'tiles.txt') @@ -610,13 +611,15 @@ def main(user_cfg, start_from=0): if start_from <= 4: if cfg['max_processes_stereo_matching'] is not None: nb_workers_stereo = cfg['max_processes_stereo_matching'] - else: + elif cfg['matching_algorithm'] == 'mgm_multi': # Set the number of stereo workers to the number of workers divided # by a certain amount depending on the tile_size and number of tiles # this should be a generally safe number of workers. - divider = (cfg['tile_size'] / 800.0) * (cfg['tile_size'] / 800.0) + divider = 2 * (cfg['tile_size'] / 800.0) * (cfg['tile_size'] / 800.0) divider *= (len(tiles_pairs) / 500.0) nb_workers_stereo = int(min(nb_workers, max(1, int(nb_workers / divider)))) + else: + nb_workers_stereo = int((2/3) * nb_workers) try: print(f'4) running stereo matching using {nb_workers_stereo} workers...') parallel.launch_calls(stereo_matching, tiles_pairs, nb_workers_stereo, @@ -657,7 +660,7 @@ def main(user_cfg, start_from=0): # local-dsm-rasterization step: if start_from <= 6: - print('computing DSM by tile...') + print('6) computing DSM by tile...') parallel.launch_calls(plys_to_dsm, tiles, nb_workers, timeout=timeout) # global-dsm-rasterization step: From 9eb9a6bd0543e480d7b5c771c6f83a42b1aebba6 Mon Sep 17 00:00:00 2001 From: David de Meij Date: Mon, 19 Sep 2022 19:18:25 +0200 Subject: [PATCH 13/13] Use better number of stereo workers --- s2p/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/s2p/__init__.py b/s2p/__init__.py index 7453fab5..9639c6c8 100644 --- a/s2p/__init__.py +++ b/s2p/__init__.py @@ -611,15 +611,17 @@ def main(user_cfg, start_from=0): if start_from <= 4: if cfg['max_processes_stereo_matching'] is not None: nb_workers_stereo = cfg['max_processes_stereo_matching'] - elif cfg['matching_algorithm'] == 'mgm_multi': + else: # Set the number of stereo workers to the number of workers divided # by a certain amount depending on the tile_size and number of tiles # this should be a generally safe number of workers. divider = 2 * (cfg['tile_size'] / 800.0) * (cfg['tile_size'] / 800.0) divider *= (len(tiles_pairs) / 500.0) - nb_workers_stereo = int(min(nb_workers, max(1, int(nb_workers / divider)))) - else: - nb_workers_stereo = int((2/3) * nb_workers) + if cfg['matching_algorithm'] == 'mgm_multi': + nb_workers_stereo = int(min(nb_workers, max(1, int(nb_workers / divider)))) + else: + # For non mgm_multi don't use less than 2/3 of the workers (much less RAM intensive) + nb_workers_stereo = int(min(nb_workers, max(((2 / 3) * nb_workers), nb_workers / divider))) try: print(f'4) running stereo matching using {nb_workers_stereo} workers...') parallel.launch_calls(stereo_matching, tiles_pairs, nb_workers_stereo,