diff --git a/geo_autoRIFT/autoRIFT/autoRIFT.py b/geo_autoRIFT/autoRIFT/autoRIFT.py index 976a29b..78d3ff1 100755 --- a/geo_autoRIFT/autoRIFT/autoRIFT.py +++ b/geo_autoRIFT/autoRIFT/autoRIFT.py @@ -56,8 +56,7 @@ def preprocess_filt_wal(self): # import scipy.io as sio - if self.zeroMask is not None: - self.zeroMask = (self.I1 == 0) + self.zeroMask = (self.I1 == 0) kernel = np.ones((self.WallisFilterWidth,self.WallisFilterWidth), dtype=np.float32) @@ -115,8 +114,6 @@ def preprocess_filt_hps(self): import numpy as np - if self.zeroMask is not None: - self.zeroMask = (self.I1 == 0) kernel = -np.ones((self.WallisFilterWidth,self.WallisFilterWidth), dtype=np.float32) @@ -140,8 +137,8 @@ def preprocess_db(self): import numpy as np - if self.zeroMask is not None: - self.zeroMask = (self.I1 == 0) + + self.zeroMask = (self.I1 == 0) # pdb.set_trace() @@ -161,9 +158,7 @@ def preprocess_filt_sob(self): - if self.zeroMask is not None: - self.zeroMask = (self.I1 == 0) - + sobelx = cv2.getDerivKernels(1,0,self.WallisFilterWidth) kernelx = np.outer(sobelx[0],sobelx[1]) @@ -194,8 +189,8 @@ def preprocess_filt_lap(self): import numpy as np - if self.zeroMask is not None: - self.zeroMask = (self.I1 == 0) + + self.zeroMask = (self.I1 == 0) self.I1 = 20.0 * np.log10(self.I1) self.I1 = cv2.Laplacian(self.I1,-1,ksize=self.WallisFilterWidth,borderType=cv2.BORDER_CONSTANT) @@ -216,19 +211,27 @@ def uniform_data_type(self): import numpy as np if self.DataType == 0: -# validData = np.logical_not(np.isnan(self.I1)) - validData = np.isfinite(self.I1) - S1 = np.std(self.I1[validData])*np.sqrt(self.I1[validData].size/(self.I1[validData].size-1.0)) - M1 = np.mean(self.I1[validData]) + if self.zeroMask is not None: + # validData = np.logical_not(np.isnan(self.I1)) + validData = np.isfinite(self.I1) + temp = self.I1[validData] + else: + temp = self.I1 + S1 = np.std(temp)*np.sqrt(temp.size/(temp.size-1.0)) + M1 = np.mean(temp) self.I1 = (self.I1 - (M1 - 3*S1)) / (6*S1) * (2**8 - 0) # self.I1[np.logical_not(np.isfinite(self.I1))] = 0 self.I1 = np.round(np.clip(self.I1, 0, 255)).astype(np.uint8) -# validData = np.logical_not(np.isnan(self.I2)) - validData = np.isfinite(self.I2) - S2 = np.std(self.I2[validData])*np.sqrt(self.I2[validData].size/(self.I2[validData].size-1.0)) - M2 = np.mean(self.I2[validData]) + if self.zeroMask is not None: + # validData = np.logical_not(np.isnan(self.I2)) + validData = np.isfinite(self.I2) + temp = self.I2[validData] + else: + temp = self.I2 + S2 = np.std(temp)*np.sqrt(temp.size/(temp.size-1.0)) + M2 = np.mean(temp) self.I2 = (self.I2 - (M2 - 3*S2)) / (6*S2) * (2**8 - 0) # self.I2[np.logical_not(np.isfinite(self.I2))] = 0 @@ -240,8 +243,11 @@ def uniform_data_type(self): self.zeroMask = None elif self.DataType == 1: - self.I1[np.logical_not(np.isfinite(self.I1))] = 0 - self.I2[np.logical_not(np.isfinite(self.I2))] = 0 + + if self.zeroMask is not None: + self.I1[np.logical_not(np.isfinite(self.I1))] = 0 + self.I2[np.logical_not(np.isfinite(self.I2))] = 0 + self.I1 = self.I1.astype(np.float32) self.I2 = self.I2.astype(np.float32) @@ -409,9 +415,9 @@ def autorift(self): # pdb.set_trace() if self.I1.dtype == np.uint8: - DxC, DyC = arImgDisp_u(self.I2.copy(), self.I1.copy(), xGrid0C.copy(), yGrid0C.copy(), ChipSizeXC, ChipSizeYC, SearchLimitX0C.copy(), SearchLimitY0C.copy(), Dx0C.copy(), Dy0C.copy(), SubPixFlag, overSampleRatio) + DxC, DyC = arImgDisp_u(self.I2.copy(), self.I1.copy(), xGrid0C.copy(), yGrid0C.copy(), ChipSizeXC, ChipSizeYC, SearchLimitX0C.copy(), SearchLimitY0C.copy(), Dx0C.copy(), Dy0C.copy(), SubPixFlag, overSampleRatio, self.MultiThread) elif self.I1.dtype == np.float32: - DxC, DyC = arImgDisp_s(self.I2.copy(), self.I1.copy(), xGrid0C.copy(), yGrid0C.copy(), ChipSizeXC, ChipSizeYC, SearchLimitX0C.copy(), SearchLimitY0C.copy(), Dx0C.copy(), Dy0C.copy(), SubPixFlag, overSampleRatio) + DxC, DyC = arImgDisp_s(self.I2.copy(), self.I1.copy(), xGrid0C.copy(), yGrid0C.copy(), ChipSizeXC, ChipSizeYC, SearchLimitX0C.copy(), SearchLimitY0C.copy(), Dx0C.copy(), Dy0C.copy(), SubPixFlag, overSampleRatio, self.MultiThread) else: sys.exit('invalid data type for the image pair which must be unsigned integer 8 or 32-bit float') @@ -459,9 +465,9 @@ def autorift(self): ChipSizeYF = np.float32(np.round(ChipSizeXF*self.ScaleChipSizeY/2)*2) # pdb.set_trace() if self.I1.dtype == np.uint8: - DxF, DyF = arImgDisp_u(self.I2.copy(), self.I1.copy(), xGrid0.copy(), yGrid0.copy(), ChipSizeXF, ChipSizeYF, SearchLimitX0.copy(), SearchLimitY0.copy(), Dx00.copy(), Dy00.copy(), SubPixFlag, overSampleRatio) + DxF, DyF = arImgDisp_u(self.I2.copy(), self.I1.copy(), xGrid0.copy(), yGrid0.copy(), ChipSizeXF, ChipSizeYF, SearchLimitX0.copy(), SearchLimitY0.copy(), Dx00.copy(), Dy00.copy(), SubPixFlag, overSampleRatio, self.MultiThread) elif self.I1.dtype == np.float32: - DxF, DyF = arImgDisp_s(self.I2.copy(), self.I1.copy(), xGrid0.copy(), yGrid0.copy(), ChipSizeXF, ChipSizeYF, SearchLimitX0.copy(), SearchLimitY0.copy(), Dx00.copy(), Dy00.copy(), SubPixFlag, overSampleRatio) + DxF, DyF = arImgDisp_s(self.I2.copy(), self.I1.copy(), xGrid0.copy(), yGrid0.copy(), ChipSizeXF, ChipSizeYF, SearchLimitX0.copy(), SearchLimitY0.copy(), Dx00.copy(), Dy00.copy(), SubPixFlag, overSampleRatio, self.MultiThread) else: sys.exit('invalid data type for the image pair which must be unsigned integer 8 or 32-bit float') @@ -647,6 +653,7 @@ def __init__(self): self.CoarseCorCutoff = 0.01 self.OverSampleRatio = 16 self.DataType = 0 + self.MultiThread = 0 @@ -659,11 +666,188 @@ def __init__(self): self._autoriftcore = None +var_dict = {} + +def initializer(I1, I2, xGrid, yGrid, SearchLimitX, SearchLimitY, ChipSizeX, ChipSizeY, Dx0, Dy0): + var_dict['I1'] = I1 + var_dict['I2'] = I2 + var_dict['xGrid'] = xGrid + var_dict['yGrid'] = yGrid + var_dict['SearchLimitX'] = SearchLimitX + var_dict['SearchLimitY'] = SearchLimitY + var_dict['ChipSizeX'] = ChipSizeX + var_dict['ChipSizeY'] = ChipSizeY + var_dict['Dx0'] = Dx0 + var_dict['Dy0'] = Dy0 + + + +def unpacking_loop_u(tup): + + import numpy as np + from . import autoriftcore + + core = AUTO_RIFT_CORE() + if core._autoriftcore is not None: + autoriftcore.destroyAutoRiftCore_Py(core._autoriftcore) + + core._autoriftcore = autoriftcore.createAutoRiftCore_Py() + + k, chunkInds, SubPixFlag, oversample, in_shape, I_shape = tup + + I1 = np.frombuffer(var_dict['I1'],dtype=np.uint8).reshape(I_shape) + I2 = np.frombuffer(var_dict['I2'],dtype=np.uint8).reshape(I_shape) + xGrid = np.frombuffer(var_dict['xGrid'],dtype=np.float32).reshape(in_shape) + yGrid = np.frombuffer(var_dict['yGrid'],dtype=np.float32).reshape(in_shape) + SearchLimitX = np.frombuffer(var_dict['SearchLimitX'],dtype=np.float32).reshape(in_shape) + SearchLimitY = np.frombuffer(var_dict['SearchLimitY'],dtype=np.float32).reshape(in_shape) + ChipSizeX = np.frombuffer(var_dict['ChipSizeX'],dtype=np.float32).reshape(in_shape) + ChipSizeY = np.frombuffer(var_dict['ChipSizeY'],dtype=np.float32).reshape(in_shape) + Dx0 = np.frombuffer(var_dict['Dx0'],dtype=np.float32).reshape(in_shape) + Dy0 = np.frombuffer(var_dict['Dy0'],dtype=np.float32).reshape(in_shape) + + Dx = np.empty(chunkInds.shape,dtype=np.float32) + Dx.fill(np.nan) + Dy = Dx.copy() + + +# print(k) +# print(np.min(chunkInds),np.max(chunkInds)) +# print(chunkInds.shape) + + for ind in chunkInds: + + ind1 = np.where(chunkInds == ind)[0][0] + + ii, jj = [v[0] for v in np.unravel_index([ind], in_shape)] + + if (SearchLimitX[ii,jj] == 0) & (SearchLimitY[ii,jj] == 0): + continue + + # remember motion terms Dx and Dy correspond to I1 relative to I2 (reference) + clx = np.floor(ChipSizeX[ii,jj]/2) + ChipRangeX = slice(int(-clx - Dx0[ii,jj] + xGrid[ii,jj]) , int(clx - Dx0[ii,jj] + xGrid[ii,jj])) + cly = np.floor(ChipSizeY[ii,jj]/2) + ChipRangeY = slice(int(-cly - Dy0[ii,jj] + yGrid[ii,jj]) , int(cly - Dy0[ii,jj] + yGrid[ii,jj])) + ChipI = I2[ChipRangeY,ChipRangeX] + + SearchRangeX = slice(int(-clx - SearchLimitX[ii,jj] + xGrid[ii,jj]) , int(clx + SearchLimitX[ii,jj] - 1 + xGrid[ii,jj])) + SearchRangeY = slice(int(-cly - SearchLimitY[ii,jj] + yGrid[ii,jj]) , int(cly + SearchLimitY[ii,jj] - 1 + yGrid[ii,jj])) + RefI = I1[SearchRangeY,SearchRangeX] + + minChipI = np.min(ChipI) + if minChipI < 0: + ChipI = ChipI - minChipI + if np.all(ChipI == ChipI[0,0]): + continue + + minRefI = np.min(RefI) + if minRefI < 0: + RefI = RefI - minRefI + if np.all(RefI == RefI[0,0]): + continue + + + if SubPixFlag: + # call C++ + Dx[ind1], Dy[ind1] = np.float32(autoriftcore.arSubPixDisp_u_Py(core._autoriftcore,ChipI.shape[1],ChipI.shape[0],ChipI.ravel(),RefI.shape[1],RefI.shape[0],RefI.ravel(),oversample)) +# # call Python +# Dx1[ii], Dy1[ii] = arSubPixDisp(ChipI,RefI) + else: + # call C++ + Dx[ind1], Dy[ind1] = np.float32(autoriftcore.arPixDisp_u_Py(core._autoriftcore,ChipI.shape[1],ChipI.shape[0],ChipI.ravel(),RefI.shape[1],RefI.shape[0],RefI.ravel())) +# # call Python +# Dx1[ii], Dy1[ii] = arPixDisp(ChipI,RefI) + return Dx, Dy + + +def unpacking_loop_s(tup): + + import numpy as np + from . import autoriftcore + + core = AUTO_RIFT_CORE() + if core._autoriftcore is not None: + autoriftcore.destroyAutoRiftCore_Py(core._autoriftcore) + + core._autoriftcore = autoriftcore.createAutoRiftCore_Py() + + + k, chunkInds, SubPixFlag, oversample, in_shape, I_shape = tup + + I1 = np.frombuffer(var_dict['I1'],dtype=np.float32).reshape(I_shape) + I2 = np.frombuffer(var_dict['I2'],dtype=np.float32).reshape(I_shape) + xGrid = np.frombuffer(var_dict['xGrid'],dtype=np.float32).reshape(in_shape) + yGrid = np.frombuffer(var_dict['yGrid'],dtype=np.float32).reshape(in_shape) + SearchLimitX = np.frombuffer(var_dict['SearchLimitX'],dtype=np.float32).reshape(in_shape) + SearchLimitY = np.frombuffer(var_dict['SearchLimitY'],dtype=np.float32).reshape(in_shape) + ChipSizeX = np.frombuffer(var_dict['ChipSizeX'],dtype=np.float32).reshape(in_shape) + ChipSizeY = np.frombuffer(var_dict['ChipSizeY'],dtype=np.float32).reshape(in_shape) + Dx0 = np.frombuffer(var_dict['Dx0'],dtype=np.float32).reshape(in_shape) + Dy0 = np.frombuffer(var_dict['Dy0'],dtype=np.float32).reshape(in_shape) + + + Dx = np.empty(chunkInds.shape,dtype=np.float32) + Dx.fill(np.nan) + Dy = Dx.copy() + +# print(k) +# print(np.min(chunkInds),np.max(chunkInds)) +# print(chunkInds.shape) + + for ind in chunkInds: + + ind1 = np.where(chunkInds == ind)[0][0] + + ii, jj = [v[0] for v in np.unravel_index([ind], in_shape)] + + if (SearchLimitX[ii,jj] == 0) & (SearchLimitY[ii,jj] == 0): + continue + + # remember motion terms Dx and Dy correspond to I1 relative to I2 (reference) + clx = np.floor(ChipSizeX[ii,jj]/2) + ChipRangeX = slice(int(-clx - Dx0[ii,jj] + xGrid[ii,jj]) , int(clx - Dx0[ii,jj] + xGrid[ii,jj])) + cly = np.floor(ChipSizeY[ii,jj]/2) + ChipRangeY = slice(int(-cly - Dy0[ii,jj] + yGrid[ii,jj]) , int(cly - Dy0[ii,jj] + yGrid[ii,jj])) + ChipI = I2[ChipRangeY,ChipRangeX] + + SearchRangeX = slice(int(-clx - SearchLimitX[ii,jj] + xGrid[ii,jj]) , int(clx + SearchLimitX[ii,jj] - 1 + xGrid[ii,jj])) + SearchRangeY = slice(int(-cly - SearchLimitY[ii,jj] + yGrid[ii,jj]) , int(cly + SearchLimitY[ii,jj] - 1 + yGrid[ii,jj])) + RefI = I1[SearchRangeY,SearchRangeX] + + minChipI = np.min(ChipI) + if minChipI < 0: + ChipI = ChipI - minChipI + if np.all(ChipI == ChipI[0,0]): + continue + + minRefI = np.min(RefI) + if minRefI < 0: + RefI = RefI - minRefI + if np.all(RefI == RefI[0,0]): + continue + + + if SubPixFlag: + # call C++ + Dx[ind1], Dy[ind1] = np.float32(autoriftcore.arSubPixDisp_s_Py(core._autoriftcore,ChipI.shape[1],ChipI.shape[0],ChipI.ravel(),RefI.shape[1],RefI.shape[0],RefI.ravel(),oversample)) +# # call Python +# Dx1[ii], Dy1[ii] = arSubPixDisp(ChipI,RefI) + else: + # call C++ + Dx[ind1], Dy[ind1] = np.float32(autoriftcore.arPixDisp_s_Py(core._autoriftcore,ChipI.shape[1],ChipI.shape[0],ChipI.ravel(),RefI.shape[1],RefI.shape[0],RefI.ravel())) +# # call Python +# Dx1[ii], Dy1[ii] = arPixDisp(ChipI,RefI) + return Dx, Dy + + + -def arImgDisp_u(I1, I2, xGrid, yGrid, ChipSizeX, ChipSizeY, SearchLimitX, SearchLimitY, Dx0, Dy0, SubPixFlag, oversample): +def arImgDisp_u(I1, I2, xGrid, yGrid, ChipSizeX, ChipSizeY, SearchLimitX, SearchLimitY, Dx0, Dy0, SubPixFlag, oversample, MultiThread): import numpy as np from . import autoriftcore + import multiprocessing as mp core = AUTO_RIFT_CORE() if core._autoriftcore is not None: @@ -756,50 +940,136 @@ def arImgDisp_u(I1, I2, xGrid, yGrid, ChipSizeX, ChipSizeY, SearchLimitX, Search Dx.fill(np.nan) Dy = Dx.copy() - - for jj in range(xGrid.shape[1]): - if np.all(SearchLimitX[:,jj] == 0) & np.all(SearchLimitY[:,jj] == 0): - continue - Dx1 = Dx[:,jj] - Dy1 = Dy[:,jj] - for ii in range(xGrid.shape[0]): - if (SearchLimitX[ii,jj] == 0) & (SearchLimitY[ii,jj] == 0): + if MultiThread == 0: + for jj in range(xGrid.shape[1]): + if np.all(SearchLimitX[:,jj] == 0) & np.all(SearchLimitY[:,jj] == 0): continue - - # remember motion terms Dx and Dy correspond to I1 relative to I2 (reference) - clx = np.floor(ChipSizeX[ii,jj]/2) - ChipRangeX = slice(int(-clx - Dx0[ii,jj] + xGrid[ii,jj]) , int(clx - Dx0[ii,jj] + xGrid[ii,jj])) - cly = np.floor(ChipSizeY[ii,jj]/2) - ChipRangeY = slice(int(-cly - Dy0[ii,jj] + yGrid[ii,jj]) , int(cly - Dy0[ii,jj] + yGrid[ii,jj])) - ChipI = I2[ChipRangeY,ChipRangeX] - - SearchRangeX = slice(int(-clx - SearchLimitX[ii,jj] + xGrid[ii,jj]) , int(clx + SearchLimitX[ii,jj] - 1 + xGrid[ii,jj])) - SearchRangeY = slice(int(-cly - SearchLimitY[ii,jj] + yGrid[ii,jj]) , int(cly + SearchLimitY[ii,jj] - 1 + yGrid[ii,jj])) - RefI = I1[SearchRangeY,SearchRangeX] - - minChipI = np.min(ChipI) - if minChipI < 0: - ChipI = ChipI - minChipI - if np.all(ChipI == ChipI[0,0]): - continue - - minRefI = np.min(RefI) - if minRefI < 0: - RefI = RefI - minRefI - if np.all(RefI == RefI[0,0]): - continue - + Dx1 = Dx[:,jj] + Dy1 = Dy[:,jj] + for ii in range(xGrid.shape[0]): + if (SearchLimitX[ii,jj] == 0) & (SearchLimitY[ii,jj] == 0): + continue + + # remember motion terms Dx and Dy correspond to I1 relative to I2 (reference) + clx = np.floor(ChipSizeX[ii,jj]/2) + ChipRangeX = slice(int(-clx - Dx0[ii,jj] + xGrid[ii,jj]) , int(clx - Dx0[ii,jj] + xGrid[ii,jj])) + cly = np.floor(ChipSizeY[ii,jj]/2) + ChipRangeY = slice(int(-cly - Dy0[ii,jj] + yGrid[ii,jj]) , int(cly - Dy0[ii,jj] + yGrid[ii,jj])) + ChipI = I2[ChipRangeY,ChipRangeX] + + SearchRangeX = slice(int(-clx - SearchLimitX[ii,jj] + xGrid[ii,jj]) , int(clx + SearchLimitX[ii,jj] - 1 + xGrid[ii,jj])) + SearchRangeY = slice(int(-cly - SearchLimitY[ii,jj] + yGrid[ii,jj]) , int(cly + SearchLimitY[ii,jj] - 1 + yGrid[ii,jj])) + RefI = I1[SearchRangeY,SearchRangeX] + + minChipI = np.min(ChipI) + if minChipI < 0: + ChipI = ChipI - minChipI + if np.all(ChipI == ChipI[0,0]): + continue + + minRefI = np.min(RefI) + if minRefI < 0: + RefI = RefI - minRefI + if np.all(RefI == RefI[0,0]): + continue + - if SubPixFlag: - # call C++ - Dx1[ii], Dy1[ii] = np.float32(autoriftcore.arSubPixDisp_u_Py(core._autoriftcore,ChipI.shape[1],ChipI.shape[0],ChipI.ravel(),RefI.shape[1],RefI.shape[0],RefI.ravel(),oversample)) + if SubPixFlag: + # call C++ + Dx1[ii], Dy1[ii] = np.float32(autoriftcore.arSubPixDisp_u_Py(core._autoriftcore,ChipI.shape[1],ChipI.shape[0],ChipI.ravel(),RefI.shape[1],RefI.shape[0],RefI.ravel(),oversample)) # # call Python # Dx1[ii], Dy1[ii] = arSubPixDisp(ChipI,RefI) - else: - # call C++ - Dx1[ii], Dy1[ii] = np.float32(autoriftcore.arPixDisp_u_Py(core._autoriftcore,ChipI.shape[1],ChipI.shape[0],ChipI.ravel(),RefI.shape[1],RefI.shape[0],RefI.ravel())) + else: + # call C++ + Dx1[ii], Dy1[ii] = np.float32(autoriftcore.arPixDisp_u_Py(core._autoriftcore,ChipI.shape[1],ChipI.shape[0],ChipI.ravel(),RefI.shape[1],RefI.shape[0],RefI.ravel())) # # call Python # Dx1[ii], Dy1[ii] = arPixDisp(ChipI,RefI) + else: + # Preparation for parallel + in_shape = xGrid.shape + I_shape = I1.shape + shape_prod = np.asscalar(np.prod(in_shape)) + + # import pdb + # pdb.set_trace() + XI1 = mp.RawArray('b', np.asscalar(np.prod(I_shape))) + XI1_np = np.frombuffer(XI1,dtype=np.uint8).reshape(I_shape) + np.copyto(XI1_np,I1) + del I1 + + XI2 = mp.RawArray('b', np.asscalar(np.prod(I_shape))) + XI2_np = np.frombuffer(XI2,dtype=np.uint8).reshape(I_shape) + np.copyto(XI2_np,I2) + del I2 + + XxGrid = mp.RawArray('f', shape_prod) + XxGrid_np = np.frombuffer(XxGrid,dtype=np.float32).reshape(in_shape) + np.copyto(XxGrid_np,xGrid) + del xGrid + + XyGrid = mp.RawArray('f', shape_prod) + XyGrid_np = np.frombuffer(XyGrid,dtype=np.float32).reshape(in_shape) + np.copyto(XyGrid_np,yGrid) + del yGrid + + XSearchLimitX = mp.RawArray('f', shape_prod) + XSearchLimitX_np = np.frombuffer(XSearchLimitX,dtype=np.float32).reshape(in_shape) + np.copyto(XSearchLimitX_np,SearchLimitX) + + XSearchLimitY = mp.RawArray('f', shape_prod) + XSearchLimitY_np = np.frombuffer(XSearchLimitY,dtype=np.float32).reshape(in_shape) + np.copyto(XSearchLimitY_np,SearchLimitY) + + XChipSizeX = mp.RawArray('f', shape_prod) + XChipSizeX_np = np.frombuffer(XChipSizeX,dtype=np.float32).reshape(in_shape) + np.copyto(XChipSizeX_np,ChipSizeX) + del ChipSizeX + + XChipSizeY = mp.RawArray('f', shape_prod) + XChipSizeY_np = np.frombuffer(XChipSizeY,dtype=np.float32).reshape(in_shape) + np.copyto(XChipSizeY_np,ChipSizeY) + del ChipSizeY + + XDx0 = mp.RawArray('f', shape_prod) + XDx0_np = np.frombuffer(XDx0,dtype=np.float32).reshape(in_shape) + np.copyto(XDx0_np,Dx0) + + XDy0 = mp.RawArray('f', shape_prod) + XDy0_np = np.frombuffer(XDy0,dtype=np.float32).reshape(in_shape) + np.copyto(XDy0_np,Dy0) + # import pdb + # pdb.set_trace() + + +# Nchunks = mp.cpu_count() // 8 * MultiThread + Nchunks = MultiThread + chunkSize = int(np.floor(shape_prod / Nchunks)) + chunkRem = shape_prod - chunkSize * Nchunks + + CHUNKS = [] + + for k in range(Nchunks): +# print(k) + if k == (Nchunks-1): + chunkInds = np.arange(k*chunkSize, (k+1)*chunkSize+chunkRem) + else: + chunkInds = np.arange(k*chunkSize, (k+1)*chunkSize) + CHUNKS.append(chunkInds) +# print(CHUNKS) + + chunk_inputs = [(kk, CHUNKS[kk], SubPixFlag, oversample, in_shape, I_shape) + for kk in range(Nchunks)] + + with mp.Pool(initializer=initializer, initargs=(XI1, XI2, XxGrid, XyGrid, XSearchLimitX, XSearchLimitY, XChipSizeX, XChipSizeY, XDx0, XDy0)) as pool: + Dx, Dy = zip(*pool.map(unpacking_loop_u, chunk_inputs)) + + Dx = np.concatenate(Dx) + Dy = np.concatenate(Dy) + + Dx = np.reshape(Dx, in_shape) + Dy = np.reshape(Dy, in_shape) + + # add back 1) I1 (RefI) relative to I2 (ChipI) initial offset Dx0 and Dy0, and @@ -821,7 +1091,7 @@ def arImgDisp_u(I1, I2, xGrid, yGrid, ChipSizeX, ChipSizeY, SearchLimitX, Search -def arImgDisp_s(I1, I2, xGrid, yGrid, ChipSizeX, ChipSizeY, SearchLimitX, SearchLimitY, Dx0, Dy0, SubPixFlag, oversample): +def arImgDisp_s(I1, I2, xGrid, yGrid, ChipSizeX, ChipSizeY, SearchLimitX, SearchLimitY, Dx0, Dy0, SubPixFlag, oversample, MultiThread): import numpy as np from . import autoriftcore @@ -917,51 +1187,134 @@ def arImgDisp_s(I1, I2, xGrid, yGrid, ChipSizeX, ChipSizeY, SearchLimitX, Search Dx.fill(np.nan) Dy = Dx.copy() - - for jj in range(xGrid.shape[1]): - if np.all(SearchLimitX[:,jj] == 0) & np.all(SearchLimitY[:,jj] == 0): - continue - Dx1 = Dx[:,jj] - Dy1 = Dy[:,jj] - for ii in range(xGrid.shape[0]): - if (SearchLimitX[ii,jj] == 0) & (SearchLimitY[ii,jj] == 0): + if MultiThread == 0: + for jj in range(xGrid.shape[1]): + if np.all(SearchLimitX[:,jj] == 0) & np.all(SearchLimitY[:,jj] == 0): continue - - # remember motion terms Dx and Dy correspond to I1 relative to I2 (reference) - clx = np.floor(ChipSizeX[ii,jj]/2) - ChipRangeX = slice(int(-clx - Dx0[ii,jj] + xGrid[ii,jj]) , int(clx - Dx0[ii,jj] + xGrid[ii,jj])) - cly = np.floor(ChipSizeY[ii,jj]/2) - ChipRangeY = slice(int(-cly - Dy0[ii,jj] + yGrid[ii,jj]) , int(cly - Dy0[ii,jj] + yGrid[ii,jj])) - ChipI = I2[ChipRangeY,ChipRangeX] + Dx1 = Dx[:,jj] + Dy1 = Dy[:,jj] + for ii in range(xGrid.shape[0]): + if (SearchLimitX[ii,jj] == 0) & (SearchLimitY[ii,jj] == 0): + continue - SearchRangeX = slice(int(-clx - SearchLimitX[ii,jj] + xGrid[ii,jj]) , int(clx + SearchLimitX[ii,jj] - 1 + xGrid[ii,jj])) - SearchRangeY = slice(int(-cly - SearchLimitY[ii,jj] + yGrid[ii,jj]) , int(cly + SearchLimitY[ii,jj] - 1 + yGrid[ii,jj])) - RefI = I1[SearchRangeY,SearchRangeX] - - minChipI = np.min(ChipI) - if minChipI < 0: - ChipI = ChipI - minChipI - if np.all(ChipI == ChipI[0,0]): - continue - - minRefI = np.min(RefI) - if minRefI < 0: - RefI = RefI - minRefI - if np.all(RefI == RefI[0,0]): - continue + # remember motion terms Dx and Dy correspond to I1 relative to I2 (reference) + clx = np.floor(ChipSizeX[ii,jj]/2) + ChipRangeX = slice(int(-clx - Dx0[ii,jj] + xGrid[ii,jj]) , int(clx - Dx0[ii,jj] + xGrid[ii,jj])) + cly = np.floor(ChipSizeY[ii,jj]/2) + ChipRangeY = slice(int(-cly - Dy0[ii,jj] + yGrid[ii,jj]) , int(cly - Dy0[ii,jj] + yGrid[ii,jj])) + ChipI = I2[ChipRangeY,ChipRangeX] + + SearchRangeX = slice(int(-clx - SearchLimitX[ii,jj] + xGrid[ii,jj]) , int(clx + SearchLimitX[ii,jj] - 1 + xGrid[ii,jj])) + SearchRangeY = slice(int(-cly - SearchLimitY[ii,jj] + yGrid[ii,jj]) , int(cly + SearchLimitY[ii,jj] - 1 + yGrid[ii,jj])) + RefI = I1[SearchRangeY,SearchRangeX] + + minChipI = np.min(ChipI) + if minChipI < 0: + ChipI = ChipI - minChipI + if np.all(ChipI == ChipI[0,0]): + continue + + minRefI = np.min(RefI) + if minRefI < 0: + RefI = RefI - minRefI + if np.all(RefI == RefI[0,0]): + continue + - - if SubPixFlag: - # call C++ - Dx1[ii], Dy1[ii] = np.float32(autoriftcore.arSubPixDisp_s_Py(core._autoriftcore,ChipI.shape[1],ChipI.shape[0],ChipI.ravel(),RefI.shape[1],RefI.shape[0],RefI.ravel(), oversample)) + if SubPixFlag: + # call C++ + Dx1[ii], Dy1[ii] = np.float32(autoriftcore.arSubPixDisp_s_Py(core._autoriftcore,ChipI.shape[1],ChipI.shape[0],ChipI.ravel(),RefI.shape[1],RefI.shape[0],RefI.ravel(),oversample)) # # call Python # Dx1[ii], Dy1[ii] = arSubPixDisp(ChipI,RefI) - else: - # call C++ - Dx1[ii], Dy1[ii] = np.float32(autoriftcore.arPixDisp_s_Py(core._autoriftcore,ChipI.shape[1],ChipI.shape[0],ChipI.ravel(),RefI.shape[1],RefI.shape[0],RefI.ravel())) + else: + # call C++ + Dx1[ii], Dy1[ii] = np.float32(autoriftcore.arPixDisp_s_Py(core._autoriftcore,ChipI.shape[1],ChipI.shape[0],ChipI.ravel(),RefI.shape[1],RefI.shape[0],RefI.ravel())) # # call Python # Dx1[ii], Dy1[ii] = arPixDisp(ChipI,RefI) - + else: + # Preparation for parallel + in_shape = xGrid.shape + I_shape = I1.shape + shape_prod = np.asscalar(np.prod(in_shape)) + + # import pdb + # pdb.set_trace() + XI1 = mp.RawArray('f', np.asscalar(np.prod(I_shape))) + XI1_np = np.frombuffer(XI1,dtype=np.float32).reshape(I_shape) + np.copyto(XI1_np,I1) + del I1 + + XI2 = mp.RawArray('f', np.asscalar(np.prod(I_shape))) + XI2_np = np.frombuffer(XI2,dtype=np.float32).reshape(I_shape) + np.copyto(XI2_np,I2) + del I2 + + XxGrid = mp.RawArray('f', shape_prod) + XxGrid_np = np.frombuffer(XxGrid,dtype=np.float32).reshape(in_shape) + np.copyto(XxGrid_np,xGrid) + del xGrid + + XyGrid = mp.RawArray('f', shape_prod) + XyGrid_np = np.frombuffer(XyGrid,dtype=np.float32).reshape(in_shape) + np.copyto(XyGrid_np,yGrid) + del yGrid + + XSearchLimitX = mp.RawArray('f', shape_prod) + XSearchLimitX_np = np.frombuffer(XSearchLimitX,dtype=np.float32).reshape(in_shape) + np.copyto(XSearchLimitX_np,SearchLimitX) + + XSearchLimitY = mp.RawArray('f', shape_prod) + XSearchLimitY_np = np.frombuffer(XSearchLimitY,dtype=np.float32).reshape(in_shape) + np.copyto(XSearchLimitY_np,SearchLimitY) + + XChipSizeX = mp.RawArray('f', shape_prod) + XChipSizeX_np = np.frombuffer(XChipSizeX,dtype=np.float32).reshape(in_shape) + np.copyto(XChipSizeX_np,ChipSizeX) + del ChipSizeX + + XChipSizeY = mp.RawArray('f', shape_prod) + XChipSizeY_np = np.frombuffer(XChipSizeY,dtype=np.float32).reshape(in_shape) + np.copyto(XChipSizeY_np,ChipSizeY) + del ChipSizeY + + XDx0 = mp.RawArray('f', shape_prod) + XDx0_np = np.frombuffer(XDx0,dtype=np.float32).reshape(in_shape) + np.copyto(XDx0_np,Dx0) + + XDy0 = mp.RawArray('f', shape_prod) + XDy0_np = np.frombuffer(XDy0,dtype=np.float32).reshape(in_shape) + np.copyto(XDy0_np,Dy0) + # import pdb + # pdb.set_trace() + + +# Nchunks = mp.cpu_count() // 8 * MultiThread + Nchunks = MultiThread + chunkSize = int(np.floor(shape_prod / Nchunks)) + chunkRem = shape_prod - chunkSize * Nchunks + + CHUNKS = [] + + for k in range(Nchunks): + # print(k) + if k == (Nchunks-1): + chunkInds = np.arange(k*chunkSize, (k+1)*chunkSize+chunkRem) + else: + chunkInds = np.arange(k*chunkSize, (k+1)*chunkSize) + CHUNKS.append(chunkInds) + # print(CHUNKS) + + chunk_inputs = [(kk, CHUNKS[kk], SubPixFlag, oversample, in_shape, I_shape) + for kk in range(Nchunks)] + + with mp.Pool(initializer=initializer, initargs=(XI1, XI2, XxGrid, XyGrid, XSearchLimitX, XSearchLimitY, XChipSizeX, XChipSizeY, XDx0, XDy0)) as pool: + Dx, Dy = zip(*pool.map(unpacking_loop_s, chunk_inputs)) + + Dx = np.concatenate(Dx) + Dy = np.concatenate(Dy) + + Dx = np.reshape(Dx, in_shape) + Dy = np.reshape(Dy, in_shape) # add back 1) I1 (RefI) relative to I2 (ChipI) initial offset Dx0 and Dy0, and # 2) RefI relative to ChipI has a left/top boundary offset of -SearchLimitX and -SearchLimitY diff --git a/geo_autoRIFT/autoRIFT/autoRIFT_ISCE.py b/geo_autoRIFT/autoRIFT/autoRIFT_ISCE.py index 7345e59..39bd76b 100755 --- a/geo_autoRIFT/autoRIFT/autoRIFT_ISCE.py +++ b/geo_autoRIFT/autoRIFT/autoRIFT_ISCE.py @@ -186,6 +186,13 @@ mandatory=False, doc = 'Input data type: 0 -> uint8, 1 -> float32') +MULTI_THREAD = Component.Parameter('MultiThread', + public_name = 'MULTI_THREAD', + default = 0, + type = int, + mandatory=False, + doc = 'Number of Threads; default specified by 0 uses single core and surpasses the multithreading routine') + try: # Try Autorift within ISCE first @@ -223,7 +230,8 @@ class autoRIFT_ISCE(autoRIFT, Component): BUFF_DISTANCE_C, COARSE_COR_CUTOFF, OVER_SAMPLE_RATIO, - DATA_TYPE) + DATA_TYPE, + MULTI_THREAD) diff --git a/geo_autoRIFT/geogrid/Geogrid.py b/geo_autoRIFT/geogrid/Geogrid.py index 77f9c6a..5e0a96a 100755 --- a/geo_autoRIFT/geogrid/Geogrid.py +++ b/geo_autoRIFT/geogrid/Geogrid.py @@ -77,7 +77,10 @@ def getProjectionSystem(self): raise Exception('At least the DEM parameter must be set for geogrid') from osgeo import gdal, osr - ds = gdal.Open(self.demname, gdal.GA_ReadOnly) + if self.urlflag == 1: + ds = gdal.Open('/vsicurl/%s' %(self.demname)) + else: + ds = gdal.Open(self.demname, gdal.GA_ReadOnly) srs = osr.SpatialReference() srs.ImportFromWkt(ds.GetProjection()) srs.AutoIdentifyEPSG() @@ -93,7 +96,10 @@ def getProjectionSystem(self): else: raise Exception('Non-standard coordinate system encountered') if not epsgstr: #Empty string->use shell command gdalsrsinfo for last trial - cmd = 'gdalsrsinfo -o epsg {0}'.format(self.demname) + if self.urlflag == 1: + cmd = 'gdalsrsinfo -o epsg /vsicurl/{0}'.format(self.demname) + else: + cmd = 'gdalsrsinfo -o epsg {0}'.format(self.demname) epsgstr = subprocess.check_output(cmd, shell=True) # pdb.set_trace() epsgstr = re.findall("EPSG:(\d+)", str(epsgstr))[0] @@ -274,6 +280,9 @@ def setState(self): geogrid.setRO2VYFilename_Py( self._geogrid, self.winro2vyname) geogrid.setLookSide_Py(self._geogrid, self.lookSide) geogrid.setNodataOut_Py(self._geogrid, self.nodata_out) + if self.urlflag is None: + self.urlflag = 0 + geogrid.setUrlFlag_Py(self._geogrid, self.urlflag) self._orbit = self.orbit.exportToC() geogrid.setOrbit_Py(self._geogrid, self._orbit) @@ -321,6 +330,7 @@ def __init__(self): self.csmaxxname = None self.csmaxyname = None self.ssmname = None + self.urlflag = None ##Output related parameters self.winlocname = None diff --git a/geo_autoRIFT/geogrid/GeogridOptical.py b/geo_autoRIFT/geogrid/GeogridOptical.py index 3559d7e..a120603 100755 --- a/geo_autoRIFT/geogrid/GeogridOptical.py +++ b/geo_autoRIFT/geogrid/GeogridOptical.py @@ -67,7 +67,7 @@ def getProjectionSystem(self, filename, urlflag): raise Exception('File {0} does not exist'.format(filename)) from osgeo import gdal, osr - if urlflag is 1: + if urlflag == 1: ds = gdal.Open('/vsicurl/%s' %(filename)) else: ds = gdal.Open(filename, gdal.GA_ReadOnly) @@ -86,7 +86,7 @@ def getProjectionSystem(self, filename, urlflag): else: raise Exception('Non-standard coordinate system encountered') if not epsgstr: #Empty string->use shell command gdalsrsinfo for last trial - if urlflag is 1: + if urlflag == 1: cmd = 'gdalsrsinfo -o epsg /vsicurl/{0}'.format(filename) else: cmd = 'gdalsrsinfo -o epsg {0}'.format(filename) @@ -163,7 +163,7 @@ def geogrid(self): urlflag = self.urlflag - if urlflag is 1: + if urlflag == 1: print("\nReading input images into memory directly from URL's") else: print("\nReading input images locally from files") @@ -231,56 +231,45 @@ def geogrid(self): import struct # pdb.set_trace() - if urlflag is 1: - demDS = gdal.Open('/vsicurl/%s' %(self.demname)) - else: - demDS = gdal.Open(self.demname, gdal.GA_ReadOnly) + if urlflag == 1: + self.demname = '/vsicurl/%s' %(self.demname) + self.dhdxname = '/vsicurl/%s' %(self.dhdxname) + self.dhdyname = '/vsicurl/%s' %(self.dhdyname) + self.vxname = '/vsicurl/%s' %(self.vxname) + self.vyname = '/vsicurl/%s' %(self.vyname) + self.srxname = '/vsicurl/%s' %(self.srxname) + self.sryname = '/vsicurl/%s' %(self.sryname) + self.csminxname = '/vsicurl/%s' %(self.csminxname) + self.csminyname = '/vsicurl/%s' %(self.csminyname) + self.csmaxxname = '/vsicurl/%s' %(self.csmaxxname) + self.csmaxyname = '/vsicurl/%s' %(self.csmaxyname) + self.ssmname = '/vsicurl/%s' %(self.ssmname) + + + demDS = gdal.Open(self.demname, gdal.GA_ReadOnly) if (self.dhdxname != ""): - if urlflag is 1: - sxDS = gdal.Open('/vsicurl/%s' %(self.dhdxname)) - syDS = gdal.Open('/vsicurl/%s' %(self.dhdyname)) - else: - sxDS = gdal.Open(self.dhdxname, gdal.GA_ReadOnly) - syDS = gdal.Open(self.dhdyname, gdal.GA_ReadOnly) + sxDS = gdal.Open(self.dhdxname, gdal.GA_ReadOnly) + syDS = gdal.Open(self.dhdyname, gdal.GA_ReadOnly) if (self.vxname != ""): - if urlflag is 1: - vxDS = gdal.Open('/vsicurl/%s' %(self.vxname)) - vyDS = gdal.Open('/vsicurl/%s' %(self.vyname)) - else: - vxDS = gdal.Open(self.vxname, gdal.GA_ReadOnly) - vyDS = gdal.Open(self.vyname, gdal.GA_ReadOnly) + vxDS = gdal.Open(self.vxname, gdal.GA_ReadOnly) + vyDS = gdal.Open(self.vyname, gdal.GA_ReadOnly) if (self.srxname != ""): - if urlflag is 1: - srxDS = gdal.Open('/vsicurl/%s' %(self.srxname)) - sryDS = gdal.Open('/vsicurl/%s' %(self.sryname)) - else: - srxDS = gdal.Open(self.srxname, gdal.GA_ReadOnly) - sryDS = gdal.Open(self.sryname, gdal.GA_ReadOnly) + srxDS = gdal.Open(self.srxname, gdal.GA_ReadOnly) + sryDS = gdal.Open(self.sryname, gdal.GA_ReadOnly) if (self.csminxname != ""): - if urlflag is 1: - csminxDS = gdal.Open('/vsicurl/%s' %(self.csminxname)) - csminyDS = gdal.Open('/vsicurl/%s' %(self.csminyname)) - else: - csminxDS = gdal.Open(self.csminxname, gdal.GA_ReadOnly) - csminyDS = gdal.Open(self.csminyname, gdal.GA_ReadOnly) + csminxDS = gdal.Open(self.csminxname, gdal.GA_ReadOnly) + csminyDS = gdal.Open(self.csminyname, gdal.GA_ReadOnly) if (self.csmaxxname != ""): - if urlflag is 1: - csmaxxDS = gdal.Open('/vsicurl/%s' %(self.csmaxxname)) - csmaxyDS = gdal.Open('/vsicurl/%s' %(self.csmaxyname)) - else: - csmaxxDS = gdal.Open(self.csmaxxname, gdal.GA_ReadOnly) - csmaxyDS = gdal.Open(self.csmaxyname, gdal.GA_ReadOnly) + csmaxxDS = gdal.Open(self.csmaxxname, gdal.GA_ReadOnly) + csmaxyDS = gdal.Open(self.csmaxyname, gdal.GA_ReadOnly) if (self.ssmname != ""): - if urlflag is 1: - ssmDS = gdal.Open('/vsicurl/%s' %(self.ssmname)) - else: - ssmDS = gdal.Open(self.ssmname, gdal.GA_ReadOnly) + ssmDS = gdal.Open(self.ssmname, gdal.GA_ReadOnly) if demDS is None: raise Exception('Error opening DEM file {0}'.format(self.demname)) @@ -777,7 +766,7 @@ def coregister(self,in1,in2,urlflag): from osgeo import gdal, osr import struct - if urlflag is 1: + if urlflag == 1: DS1 = gdal.Open('/vsicurl/%s' %(in1)) else: DS1 = gdal.Open(in1, gdal.GA_ReadOnly) @@ -785,7 +774,7 @@ def coregister(self,in1,in2,urlflag): xsize1 = DS1.RasterXSize ysize1 = DS1.RasterYSize - if urlflag is 1: + if urlflag == 1: DS2 = gdal.Open('/vsicurl/%s' %(in2)) else: DS2 = gdal.Open(in2, gdal.GA_ReadOnly) @@ -837,7 +826,7 @@ def coregister(self,in1,in2,urlflag): trans = (W, trans1[1], 0.0, N, 0.0, trans1[5]) - if urlflag is 0: + if urlflag == 0: I1 = DS1.ReadAsArray(xoff=x1a, yoff=y1a, xsize=x1b-x1a+1, ysize=y1b-y1a+1) I2 = DS2.ReadAsArray(xoff=x2a, yoff=y2a, xsize=x2b-x2a+1, ysize=y2b-y2a+1) diff --git a/geo_autoRIFT/geogrid/bindings/geogridmodule.cpp b/geo_autoRIFT/geogrid/bindings/geogridmodule.cpp index 4a5c3f3..7dd37cc 100755 --- a/geo_autoRIFT/geogrid/bindings/geogridmodule.cpp +++ b/geo_autoRIFT/geogrid/bindings/geogridmodule.cpp @@ -430,6 +430,19 @@ PyObject* setNodataOut(PyObject *self, PyObject *args) return Py_BuildValue("i", 0); } +PyObject* setUrlFlag(PyObject *self, PyObject *args) +{ + uint64_t ptr; + int urlflag; + if (!PyArg_ParseTuple(args, "Ki", &ptr, &urlflag)) + { + return NULL; + } + + ((geoGrid*)(ptr))->urlflag = urlflag; + return Py_BuildValue("i", 0); +} + PyObject* setOrbit(PyObject *self, PyObject *args) { uint64_t ptr, orb; diff --git a/geo_autoRIFT/geogrid/include/geogrid.h b/geo_autoRIFT/geogrid/include/geogrid.h index 966df6c..a856c61 100755 --- a/geo_autoRIFT/geogrid/include/geogrid.h +++ b/geo_autoRIFT/geogrid/include/geogrid.h @@ -69,6 +69,7 @@ struct geoGrid int nPixels; int lookSide; int nodata_out; + int urlflag; double incidenceAngle; //Output file names diff --git a/geo_autoRIFT/geogrid/include/geogridmodule.h b/geo_autoRIFT/geogrid/include/geogridmodule.h index 22bb6d0..77c205e 100755 --- a/geo_autoRIFT/geogrid/include/geogridmodule.h +++ b/geo_autoRIFT/geogrid/include/geogridmodule.h @@ -53,6 +53,7 @@ extern "C" PyObject * setOrbit(PyObject *, PyObject *); PyObject * setLookSide(PyObject *, PyObject *); PyObject * setNodataOut(PyObject *, PyObject *); + PyObject * setUrlFlag(PyObject *, PyObject *); PyObject * setWindowLocationsFilename(PyObject *, PyObject *); PyObject * setWindowOffsetsFilename(PyObject *, PyObject *); @@ -91,6 +92,7 @@ static PyMethodDef geogrid_methods[] = {"setOrbit_Py", setOrbit, METH_VARARGS, " "}, {"setLookSide_Py", setLookSide, METH_VARARGS, " "}, {"setNodataOut_Py", setNodataOut, METH_VARARGS, " "}, + {"setUrlFlag_Py", setUrlFlag, METH_VARARGS, " "}, {"setXLimits_Py", setXLimits, METH_VARARGS, " "}, {"setYLimits_Py", setYLimits, METH_VARARGS, " "}, {"setWindowLocationsFilename_Py", setWindowLocationsFilename, METH_VARARGS, " "}, diff --git a/geo_autoRIFT/geogrid/src/geogrid.cpp b/geo_autoRIFT/geogrid/src/geogrid.cpp index abb2d2f..ea6e07d 100755 --- a/geo_autoRIFT/geogrid/src/geogrid.cpp +++ b/geo_autoRIFT/geogrid/src/geogrid.cpp @@ -47,6 +47,14 @@ void geoGrid::geogrid() //For now print inputs that were obtained + if (urlflag == 1){ + std::cout << "\nReading input images into memory directly from URL's\n"; + } + else + { + std::cout << "\nReading input images locally from files\n"; + } + std::cout << "\nRadar parameters: \n"; std::cout << "Range: " << startingRange << " " << dr << "\n"; std::cout << "Azimuth: " << sensingStart << " " << prf << "\n"; @@ -144,7 +152,23 @@ void geoGrid::geogrid() GDALDataset* ssmDS = NULL; double geoTrans[6]; - + + std::string url ("/vsicurl/"); + if (urlflag == 1) + { + demname = url + demname; + dhdxname = url + dhdxname; + dhdyname = url + dhdyname; + vxname = url + vxname; + vyname = url + vyname; + srxname = url + srxname; + sryname = url + sryname; + csminxname = url + csminxname; + csminyname = url + csminyname; + csmaxxname = url + csmaxxname; + csmaxyname = url + csmaxyname; + ssmname = url + ssmname; + } demDS = reinterpret_cast(GDALOpenShared(demname.c_str(), GA_ReadOnly)); if (dhdxname != "") { diff --git a/testautoRIFT.py b/testautoRIFT.py index 97c77fb..1cfc5b9 100755 --- a/testautoRIFT.py +++ b/testautoRIFT.py @@ -76,7 +76,9 @@ def cmdLineParse(): parser.add_argument('-nc', '--sensor_flag_netCDF', dest='nc_sensor', type=str, required=False, default=None, help='flag for packaging output formatted for Sentinel ("S") and Landsat ("L") dataset; default is None') parser.add_argument('-urlflag', '--urlflag', dest='urlflag', type=int, required=False, default=0, - help='flag for reading and coregistering optical data (GeoTIFF images, e.g. Landsat): use 1 for url read and 0 for local machine read; if not specified (i.e. None; default), will just read from local machine without coregistration') + help='flag for reading and coregistering optical data (GeoTIFF images, e.g. Landsat): use 1 for url read and 0 (default) for local machine read') + parser.add_argument('-mpflag', '--mpflag', dest='mpflag', type=int, required=False, default=0, + help='number of threads for multiple threading (default is specified by 0, which uses the original single-core version and surpasses the multithreading routine)') return parser.parse_args() @@ -145,7 +147,7 @@ def loadProductOpticalURL(file_m, file_s): return I1, I2 -def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CSMAXx0, CSMAXy0, noDataMask, optflag, nodata): +def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CSMAXx0, CSMAXy0, noDataMask, optflag, nodata, mpflag): ''' Wire and run geogrid. ''' @@ -166,7 +168,8 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS # I1 = I1.astype(np.uint8) # I2 = I2.astype(np.uint8) - + obj.MultiThread = mpflag + # take the amplitude only for the radar images if optflag == 0: I1 = np.abs(I1) @@ -395,10 +398,12 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS inps = cmdLineParse() + mpflag = inps.mpflag + urlflag = inps.urlflag if inps.optical_flag == 1: - if urlflag is 1: + if urlflag == 1: data_m, data_s = loadProductOpticalURL(inps.indir_m, inps.indir_s) else: data_m = loadProductOptical(inps.indir_m) @@ -486,7 +491,7 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS ds=None - Dx, Dy, InterpMask, ChipSizeX, ScaleChipSizeY, SearchLimitX, SearchLimitY, origSize, noDataMask = runAutorift(data_m, data_s, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CSMAXx0, CSMAXy0, noDataMask, inps.optical_flag, nodata) + Dx, Dy, InterpMask, ChipSizeX, ScaleChipSizeY, SearchLimitX, SearchLimitY, origSize, noDataMask = runAutorift(data_m, data_s, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CSMAXx0, CSMAXy0, noDataMask, inps.optical_flag, nodata, mpflag) if inps.optical_flag == 0: Dy = -Dy @@ -605,7 +610,7 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS xcount = int(str.split(runCmd('fgrep "Dimensions of geogrid:" testGeogrid.txt'))[3]) ycount = int(str.split(runCmd('fgrep "Dimensions of geogrid:" testGeogrid.txt'))[5]) - if urlflag is 1: + if urlflag == 1: ds = gdal.Open('/vsicurl/%s' %(vxrefname)) else: ds = gdal.Open(vxrefname) @@ -614,7 +619,7 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS ds = None band = None - if urlflag is 1: + if urlflag == 1: ds = gdal.Open('/vsicurl/%s' %(vyrefname)) else: ds = gdal.Open(vyrefname) @@ -623,7 +628,7 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS ds = None band = None - if urlflag is 1: + if urlflag == 1: ds = gdal.Open('/vsicurl/%s' %(sxname)) else: ds = gdal.Open(sxname) @@ -632,7 +637,7 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS ds = None band = None - if urlflag is 1: + if urlflag == 1: ds = gdal.Open('/vsicurl/%s' %(syname)) else: ds = gdal.Open(syname) @@ -641,7 +646,7 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS ds = None band = None - if urlflag is 1: + if urlflag == 1: ds = gdal.Open('/vsicurl/%s' %(maskname)) else: ds = gdal.Open(maskname) @@ -653,7 +658,7 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS DXref = offset2vy_2 / (offset2vx_1 * offset2vy_2 - offset2vx_2 * offset2vy_1) * VXref - offset2vx_2 / (offset2vx_1 * offset2vy_2 - offset2vx_2 * offset2vy_1) * VYref DYref = offset2vx_1 / (offset2vx_1 * offset2vy_2 - offset2vx_2 * offset2vy_1) * VYref - offset2vy_1 / (offset2vx_1 * offset2vy_2 - offset2vx_2 * offset2vy_1) * VXref - stable_count = np.sum(SSM & np.logical_not(np.isnan(DX))) + stable_count = np.sum(SSM & np.logical_not(np.isnan(DX)) & (DX-DXref > -5) & (DX-DXref < 5) & (DY-DYref > -5) & (DY-DYref < 5)) if stable_count == 0: stable_shift_applied = 0 @@ -704,10 +709,12 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS pair_type = 'radar' detection_method = 'feature' coordinates = 'radar' + roi_valid_percentage = int(round(np.sum(CHIPSIZEX!=0)/np.sum(SEARCHLIMITX!=0)*1000.0))/1000 # out_nc_filename = 'Jakobshavn.nc' - out_nc_filename = master_filename[0:-4]+'_'+slave_filename[0:-4]+'.nc' + PPP = roi_valid_percentage * 100 + out_nc_filename = master_filename[0:-4]+'_X_'+slave_filename[0:-4]+'_G0240V02_P{0}{1}{2}.nc'.format(int(PPP/10),int((PPP-int(PPP/10)*10)),int((PPP-int(PPP/10)*10-int((PPP-int(PPP/10)*10)))*10)) out_nc_filename = './' + out_nc_filename - roi_valid_percentage = int(round(np.sum(CHIPSIZEX!=0)/np.sum(SEARCHLIMITX!=0)*1000.0))/1000 + CHIPSIZEY = np.round(CHIPSIZEX * ScaleChipSizeY / 2) * 2 @@ -763,10 +770,11 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS pair_type = 'optical' detection_method = 'feature' coordinates = 'map' + roi_valid_percentage = int(round(np.sum(CHIPSIZEX!=0)/np.sum(SEARCHLIMITX!=0)*1000.0))/1000 # out_nc_filename = 'Jakobshavn_opt.nc' - out_nc_filename = master_filename[0:-4]+'_'+slave_filename[0:-4]+'.nc' + PPP = roi_valid_percentage * 100 + out_nc_filename = master_filename[0:-8]+'_X_'+slave_filename[0:-8]+'_G0240V02_P{0}{1}{2}.nc'.format(int(PPP/10),int((PPP-int(PPP/10)*10)),int((PPP-int(PPP/10)*10-int((PPP-int(PPP/10)*10)))*10)) out_nc_filename = './' + out_nc_filename - roi_valid_percentage = int(round(np.sum(CHIPSIZEX!=0)/np.sum(SEARCHLIMITX!=0)*1000.0))/1000 CHIPSIZEY = np.round(CHIPSIZEX * ScaleChipSizeY / 2) * 2 from datetime import date @@ -819,10 +827,10 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS pair_type = 'optical' detection_method = 'feature' coordinates = 'map' - - out_nc_filename = master_filename[0:-4]+'_'+slave_filename[0:-4]+'.nc' - out_nc_filename = './' + out_nc_filename roi_valid_percentage = int(round(np.sum(CHIPSIZEX!=0)/np.sum(SEARCHLIMITX!=0)*1000.0))/1000 + PPP = roi_valid_percentage * 100 + out_nc_filename = master_filename[0:-8]+'_X_'+slave_filename[0:-8]+'_G0240V02_P{0}{1}{2}.nc'.format(int(PPP/10),int((PPP-int(PPP/10)*10)),int((PPP-int(PPP/10)*10-int((PPP-int(PPP/10)*10)))*10)) + out_nc_filename = './' + out_nc_filename CHIPSIZEY = np.round(CHIPSIZEX * ScaleChipSizeY / 2) * 2 from datetime import date diff --git a/testautoRIFT_ISCE.py b/testautoRIFT_ISCE.py index ba79211..be118ca 100755 --- a/testautoRIFT_ISCE.py +++ b/testautoRIFT_ISCE.py @@ -76,8 +76,9 @@ def cmdLineParse(): parser.add_argument('-nc', '--sensor_flag_netCDF', dest='nc_sensor', type=str, required=False, default=None, help='flag for packaging output formatted for Sentinel ("S") and Landsat ("L") dataset; default is None') parser.add_argument('-urlflag', '--urlflag', dest='urlflag', type=int, required=False, default=0, - help='flag for reading and coregistering optical data (GeoTIFF images, e.g. Landsat): use 1 for url read and 0 for local machine read; if not specified (i.e. None; default), will just read from local machine without coregistration') - + help='flag for reading and coregistering optical data (GeoTIFF images, e.g. Landsat): use 1 for url read and 0 (default) for local machine read') + parser.add_argument('-mpflag', '--mpflag', dest='mpflag', type=int, required=False, default=0, + help='number of threads for multiple threading (default is specified by 0, which uses the original single-core version and surpasses the multithreading routine)') return parser.parse_args() class Dummy(object): @@ -143,7 +144,7 @@ def loadProductOpticalURL(file_m, file_s): return I1, I2 -def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CSMAXx0, CSMAXy0, noDataMask, optflag, nodata): +def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CSMAXx0, CSMAXy0, noDataMask, optflag, nodata, mpflag): ''' Wire and run geogrid. ''' @@ -159,10 +160,11 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS obj = autoRIFT_ISCE() obj.configure() -# ########## uncomment if starting from preprocessed images + ########## uncomment if starting from preprocessed images # I1 = I1.astype(np.uint8) # I2 = I2.astype(np.uint8) + obj.MultiThread = mpflag # take the amplitude only for the radar images if optflag == 0: @@ -186,6 +188,9 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS # obj.SkipSampleX=8 # obj.SkipSampleY=8 + # test with small tiff images +# obj.SkipSampleX=16 +# obj.SkipSampleY=16 # create the grid if it does not exist if xGrid is None: @@ -279,10 +284,7 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS ######## preprocessing t1 = time.time() print("Pre-process Start!!!") -# obj.zeroMask = 1 obj.preprocess_filt_hps() -# obj.I1 = np.abs(I1) -# obj.I2 = np.abs(I2) print("Pre-process Done!!!") print(time.time()-t1) @@ -388,10 +390,12 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS inps = cmdLineParse() + mpflag = inps.mpflag + urlflag = inps.urlflag if inps.optical_flag == 1: - if urlflag is 1: + if urlflag == 1: data_m, data_s = loadProductOpticalURL(inps.indir_m, inps.indir_s) else: data_m = loadProductOptical(inps.indir_m) @@ -481,7 +485,7 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS - Dx, Dy, InterpMask, ChipSizeX, ScaleChipSizeY, SearchLimitX, SearchLimitY, origSize, noDataMask = runAutorift(data_m, data_s, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CSMAXx0, CSMAXy0, noDataMask, inps.optical_flag, nodata) + Dx, Dy, InterpMask, ChipSizeX, ScaleChipSizeY, SearchLimitX, SearchLimitY, origSize, noDataMask = runAutorift(data_m, data_s, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CSMAXx0, CSMAXy0, noDataMask, inps.optical_flag, nodata, mpflag) if inps.optical_flag == 0: Dy = -Dy @@ -600,7 +604,7 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS xcount = int(str.split(runCmd('fgrep "Dimensions of geogrid:" testGeogrid.txt'))[3]) ycount = int(str.split(runCmd('fgrep "Dimensions of geogrid:" testGeogrid.txt'))[5]) - if urlflag is 1: + if urlflag == 1: ds = gdal.Open('/vsicurl/%s' %(vxrefname)) else: ds = gdal.Open(vxrefname) @@ -609,7 +613,7 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS ds = None band = None - if urlflag is 1: + if urlflag == 1: ds = gdal.Open('/vsicurl/%s' %(vyrefname)) else: ds = gdal.Open(vyrefname) @@ -618,7 +622,7 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS ds = None band = None - if urlflag is 1: + if urlflag == 1: ds = gdal.Open('/vsicurl/%s' %(sxname)) else: ds = gdal.Open(sxname) @@ -627,7 +631,7 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS ds = None band = None - if urlflag is 1: + if urlflag == 1: ds = gdal.Open('/vsicurl/%s' %(syname)) else: ds = gdal.Open(syname) @@ -636,7 +640,7 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS ds = None band = None - if urlflag is 1: + if urlflag == 1: ds = gdal.Open('/vsicurl/%s' %(maskname)) else: ds = gdal.Open(maskname) @@ -648,7 +652,7 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS DXref = offset2vy_2 / (offset2vx_1 * offset2vy_2 - offset2vx_2 * offset2vy_1) * VXref - offset2vx_2 / (offset2vx_1 * offset2vy_2 - offset2vx_2 * offset2vy_1) * VYref DYref = offset2vx_1 / (offset2vx_1 * offset2vy_2 - offset2vx_2 * offset2vy_1) * VYref - offset2vy_1 / (offset2vx_1 * offset2vy_2 - offset2vx_2 * offset2vy_1) * VXref - stable_count = np.sum(SSM & np.logical_not(np.isnan(DX))) + stable_count = np.sum(SSM & np.logical_not(np.isnan(DX)) & (DX-DXref > -5) & (DX-DXref < 5) & (DY-DYref > -5) & (DY-DYref < 5)) if stable_count == 0: stable_shift_applied = 0 @@ -695,14 +699,15 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS slave_split = str.split(slave_filename,'_') import netcdf_output as no - version = '1.0.7' + version = '1.1.0' pair_type = 'radar' detection_method = 'feature' coordinates = 'radar' + roi_valid_percentage = int(round(np.sum(CHIPSIZEX!=0)/np.sum(SEARCHLIMITX!=0)*1000.0))/1000 # out_nc_filename = 'Jakobshavn.nc' - out_nc_filename = master_filename[0:-4]+'_'+slave_filename[0:-4]+'.nc' + PPP = roi_valid_percentage * 100 + out_nc_filename = master_filename[0:-4]+'_X_'+slave_filename[0:-4]+'_G0240V02_P{0}{1}{2}.nc'.format(int(PPP/10),int((PPP-int(PPP/10)*10)),int((PPP-int(PPP/10)*10-int((PPP-int(PPP/10)*10)))*10)) out_nc_filename = './' + out_nc_filename - roi_valid_percentage = int(round(np.sum(CHIPSIZEX!=0)/np.sum(SEARCHLIMITX!=0)*1000.0))/1000 CHIPSIZEY = np.round(CHIPSIZEX * ScaleChipSizeY / 2) * 2 @@ -755,14 +760,15 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS slave_time = time1(int(slave_time[0]),int(slave_time[1]),int(float(slave_time[2]))) import netcdf_output as no - version = '1.0.7' + version = '1.1.0' pair_type = 'optical' detection_method = 'feature' coordinates = 'map' + roi_valid_percentage = int(round(np.sum(CHIPSIZEX!=0)/np.sum(SEARCHLIMITX!=0)*1000.0))/1000 # out_nc_filename = 'Jakobshavn_opt.nc' - out_nc_filename = master_filename[0:-4]+'_'+slave_filename[0:-4]+'.nc' + PPP = roi_valid_percentage * 100 + out_nc_filename = master_filename[0:-8]+'_X_'+slave_filename[0:-8]+'_G0240V02_P{0}{1}{2}.nc'.format(int(PPP/10),int((PPP-int(PPP/10)*10)),int((PPP-int(PPP/10)*10-int((PPP-int(PPP/10)*10)))*10)) out_nc_filename = './' + out_nc_filename - roi_valid_percentage = int(round(np.sum(CHIPSIZEX!=0)/np.sum(SEARCHLIMITX!=0)*1000.0))/1000 CHIPSIZEY = np.round(CHIPSIZEX * ScaleChipSizeY / 2) * 2 from datetime import date @@ -811,14 +817,14 @@ def runAutorift(I1, I2, xGrid, yGrid, Dx0, Dy0, SRx0, SRy0, CSMINx0, CSMINy0, CS slave_time = time1(int(slave_time[0]),int(slave_time[1]),int(float(slave_time[2]))) import netcdf_output as no - version = '1.0.7' + version = '1.1.0' pair_type = 'optical' detection_method = 'feature' coordinates = 'map' - - out_nc_filename = master_filename[0:-4]+'_'+slave_filename[0:-4]+'.nc' - out_nc_filename = './' + out_nc_filename roi_valid_percentage = int(round(np.sum(CHIPSIZEX!=0)/np.sum(SEARCHLIMITX!=0)*1000.0))/1000 + PPP = roi_valid_percentage * 100 + out_nc_filename = master_filename[0:-8]+'_X_'+slave_filename[0:-8]+'_G0240V02_P{0}{1}{2}.nc'.format(int(PPP/10),int((PPP-int(PPP/10)*10)),int((PPP-int(PPP/10)*10-int((PPP-int(PPP/10)*10)))*10)) + out_nc_filename = './' + out_nc_filename CHIPSIZEY = np.round(CHIPSIZEX * ScaleChipSizeY / 2) * 2 from datetime import date