Skip to content

Commit

Permalink
Merge branch 'choose_platform_and_device'
Browse files Browse the repository at this point in the history
  • Loading branch information
@bnorthan
bnorthan committed Dec 4, 2023
2 parents 817cb07 + d256d54 commit 270cdd7
Show file tree
Hide file tree
Showing 17 changed files with 267 additions and 183 deletions.
138 changes: 107 additions & 31 deletions native/clij2fft/clij2fft.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@ int globaldebug=0;
} \
} while (0)

#define MAXPLATFORMS 10
#define MAXDEVICESPERPLATFORM 10

// Author: Brian Northan
// License: BSD
Expand Down Expand Up @@ -897,29 +899,45 @@ int conv3d_32f_lp(size_t N0, size_t N1, size_t N2, long long l_image, long long
return ret;
}

int conv3d_32f(size_t N0, size_t N1, size_t N2, float *h_image, float *h_psf, float *h_out) {
return convcorr3d_32f(N0, N1, N2, h_image, h_psf, h_out, 0);
int conv3d_32f(size_t N0, size_t N1, size_t N2, float *h_image, float *h_psf, float *h_out, int platformIndex, int deviceIndex) {
return convcorr3d_32f(N0, N1, N2, h_image, h_psf, h_out, 0, platformIndex, deviceIndex);
}

int convcorr3d_32f(size_t N0, size_t N1, size_t N2, float *h_image, float *h_psf, float *h_out, bool correlate) {
int convcorr3d_32f(size_t N0, size_t N1, size_t N2, float *h_image, float *h_psf, float *h_out, bool correlate, int platformIndex, int deviceIndex) {

ENTEREXIT(1,"convcorr3d_32f");

cl_platform_id platformId = NULL;
cl_device_id deviceID = NULL;
cl_uint retNumDevices;
cl_platform_id *platformIds = new cl_platform_id[MAXPLATFORMS];
cl_uint retNumPlatforms;
cl_int ret = clGetPlatformIDs(1, &platformId, &retNumPlatforms);
cl_int ret = clGetPlatformIDs(MAXPLATFORMS, platformIds, &retNumPlatforms);
CHECKRETURN(ret, "deconv3d_32f_tf getPlatformIDs", 1);
char * platformName = new char[1000];
clGetPlatformInfo(platformIds[platformIndex], CL_PLATFORM_NAME, 1000, platformName, NULL);

cl_context_properties properties[] =
{
CL_CONTEXT_PLATFORM, (cl_context_properties)platformIds[platformIndex],
0 // signals end of property list
};

cl_device_id *deviceIDs = new cl_device_id[MAXDEVICESPERPLATFORM];
cl_uint retNumDevices;

ret = clGetDeviceIDs(platformId, CL_DEVICE_TYPE_DEFAULT, 1, &deviceID, &retNumDevices);
CHECKRETURN(ret,"convcorr3d_32f getdeviceid",1);
// Creating context.
cl_context context = clCreateContext(NULL, 1, &deviceID, NULL, NULL, &ret);
ret = clGetDeviceIDs(platformIds[platformIndex], CL_DEVICE_TYPE_ALL, MAXDEVICESPERPLATFORM, deviceIDs, &retNumDevices);

char * deviceName = new char[1000];
clGetDeviceInfo(deviceIDs[0], CL_DEVICE_NAME, 1000, deviceName, NULL);
std::cout<<std::flush;
CHECKRETURN(ret, "deconv3d_32f_tf getDeviceIDs", 1);

// Creating context.
cl_context context = clCreateContext(properties, 1, &deviceIDs[deviceIndex], NULL, NULL, &ret);
CHECKRETURN(ret, "deconv3d_32f_tf createContext", 1);

// Creating command queue
cl_command_queue commandQueue = clCreateCommandQueue(context, deviceID, 0, &ret);
CHECKRETURN(ret,"convcorr3d_32f createcommandqueue",1);
cl_command_queue commandQueue = clCreateCommandQueue(context, deviceIDs[deviceIndex], 0, &ret);
CHECKRETURN(ret, "deconv3d_32f_tf createCommandQueue", 1);

// Memory buffers for each array
cl_mem d_image = clCreateBuffer(context, CL_MEM_READ_WRITE, N2*N1*N0 * sizeof(float), NULL, &ret);
CHECKRETURN(ret,"convcorr3d_32f gpu image createbuffer",1);
Expand All @@ -934,7 +952,7 @@ int convcorr3d_32f(size_t N0, size_t N1, size_t N2, float *h_image, float *h_psf
ret = clEnqueueWriteBuffer(commandQueue, d_psf, CL_TRUE, 0, N2*N1*N0 * sizeof(float), h_psf, 0, NULL, NULL);
CHECKRETURN(ret,"convcorr3d_32f copy PSF",1);

conv3d_32f_lp(N0, N1, N2, (long long)d_image, (long long)d_psf, (long long)d_out, correlate, (long long)context, (long long)commandQueue, (long long)deviceID);
conv3d_32f_lp(N0, N1, N2, (long long)d_image, (long long)d_psf, (long long)d_out, correlate, (long long)context, (long long)commandQueue, (long long)deviceIDs[deviceIndex]);

// copy back to host
ret = clEnqueueReadBuffer( commandQueue, d_out, CL_TRUE, 0, N0*N1*N2*sizeof(float), h_out, 0, NULL, NULL );
Expand All @@ -947,6 +965,9 @@ int convcorr3d_32f(size_t N0, size_t N1, size_t N2, float *h_image, float *h_psf
// Release OpenCL working objects.
clReleaseCommandQueue( commandQueue );
clReleaseContext( context );

delete platformIds;
delete deviceIDs;

ENTEREXIT(0,"convcorr3d_32f");

Expand Down Expand Up @@ -1140,7 +1161,7 @@ int deconv3d_32f_lp_tv(int iterations, float regularizationFactor, size_t N0, si
}
}

printf("\nRichardson Lucy Finished");
printf("\nRichardson Lucy Finished\n");
std::cout<<std::flush;

// Release OpenCL memory objects.
Expand Down Expand Up @@ -1176,33 +1197,85 @@ int deconv3d_32f_lp_tv(int iterations, float regularizationFactor, size_t N0, si

}

int deconv3d_32f(int iterations, size_t N0, size_t N1, size_t N2, float *h_image, float *h_psf, float *h_out, float * normal) {
int deconv3d_32f(int iterations, size_t N0, size_t N1, size_t N2, float *h_image, float *h_psf, float *h_out, float * normal, int platformIndex, int deviceIndex) {

return deconv3d_32f_tv(iterations, 0.0, N0, N1,N2, h_image, h_psf, h_out, normal);
return deconv3d_32f_tv(iterations, 0.0, N0, N1,N2, h_image, h_psf, h_out, normal, platformIndex, deviceIndex);

}

int deconv3d_32f_tv(int iterations, float regularizationFactor, size_t N0, size_t N1, size_t N2, float *h_image, float *h_psf, float *h_out, float * h_normal) {
int print_platforms_and_devices() {

ENTEREXIT(1, "deconv3d_32f_tv");

cl_platform_id platformId = NULL;
cl_device_id deviceID = NULL;
cl_platform_id *platformId = new cl_platform_id[MAXPLATFORMS];
cl_uint retNumDevices;
cl_uint retNumPlatforms;

cl_int ret = clGetPlatformIDs(1, &platformId, &retNumPlatforms);
cl_int ret = clGetPlatformIDs(MAXPLATFORMS, platformId, &retNumPlatforms);
CHECKRETURN(ret, "deconv3d_32f_tf getPlatformIDs", 1);

ret = clGetDeviceIDs(platformId, CL_DEVICE_TYPE_DEFAULT, 1, &deviceID, &retNumDevices);
CHECKRETURN(ret, "deconv3d_32f_tf getDeviceIDs", 1);

for (int i=0;i<retNumPlatforms;i++) {
char * platformName = new char[1000];
clGetPlatformInfo(platformId[i], CL_PLATFORM_NAME, 1000, platformName, NULL);
printf("platform %d %s\n",i, platformName);

cl_device_id *deviceIDs = new cl_device_id[MAXDEVICESPERPLATFORM];

ret = clGetDeviceIDs(platformId[i], CL_DEVICE_TYPE_ALL, MAXDEVICESPERPLATFORM, deviceIDs, &retNumDevices);

for (int j=0;j<retNumDevices;j++) {
char * deviceName = new char[1000];
clGetDeviceInfo(deviceIDs[j], CL_DEVICE_NAME, 1000, deviceName, NULL);
printf(" device name %d %s\n", j, deviceName);
std::cout<<std::flush;
delete deviceName;
}
printf("\n");

delete platformName;
delete deviceIDs;

std::cout<<std::flush;
CHECKRETURN(ret, "deconv3d_32f_tf getDeviceIDs", 1);
}


return 0;
}

int deconv3d_32f_tv(int iterations, float regularizationFactor, size_t N0, size_t N1, size_t N2, float *h_image, float *h_psf, float *h_out, float * h_normal, int platformIndex, int deviceIndex) {

ENTEREXIT(1, "deconv3d_32f_tv");

cl_platform_id *platformIds = new cl_platform_id[MAXPLATFORMS];
cl_uint retNumPlatforms;
cl_int ret = clGetPlatformIDs(MAXPLATFORMS, platformIds, &retNumPlatforms);
CHECKRETURN(ret, "deconv3d_32f_tf getPlatformIDs", 1);
char * platformName = new char[1000];
clGetPlatformInfo(platformIds[platformIndex], CL_PLATFORM_NAME, 1000, platformName, NULL);
printf("\nplatform %d %s\n", platformIndex, platformName);

cl_context_properties properties[] =
{
CL_CONTEXT_PLATFORM, (cl_context_properties)platformIds[platformIndex],
0 // signals end of property list
};

cl_device_id *deviceIDs = new cl_device_id[MAXDEVICESPERPLATFORM];
cl_uint retNumDevices;

ret = clGetDeviceIDs(platformIds[platformIndex], CL_DEVICE_TYPE_ALL, MAXDEVICESPERPLATFORM, deviceIDs, &retNumDevices);

char * deviceName = new char[1000];
clGetDeviceInfo(deviceIDs[0], CL_DEVICE_NAME, 1000, deviceName, NULL);
printf("device name %d %s\n", deviceIndex, deviceName);
std::cout<<std::flush;
CHECKRETURN(ret, "deconv3d_32f_tf getDeviceIDs", 1);

// Creating context.
cl_context context = clCreateContext(NULL, 1, &deviceID, NULL, NULL, &ret);
cl_context context = clCreateContext(properties, 1, &deviceIDs[deviceIndex], NULL, NULL, &ret);
CHECKRETURN(ret, "deconv3d_32f_tf createContext", 1);

// Creating command queue
cl_command_queue commandQueue = clCreateCommandQueue(context, deviceID, 0, &ret);
cl_command_queue commandQueue = clCreateCommandQueue(context, deviceIDs[deviceIndex], 0, &ret);
CHECKRETURN(ret, "deconv3d_32f_tf createCommandQueue", 1);

// create device memory buffers for each array
Expand All @@ -1226,7 +1299,7 @@ int deconv3d_32f_tv(int iterations, float regularizationFactor, size_t N0, size_
unsigned long n = N0*N1*N2;
unsigned long nFreq=(N0/2+1)*N1*N2;

deconv3d_32f_lp_tv(iterations, regularizationFactor, N0, N1, N2, (long long)d_observed, (long long)d_psf, (long long)d_estimate, (long long)d_normal, (long long)context, (long long)commandQueue, (long long)deviceID);
deconv3d_32f_lp_tv(iterations, regularizationFactor, N0, N1, N2, (long long)d_observed, (long long)d_psf, (long long)d_estimate, (long long)d_normal, (long long)context, (long long)commandQueue, (long long)deviceIDs[deviceIndex]);

// copy back to host
ret = clEnqueueReadBuffer( commandQueue, d_estimate, CL_TRUE, 0, N0*N1*N2*sizeof(float), h_out, 0, NULL, NULL );
Expand All @@ -1243,6 +1316,9 @@ int deconv3d_32f_tv(int iterations, float regularizationFactor, size_t N0, size_
cleanup1:
clReleaseContext( context );

delete platformIds;
delete deviceIDs;

ENTEREXIT(0, "deconv3d_32f_tv");

return ret;
Expand Down
18 changes: 10 additions & 8 deletions native/clij2fft/clij2fft.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@

#ifdef _WIN64
extern "C" {
__declspec(dllexport) int conv3d_32f(size_t N1, size_t N2, size_t N3, float *h_image, float *h_psf, float *h_out);
__declspec(dllexport) int convcorr3d_32f(size_t N1, size_t N2, size_t N3, float *h_image, float *h_psf, float *h_out, bool correlate);
__declspec(dllexport) int conv3d_32f(size_t N1, size_t N2, size_t N3, float *h_image, float *h_psf, float *h_out, int platformID, int deviceID);
__declspec(dllexport) int convcorr3d_32f(size_t N1, size_t N2, size_t N3, float *h_image, float *h_psf, float *h_out, bool correlate, int platformID, int deviceID);
__declspec(dllexport) int conv3d_32f_lp(size_t N0, size_t N1, size_t N2, long long l_image, long long l_psf, long long l_output, bool correlate, long long l_context, long long l_queue, long long l_device);
__declspec(dllexport) int deconv3d_32f(int iterations, size_t N1, size_t N2, size_t N3, float *h_image, float *h_psf, float *h_out, float * normal);
__declspec(dllexport) int deconv3d_32f_tv(int iterations, float regularizationFactor, size_t N1, size_t N2, size_t N3, float *h_image, float *h_psf, float *h_out, float * normal);
__declspec(dllexport) int deconv3d_32f(int iterations, size_t N1, size_t N2, size_t N3, float *h_image, float *h_psf, float *h_out, float * normal, int platformID, int deviceID);
__declspec(dllexport) int deconv3d_32f_tv(int iterations, float regularizationFactor, size_t N1, size_t N2, size_t N3, float *h_image, float *h_psf, float *h_out, float * normal, int platformID, int deviceID);
__declspec(dllexport) int deconv3d_32f_lp(int iterations, size_t N0, size_t N1, size_t N2, long long d_image, long long d_psf, long long d_update, long long d_normal, long long l_context, long long l_queuee, long long l_device);
__declspec(dllexport) int deconv3d_32f_lp_tv(int iterations, float regularizationFactor, size_t N0, size_t N1, size_t N2, long long d_image, long long d_psf, long long d_update, long long d_normal, long long l_context, long long l_queuee, long long l_device);
__declspec(dllexport) int fft2d_32f(size_t N1, size_t N2, float *h_image, float * h_out);
Expand All @@ -16,14 +16,15 @@ extern "C" {
__declspec(dllexport)int fft3d_32f_lp(long long N0, long long N1, long long N2, long long d_image, long long d_out, long long l_context, long long l_queue);
__declspec(dllexport) int fft3dinv_32f_lp(long long N0, long long N1, long long N2, long long d_fft, long long d_out, long long l_context, long long l_queue);
__declspec(dllexport) int diagnostic();
__declspec(dllexport) int print_platforms_and_devices();
}
#else
extern "C" {
int conv3d_32f(size_t N1, size_t N2, size_t N3, float *h_image, float *h_psf, float *h_out);
int convcorr3d_32f(size_t N1, size_t N2, size_t N3, float *h_image, float *h_psf, float *h_out, bool correlate);
int conv3d_32f(size_t N1, size_t N2, size_t N3, float *h_image, float *h_psf, float *h_out, int platformID, int deviceID);
int convcorr3d_32f(size_t N1, size_t N2, size_t N3, float *h_image, float *h_psf, float *h_out, bool correlate, int platformID, int deviceID);
int conv3d_32f_lp(size_t N0, size_t N1, size_t N2, long long l_image, long long l_psf, long long l_output, bool correlate, long long l_context, long long l_queue, long long l_device);
int deconv3d_32f(int iterations, size_t N1, size_t N2, size_t N3, float *h_image, float *h_psf, float *h_out, float * normal);
int deconv3d_32f_tv(int iterations, float regularizationFactor, size_t N1, size_t N2, size_t N3, float *h_image, float *h_psf, float *h_out, float * normal);
int deconv3d_32f(int iterations, size_t N1, size_t N2, size_t N3, float *h_image, float *h_psf, float *h_out, float * normal, int platformID, int deviceID);
int deconv3d_32f_tv(int iterations, float regularizationFactor, size_t N1, size_t N2, size_t N3, float *h_image, float *h_psf, float *h_out, float * normal, int platformID, int deviceID);
int deconv3d_32f_lp(int iterations, size_t N0, size_t N1, size_t N2, long long d_image, long long d_psf, long long d_update, long long d_normal, long long l_context, long long l_queuee, long long l_device);
int deconv3d_32f_lp_tv(int iterations, float regularizationFactor, size_t N0, size_t N1, size_t N2, long long d_image, long long d_psf, long long d_update, long long d_normal, long long l_context, long long l_queuee, long long l_device);
int fft2d_32f(size_t N1, size_t N2, float *h_image, float * h_out);
Expand All @@ -33,6 +34,7 @@ extern "C" {
int fft2dinv_32f_lp(long long N1, long long N2, long long h_fft, long long h_out, long long l_context, long long l_queue);
int fft3dinv_32f_lp(long long N0, long long N1, long long N2, long long d_fft, long long d_out, long long l_context, long long l_queue);
int diagnostic();
int print_platforms_and_devices();
}
#endif

9 changes: 4 additions & 5 deletions native/test/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
project(test)

find_package(TIFF)
find_package(OpenCL)

# find Open CV
FIND_PACKAGE( OpenCV REQUIRED PATHS ${OPENCV_INSTALL_DIR})
INCLUDE_DIRECTORIES( ${OpenCV_INCLUDE_DIRS} )
INCLUDE_DIRECTORIES( ${OpenCV_INCLUDE_DIRS} ${OPENCL_INCLUDE_DIR})

# find clFFT path
FIND_PATH(CLFFT_LIBRARY_DIR $ENV{CLFFT_LIBRARY_DIR} [DOC "CLFFT library path"])
Expand All @@ -17,7 +16,7 @@ add_executable(deconvolutiontest deconvolutiontest.cpp tifstack.cpp ../clij2fft/
add_executable(convolvetest convolvetest.cpp tifstack.cpp ../clij2fft/clij2fft.cpp)
add_executable(makevariationkerneltest makevariationkerneltest.cpp ../clij2fft/clij2fft.cpp)

target_link_libraries(ffttest ${TIFF_LIBRARIES} ${OpenCV_LIBS} clFFT ${OpenCL_LIBRARY})
target_link_libraries(deconvolutiontest ${TIFF_LIBRARIES} ${OpenCV_LIBS} clFFT ${OpenCL_LIBRARY})
target_link_libraries(convolvetest ${TIFF_LIBRARIES} ${OpenCV_LIBS} clFFT ${OpenCL_LIBRARY})
target_link_libraries(ffttest ${OpenCV_LIBS} clFFT ${OpenCL_LIBRARY})
target_link_libraries(deconvolutiontest ${OpenCV_LIBS} clFFT ${OpenCL_LIBRARY})
target_link_libraries(convolvetest ${OpenCV_LIBS} clFFT ${OpenCL_LIBRARY})
target_link_libraries(makevariationkerneltest ${OpenCL_LIBRARY} clFFT)
24 changes: 11 additions & 13 deletions native/test/convolvetest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,46 +45,44 @@ int main() {
GetCurrentDir( buff, FILENAME_MAX );
cout<<"Current working dir: "<<buff<<"\n"<<flush;

unsigned int size[3];
unsigned int N0, N1, N2;

// load test images. These images are expected to be pre-conditioned
// 1. PSF and image extended to supported OpenCL FFT size
// 2. PSF and image saved as 32 bit float.
// 3. PSF center translated to 0,0,0.
float * img = read3DTiff("/home/bnorthan/code/images/barsext32f.tif", size);
float * psf = read3DTiff("/home/bnorthan/code/images/barspsfext32f.tif", size);
// 3. PSF center translated to 0,0,0.
float * img, *psf;
int rows, cols, slices;

openTifStack("D:\\images\\bars\\barsext32f.tif", &img, &N0, &N1, &N2);
openTifStack("D:\\images\\bars\\barspsfext32f.tif", &psf, &N0, &N1, &N2);

if (img==NULL) {
cout<<"File not found\n"<<flush;
return -1;
}

// input image size
unsigned int N0=size[0];
unsigned int N1=size[1];
unsigned int N2=size[2];

unsigned long n=N0*N1;

float * rescaled = new float[size[0]*size[1]];
float * rescaled = new float[n];

float * convolved = new float[N0*N1*N2];

for (int i=0;i<N0*N1*N2;i++) {
convolved[i]=img[i];
}

conv3d_32f(N0, N1, N2, img, psf, convolved);
conv3d_32f(N0, N1, N2, img, psf, convolved, 0, 0);

// rescale for visualization
rescale(img, rescaled, 1., n);
Mat cvImg(size[0], size[1], CV_32F, rescaled);
Mat cvImg(N0, N1, CV_32F, rescaled);
//namedWindow( "Display window", WINDOW_AUTOSIZE );// Create a window for display.
imshow( "Img", cvImg); // Show our image inside it.

// resclae for visualization
rescale(convolved, rescaled, 1., n);
Mat cvConv(size[0], size[1], CV_32F, rescaled);
Mat cvConv(N0, N1, CV_32F, rescaled);
//namedWindow( "Display window", WINDOW_AUTOSIZE );// Create a window for display.
imshow( "Conv", cvConv); // Show our image inside it.
waitKey(0);
Expand Down
Loading

0 comments on commit 270cdd7

Please sign in to comment.