From f897b458ea4345b03573cca21e2ef17da22ba5a7 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 30 Apr 2022 19:06:01 -0600 Subject: [PATCH] Fix szip handling --- .github/workflows/run_tests_osx.yml | 2 +- .github/workflows/run_tests_ubuntu.yml | 2 +- .github/workflows/run_tests_win_mingw.yml | 2 +- CMakeLists.txt | 34 +--- cmake/modules/FindSZIP.cmake | 179 ---------------------- cmake/modules/FindSzip.cmake | 64 ++++++++ docs/filters.md | 14 +- nc_test4/tst_specific_filters.sh | 7 + ncdump/tst_nccopy3.sh | 2 + ncdump/tst_output.sh | 1 + nczarr_test/Makefile.am | 8 +- 11 files changed, 95 insertions(+), 220 deletions(-) delete mode 100644 cmake/modules/FindSZIP.cmake create mode 100644 cmake/modules/FindSzip.cmake diff --git a/.github/workflows/run_tests_osx.yml b/.github/workflows/run_tests_osx.yml index 8e9d65a6fb..aa35ae8795 100644 --- a/.github/workflows/run_tests_osx.yml +++ b/.github/workflows/run_tests_osx.yml @@ -7,7 +7,7 @@ name: Run macOS-based netCDF Tests -on: [pull_request] +on: [pull_request,push] jobs: diff --git a/.github/workflows/run_tests_ubuntu.yml b/.github/workflows/run_tests_ubuntu.yml index a21ac5b001..826ac43589 100644 --- a/.github/workflows/run_tests_ubuntu.yml +++ b/.github/workflows/run_tests_ubuntu.yml @@ -4,7 +4,7 @@ name: Run Ubuntu/Linux netCDF Tests -on: [pull_request] +on: [pull_request,push] jobs: diff --git a/.github/workflows/run_tests_win_mingw.yml b/.github/workflows/run_tests_win_mingw.yml index 913920a3aa..aaf999e6a4 100644 --- a/.github/workflows/run_tests_win_mingw.yml +++ b/.github/workflows/run_tests_win_mingw.yml @@ -7,7 +7,7 @@ name: Run MSYS2, MinGW64-based Tests -on: [pull_request] +on: [pull_request,push] jobs: diff --git a/CMakeLists.txt b/CMakeLists.txt index 191b988b80..846ceb6416 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -623,26 +623,6 @@ ENDIF(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING) # 3. is nczarr enabled? # We need separate flags for cases 1 and 2 -# We need to determine if libsz is available both for HDF5 and NCZarr -# If user has specified the `SZIP_LIBRARY`, use it; otherwise try to find... -IF(NOT SZIP_LIBRARY) - FIND_LIBRARY(SZIP PATH NAMES szip sz sz2) - IF(SZIP) - SET(SZIP_LIBRARY ${SZIP}) - ELSE() - UNSET(SZIP_LIBRARY) - UNSET(SZIP) - ENDIF() -ENDIF() - -IF(SZIP_LIBRARY) - SET(SZIP_FOUND yes) - SET(HAVE_SZ yes) -ELSE() - SET(SZIP_FOUND no) - SET(HAVE_SZ no) -ENDIF() - ## # Option to Enable HDF5 # @@ -905,15 +885,6 @@ IF(USE_HDF5) int x = 1;}" USE_HDF5_SZIP) IF(USE_HDF5_SZIP) SET(HAVE_H5Z_SZIP yes) - # If user has specified the `SZIP_LIBRARY`, use it; otherwise try to find... - IF(SZIP_FOUND) - SET(CMAKE_REQUIRED_LIBRARIES ${SZIP_LIBRARY} ${CMAKE_REQUIRED_LIBRARIES}) - MESSAGE(STATUS "HDF5 has szip.") - ELSE() - MESSAGE(FATAL_ERROR "HDF5 Requires SZIP, but cannot find libszip or libsz.") - ENDIF() - ELSE() - SET(HAVE_H5Z_SZIP no) ENDIF() #### @@ -1124,13 +1095,15 @@ ENDIF() endmacro(set_std_filter) # Locate some compressors +FIND_PACKAGE(Szip) FIND_PACKAGE(Bz2) FIND_PACKAGE(Blosc) FIND_PACKAGE(Zstd) # Accumulate standard filters set(STD_FILTERS "deflate") # Always have deflate*/ -set_std_filter(SZIP) +set_std_filter(Szip) + SET(HAVE_SZ ${Szip_FOUND}) set_std_filter(Blosc) set_std_filter(Zstd) IF(Bz2_FOUND) @@ -1143,6 +1116,7 @@ ELSE() set(STD_FILTERS "${STD_FILTERS} bz2") ENDIF() + # If user wants, then install selected plugins SET(PLUGIN_INSTALL_DIR "" CACHE STRING "Whether and where we should install plugins") SET(ENABLE_PLUGIN_INSTALL OFF) diff --git a/cmake/modules/FindSZIP.cmake b/cmake/modules/FindSZIP.cmake deleted file mode 100644 index dc972e5bd0..0000000000 --- a/cmake/modules/FindSZIP.cmake +++ /dev/null @@ -1,179 +0,0 @@ - -# - Find SZIP library -# - Derived from the FindTiff.cmake that is included with cmake -# Find the native SZIP includes and library -# This module defines -# SZIP_INCLUDE_DIRS, where to find tiff.h, etc. -# SZIP_LIBRARIES, libraries to link against to use SZIP. -# SZIP_FOUND, If false, do not try to use SZIP. -# also defined, but not for general use are -# SZIP_LIBRARY, where to find the SZIP library. -# SZIP_LIBRARY_DEBUG - Debug version of SZIP library -# SZIP_LIBRARY_RELEASE - Release Version of SZIP library - -# MESSAGE (STATUS "Finding SZIP library and headers..." ) - -############################################ -# -# Check the existence of the libraries. -# -############################################ -# This macro was taken directly from the FindQt4.cmake file that is included -# with the CMake distribution. This is NOT my work. All work was done by the -# original authors of the FindQt4.cmake file. Only minor modifications were -# made to remove references to Qt and make this file more generally applicable -######################################################################### - -MACRO (SZIP_ADJUST_LIB_VARS basename) - IF (${basename}_INCLUDE_DIR) - - # if only the release version was found, set the debug variable also to the release version - IF (${basename}_LIBRARY_RELEASE AND NOT ${basename}_LIBRARY_DEBUG) - SET (${basename}_LIBRARY_DEBUG ${${basename}_LIBRARY_RELEASE}) - SET (${basename}_LIBRARY ${${basename}_LIBRARY_RELEASE}) - SET (${basename}_LIBRARIES ${${basename}_LIBRARY_RELEASE}) - ENDIF (${basename}_LIBRARY_RELEASE AND NOT ${basename}_LIBRARY_DEBUG) - - # if only the debug version was found, set the release variable also to the debug version - IF (${basename}_LIBRARY_DEBUG AND NOT ${basename}_LIBRARY_RELEASE) - SET (${basename}_LIBRARY_RELEASE ${${basename}_LIBRARY_DEBUG}) - SET (${basename}_LIBRARY ${${basename}_LIBRARY_DEBUG}) - SET (${basename}_LIBRARIES ${${basename}_LIBRARY_DEBUG}) - ENDIF (${basename}_LIBRARY_DEBUG AND NOT ${basename}_LIBRARY_RELEASE) - IF (${basename}_LIBRARY_DEBUG AND ${basename}_LIBRARY_RELEASE) - # if the generator supports configuration types then set - # optimized and debug libraries, or if the CMAKE_BUILD_TYPE has a value - IF (CMAKE_CONFIGURATION_TYPES OR CMAKE_BUILD_TYPE) - SET (${basename}_LIBRARY optimized ${${basename}_LIBRARY_RELEASE} debug ${${basename}_LIBRARY_DEBUG}) - ELSE(CMAKE_CONFIGURATION_TYPES OR CMAKE_BUILD_TYPE) - # if there are no configuration types and CMAKE_BUILD_TYPE has no value - # then just use the release libraries - SET (${basename}_LIBRARY ${${basename}_LIBRARY_RELEASE} ) - ENDIF (CMAKE_CONFIGURATION_TYPES OR CMAKE_BUILD_TYPE) - SET (${basename}_LIBRARIES optimized ${${basename}_LIBRARY_RELEASE} debug ${${basename}_LIBRARY_DEBUG}) - ENDIF (${basename}_LIBRARY_DEBUG AND ${basename}_LIBRARY_RELEASE) - - SET (${basename}_LIBRARY ${${basename}_LIBRARY} CACHE FILEPATH "The ${basename} library") - - IF (${basename}_LIBRARY) - SET (${basename}_FOUND 1) - ENDIF (${basename}_LIBRARY) - - ENDIF (${basename}_INCLUDE_DIR ) - - # Make variables changeble to the advanced user - MARK_AS_ADVANCED (${basename}_LIBRARY ${basename}_LIBRARY_RELEASE ${basename}_LIBRARY_DEBUG ${basename}_INCLUDE_DIR ) -ENDMACRO (SZIP_ADJUST_LIB_VARS) - - -# Look for the header file. -SET (SZIP_INCLUDE_SEARCH_DIRS - $ENV{SZIP_INSTALL}/include - $ENV{SZIP_INSTALL}/include/szip - /usr/include - /usr/include/szip -) - -SET (SZIP_LIB_SEARCH_DIRS - $ENV{SZIP_INSTALL}/lib - /usr/lib -) - -SET (SZIP_BIN_SEARCH_DIRS - $ENV{SZIP_INSTALL}/bin - /usr/bin -) - -FIND_PATH (SZIP_INCLUDE_DIR - NAMES szlib.h - PATHS ${SZIP_INCLUDE_SEARCH_DIRS} - NO_DEFAULT_PATH -) - -IF (WIN32 AND NOT MINGW) - SET (SZIP_SEARCH_DEBUG_NAMES "sz_d;libsz_d") - SET (SZIP_SEARCH_RELEASE_NAMES "sz;libsz;szip") -ELSE (WIN32 AND NOT MINGW) - SET (SZIP_SEARCH_DEBUG_NAMES "sz_d") - SET (SZIP_SEARCH_RELEASE_NAMES "sz") -ENDIF (WIN32 AND NOT MINGW) - -# Look for the library. -FIND_LIBRARY (SZIP_LIBRARY_DEBUG - NAMES ${SZIP_SEARCH_DEBUG_NAMES} - PATHS ${SZIP_LIB_SEARCH_DIRS} - NO_DEFAULT_PATH -) - -FIND_LIBRARY (SZIP_LIBRARY_RELEASE - NAMES ${SZIP_SEARCH_RELEASE_NAMES} - PATHS ${SZIP_LIB_SEARCH_DIRS} - NO_DEFAULT_PATH -) - -SZIP_ADJUST_LIB_VARS (SZIP) - -IF (SZIP_INCLUDE_DIR AND SZIP_LIBRARY) - SET (SZIP_FOUND 1) - SET (SZIP_LIBRARIES ${SZIP_LIBRARY}) - SET (SZIP_INCLUDE_DIRS ${SZIP_INCLUDE_DIR}) - IF (SZIP_LIBRARY_DEBUG) - GET_FILENAME_COMPONENT (SZIP_LIBRARY_PATH ${SZIP_LIBRARY_DEBUG} PATH) - SET (SZIP_LIB_DIR ${SZIP_LIBRARY_PATH}) - ELSEIF (SZIP_LIBRARY_RELEASE) - GET_FILENAME_COMPONENT (SZIP_LIBRARY_PATH ${SZIP_LIBRARY_RELEASE} PATH) - SET (SZIP_LIB_DIR ${SZIP_LIBRARY_PATH}) - ENDIF (SZIP_LIBRARY_DEBUG) - -ELSE (SZIP_INCLUDE_DIR AND SZIP_LIBRARY) - SET (SZIP_FOUND 0) - SET (SZIP_LIBRARIES) - SET (SZIP_INCLUDE_DIRS) -ENDIF (SZIP_INCLUDE_DIR AND SZIP_LIBRARY) - -# Report the results. -IF (NOT SZIP_FOUND) - SET (SZIP_DIR_MESSAGE - "SZip was not found. Make sure SZIP_LIBRARY and SZIP_INCLUDE_DIR are set or set the SZIP_INSTALL environment variable." - ) - IF (NOT SZIP_FIND_QUIETLY) - MESSAGE (STATUS "${SZIP_DIR_MESSAGE}") - ELSE (NOT SZIP_FIND_QUIETLY) - IF (SZIP_FIND_REQUIRED) - MESSAGE (FATAL_ERROR "SZip was NOT found and is Required by this project") - ENDIF (SZIP_FIND_REQUIRED) - ENDIF (NOT SZIP_FIND_QUIETLY) -ENDIF (NOT SZIP_FOUND) - -IF (SZIP_FOUND) - INCLUDE (CheckSymbolExists) - ############################################# - # Find out if SZIP was build using dll's - ############################################# - # Save required variable - SET (CMAKE_REQUIRED_INCLUDES_SAVE ${CMAKE_REQUIRED_INCLUDES}) - SET (CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS}) - # Add SZIP_INCLUDE_DIR to CMAKE_REQUIRED_INCLUDES - SET (CMAKE_REQUIRED_INCLUDES "${CMAKE_REQUIRED_INCLUDES};${SZIP_INCLUDE_DIRS}") - - CHECK_SYMBOL_EXISTS (SZIP_BUILT_AS_DYNAMIC_LIB "SZconfig.h" HAVE_SZIP_DLL) - - IF (HAVE_SZIP_DLL STREQUAL "TRUE") - SET (HAVE_SZIP_DLL "1") - ENDIF (HAVE_SZIP_DLL STREQUAL "TRUE") - - # Restore CMAKE_REQUIRED_INCLUDES and CMAKE_REQUIRED_FLAGS variables - SET (CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES_SAVE}) - SET (CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE}) - # - ############################################# -ENDIF (SZIP_FOUND) - -IF (FIND_SZIP_DEBUG) - MESSAGE (STATUS "SZIP_INCLUDE_DIR: ${SZIP_INCLUDE_DIR}") - MESSAGE (STATUS "SZIP_INCLUDE_DIRS: ${SZIP_INCLUDE_DIRS}") - MESSAGE (STATUS "SZIP_LIBRARY_DEBUG: ${SZIP_LIBRARY_DEBUG}") - MESSAGE (STATUS "SZIP_LIBRARY_RELEASE: ${SZIP_LIBRARY_RELEASE}") - MESSAGE (STATUS "HAVE_SZIP_DLL: ${HAVE_SZIP_DLL}") - MESSAGE (STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") -ENDIF (FIND_SZIP_DEBUG) diff --git a/cmake/modules/FindSzip.cmake b/cmake/modules/FindSzip.cmake new file mode 100644 index 0000000000..4deca3347a --- /dev/null +++ b/cmake/modules/FindSzip.cmake @@ -0,0 +1,64 @@ +# Searches for an installation of the szip library. On success, it sets the following variables: +# +# Szip_FOUND Set to true to indicate the szip library was found +# Szip_INCLUDE_DIRS The directory containing the header file szip/szip.h +# Szip_LIBRARIES The libraries needed to use the szip library +# +# To specify an additional directory to search, set Szip_ROOT. +# +# Author: Siddhartha Chaudhuri, 2009 +# + +# Look for the header, first in the user-specified location and then in the system locations +SET(Szip_INCLUDE_DOC "The directory containing the header file szip.h") +FIND_PATH(Szip_INCLUDE_DIRS NAMES szlib.h szip.h szip/szip.h PATHS ${Szip_ROOT} ${Szip_ROOT}/include DOC ${Szip_INCLUDE_DOC} NO_DEFAULT_PATH) +IF(NOT Szip_INCLUDE_DIRS) # now look in system locations + FIND_PATH(Szip_INCLUDE_DIRS NAMES szlib.h szip.h szip/szip.h DOC ${Szip_INCLUDE_DOC}) +ENDIF(NOT Szip_INCLUDE_DIRS) + +SET(Szip_FOUND FALSE) + +IF(Szip_INCLUDE_DIRS) + SET(Szip_LIBRARY_DIRS ${Szip_INCLUDE_DIRS}) + + IF("${Szip_LIBRARY_DIRS}" MATCHES "/include$") + # Strip off the trailing "/include" in the path. + GET_FILENAME_COMPONENT(Szip_LIBRARY_DIRS ${Szip_LIBRARY_DIRS} PATH) + ENDIF("${Szip_LIBRARY_DIRS}" MATCHES "/include$") + + IF(EXISTS "${Szip_LIBRARY_DIRS}/lib") + SET(Szip_LIBRARY_DIRS ${Szip_LIBRARY_DIRS}/lib) + ENDIF(EXISTS "${Szip_LIBRARY_DIRS}/lib") + + # Find Szip libraries + FIND_LIBRARY(Szip_DEBUG_LIBRARY NAMES szipd szip_d libszipd libszip_d szip libszip sz2 libsz2 + PATH_SUFFIXES Debug ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Debug + PATHS ${Szip_LIBRARY_DIRS} NO_DEFAULT_PATH) + FIND_LIBRARY(Szip_RELEASE_LIBRARY NAMES szip libszip sz libsz sz2 libsz2 + PATH_SUFFIXES Release ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Release + PATHS ${Szip_LIBRARY_DIRS} NO_DEFAULT_PATH) + + SET(Szip_LIBRARIES ) + IF(Szip_DEBUG_LIBRARY AND Szip_RELEASE_LIBRARY) + SET(Szip_LIBRARIES debug ${Szip_DEBUG_LIBRARY} optimized ${Szip_RELEASE_LIBRARY}) + ELSEIF(Szip_DEBUG_LIBRARY) + SET(Szip_LIBRARIES ${Szip_DEBUG_LIBRARY}) + ELSEIF(Szip_RELEASE_LIBRARY) + SET(Szip_LIBRARIES ${Szip_RELEASE_LIBRARY}) + ENDIF(Szip_DEBUG_LIBRARY AND Szip_RELEASE_LIBRARY) + + IF(Szip_LIBRARIES) + SET(Szip_FOUND TRUE) + ENDIF(Szip_LIBRARIES) +ENDIF(Szip_INCLUDE_DIRS) + +IF(Szip_FOUND) +# IF(NOT Szip_FIND_QUIETLY) + MESSAGE(STATUS "Found Szip: headers at ${Szip_INCLUDE_DIRS}, libraries at ${Szip_LIBRARY_DIRS}") + MESSAGE(STATUS " library is ${Szip_LIBRARIES}") +# ENDIF(NOT Szip_FIND_QUIETLY) +ELSE(Szip_FOUND) + IF(Szip_FIND_REQUIRED) + MESSAGE(FATAL_ERROR "Szip library not found") + ENDIF(Szip_FIND_REQUIRED) +ENDIF(Szip_FOUND) diff --git a/docs/filters.md b/docs/filters.md index e0335151fe..00dbe59304 100644 --- a/docs/filters.md +++ b/docs/filters.md @@ -935,10 +935,16 @@ This is for internal use only. As part of the overall build process, a number of filters are built as shared libraries in the "plugins" directory — in that directory or the "plugins/.libs" subdirectory. -An option exists to allow some of those filters to be installed into a user-specified directory. -Presumably this directory is part of the value of the HDF5_PLUGIN_PATH environment variable. -The ./configure option is ````--with-plugin-dir=<absolute-directory-path>````. -The corresponding CMake option is ````-DPLUGIN_INSTALL_DIR=<absolute-directory-path>````. +An option exists to allow some of those filters to be installed into a user-specified directory. The relevant options are as follows: +```` +./configure: --with-plugin-dir= +cmake: -DPLUGIN_INSTALL_DIR= +```` +If the value of the environment variable "HDF5_PLUGIN_PATH" is a single directory, then +a good value for the install directory is "$HDF5_PLUGIN_PATH", so for example: +```` +./configure ... --with-plugin-dir="$HDF5_PLUGIN_DIR" +```` If this option is specified, then as part of the "install" build action, a specified set of filter shared libraries will be copied into the specified directory. diff --git a/nc_test4/tst_specific_filters.sh b/nc_test4/tst_specific_filters.sh index 43235d2869..496f0b03a1 100755 --- a/nc_test4/tst_specific_filters.sh +++ b/nc_test4/tst_specific_filters.sh @@ -8,6 +8,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh +set -x set -e if test "x$TESTNCZARR" = x1 ; then @@ -96,6 +97,12 @@ rm -f $file fi setfilter $zfilt ref_any.cdl "tmp_filt_${zfilt}.cdl" "$zparams" "$zcodec" if test "x$TESTNCZARR" = x1 ; then +if test "x$zfilt" = xszip ; then +echo "@@@" +find /usr -name 'libsz*' +find ${HDF5_PLUGIN_DIR} -name '*sz*.so' +ldd ${HDF5_PLUGIN_DIR}/libh5szip.so +fi ${NCGEN} -4 -lb -o $fileurl "tmp_filt_${zfilt}.cdl" ${NCDUMP} -n $zfilt -sF $fileurl > "tmp_filt_${zfilt}.tmp" else diff --git a/ncdump/tst_nccopy3.sh b/ncdump/tst_nccopy3.sh index 449e896865..7dcfc910c5 100755 --- a/ncdump/tst_nccopy3.sh +++ b/ncdump/tst_nccopy3.sh @@ -8,7 +8,9 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh +set -x set -e + echo "" # get some config.h parameters diff --git a/ncdump/tst_output.sh b/ncdump/tst_output.sh index 40c7e7cd47..18da9c3e6f 100755 --- a/ncdump/tst_output.sh +++ b/ncdump/tst_output.sh @@ -4,6 +4,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh # This shell script tests the output from several previous tests. +set -x set -e echo "" diff --git a/nczarr_test/Makefile.am b/nczarr_test/Makefile.am index 6563344861..4e8ab47122 100644 --- a/nczarr_test/Makefile.am +++ b/nczarr_test/Makefile.am @@ -10,10 +10,10 @@ TESTS_ENVIRONMENT = #TEST_EXTENSIONS = .sh -#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -#TESTS_ENVIRONMENT += export SETX=1; +SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +TESTS_ENVIRONMENT += export SETX=1; AM_CPPFLAGS += -I${top_srcdir} -I${top_srcdir}/libnczarr AM_LDFLAGS += ${top_builddir}/liblib/libnetcdf.la