diff --git a/.github/workflows/run_tests_cdash.yml b/.github/workflows/run_tests_cdash.yml new file mode 100644 index 0000000000..b8d10ef118 --- /dev/null +++ b/.github/workflows/run_tests_cdash.yml @@ -0,0 +1,179 @@ +### +# Build hdf5 dependencies and cache them in a combined directory. +### + +name: Run CDash Ubuntu/Linux netCDF Tests + +on: workflow_dispatch + +concurrency: + group: ${{ github.workflow}}-${{ github.head_ref }} + cancel-in-progress: true + +jobs: + + build-deps-cdash: + + runs-on: ubuntu-latest + + strategy: + matrix: + hdf5: [ 1.10.8, 1.12.2, 1.14.0 ] + + steps: + - uses: actions/checkout@v3 + + - name: Install System dependencies + shell: bash -l {0} + run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev doxygen openssl + + ### + # Installing libhdf5 + ### + - name: Cache libhdf5-${{ matrix.hdf5 }} + id: cache-hdf5 + uses: actions/cache@v3 + with: + path: ~/environments/${{ matrix.hdf5 }} + key: hdf5-${{ runner.os }}-${{ matrix.hdf5 }} + + + - name: Build libhdf5-${{ matrix.hdf5 }} + if: steps.cache-hdf5.outputs.cache-hit != 'true' + run: | + set -x + + wget https://support.hdfgroup.org/ftp/HDF/releases/HDF4.2.15/src/hdf-4.2.15.tar.bz2 + tar -jxf hdf-4.2.15.tar.bz2 + pushd hdf-4.2.15 + ./configure --prefix=${HOME}/environments/${{ matrix.hdf5 }} --disable-static --enable-shared --disable-fortran --disable-netcdf --with-szlib --enable-hdf4-xdr + make -j + make install -j + popd + + wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$(echo ${{ matrix.hdf5 }} | cut -d. -f 1,2)/hdf5-${{ matrix.hdf5 }}/src/hdf5-${{ matrix.hdf5 }}.tar.bz2 + tar -jxf hdf5-${{ matrix.hdf5 }}.tar.bz2 + pushd hdf5-${{ matrix.hdf5 }} + ./configure --disable-static --enable-shared --prefix=${HOME}/environments/${{ matrix.hdf5 }} --enable-hl --with-szlib + make -j + make install -j + popd + + + build-deps-parallel: + + runs-on: ubuntu-latest + + strategy: + matrix: + hdf5: [ 1.14.0 ] + + steps: + + - uses: actions/checkout@v3 + + - name: Install System dependencies + shell: bash -l {0} + run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev mpich libmpich-dev + + ### + # Installing libhdf5 + ### + - name: Cache libhdf5-parallel-${{ matrix.hdf5 }} + id: cache-hdf5 + uses: actions/cache@v3 + with: + path: ~/environments/${{ matrix.hdf5 }} + key: hdf5-parallel-${{ runner.os }}-${{ matrix.hdf5 }} + + + - name: Build libhdf5-${{ matrix.hdf5 }}-pnetcdf-1.12.3 + if: steps.cache-hdf5.outputs.cache-hit != 'true' + run: | + set -x + + wget https://support.hdfgroup.org/ftp/HDF/releases/HDF4.2.15/src/hdf-4.2.15.tar.bz2 + tar -jxf hdf-4.2.15.tar.bz2 + pushd hdf-4.2.15 + CC=mpicc ./configure --prefix=${HOME}/environments/${{ matrix.hdf5 }} --disable-static --enable-shared --disable-fortran --disable-netcdf --with-szlib --enable-parallel --enable-hdf4-xdr + make -j + make install -j + popd + + wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$(echo ${{ matrix.hdf5 }} | cut -d. -f 1,2)/hdf5-${{ matrix.hdf5 }}/src/hdf5-${{ matrix.hdf5 }}.tar.bz2 + tar -jxf hdf5-${{ matrix.hdf5 }}.tar.bz2 + pushd hdf5-${{ matrix.hdf5 }} + CC=mpicc ./configure --disable-static --enable-shared --prefix=${HOME}/environments/${{ matrix.hdf5 }} --enable-hl --with-szlib --enable-parallel + make -j + make install -j + popd + wget https://parallel-netcdf.github.io/Release/pnetcdf-1.12.3.tar.gz + tar -zxf pnetcdf-1.12.3.tar.gz + pushd pnetcdf-1.12.3 + CC=mpicc ./configure --disable-static --enable-shared --prefix=${HOME}/environments/${{ matrix.hdf5 }} + make -j + make install -j + popd + + + ### + # Run CTest Serial Script + ### + nc-ctest-serial: + needs: build-deps-cdash + runs-on: ubuntu-latest + environment: CDashCI + + strategy: + matrix: + hdf5: [ 1.10.8, 1.12.2, 1.14.0 ] + + steps: + - uses: actions/checkout@v3 + with: + CDASH_TOKEN: ${{ secrets.CDASH_TOKEN }} + env: + CDASH_TOKEN: ${{ secrets.CDASH_TOKEN }} + + - name: Install System dependencies + shell: bash -l {0} + run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev + + ### + # Set Environmental Variables + ### + + - run: echo "CMAKE_PREFIX_PATH=${HOME}/environments/${{ matrix.hdf5 }}/" >> $GITHUB_ENV + - run: echo "LD_LIBRARY_PATH=${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV + - run: echo "CTEST_OUTPUT_ON_FAILURE=1" >> $GITHUB_ENV + + ### + # Fetch Cache + ### + + - name: Fetch HDF Cache + id: cache-hdf5 + uses: actions/cache@v3 + with: + path: ~/environments/${{ matrix.hdf5 }} + key: hdf5-${{ runner.os }}-${{ matrix.hdf5 }} + + - name: Check Cache + shell: bash -l {0} + run: ls ${HOME}/environments/${{ matrix.hdf5 }} && ls ${HOME}/environments/${{ matrix.hdf5}}/lib + + - name: Run ctest serial script + shell: bash -l {0} + env: + CDASH_TOKEN: ${{ secrets.CDASH_TOKEN }} + run: | + mkdir build + cd build + LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ctest -j 12 -V -S ../ctest_scripts/ctest_serial.ctest + + - name: Verbose Output if CTest Failure + shell: bash -l {0} + run: | + cd build + LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ctest -j 12 --rerun-failed --output-on-failure -VV + if: ${{ failure() }} diff --git a/.github/workflows/run_tests_osx.yml b/.github/workflows/run_tests_osx.yml index c887c6b1e6..8f2d9d62af 100644 --- a/.github/workflows/run_tests_osx.yml +++ b/.github/workflows/run_tests_osx.yml @@ -6,9 +6,12 @@ name: Run macOS-based netCDF Tests - on: [pull_request,workflow_dispatch] +concurrency: + group: ${{ github.workflow}}-${{ github.head_ref }} + cancel-in-progress: true + jobs: build-deps-osx: diff --git a/.github/workflows/run_tests_s3.yml b/.github/workflows/run_tests_s3.yml index 0a1c942460..ddbb353c8e 100644 --- a/.github/workflows/run_tests_s3.yml +++ b/.github/workflows/run_tests_s3.yml @@ -11,6 +11,10 @@ name: Run S3 netCDF Tests (under Ubuntu Linux) on: [workflow_dispatch] +concurrency: + group: ${{ github.workflow}}-${{ github.head_ref }} + cancel-in-progress: true + jobs: build-deps-serial: diff --git a/.github/workflows/run_tests_ubuntu.yml b/.github/workflows/run_tests_ubuntu.yml index fcd2eba7db..64b91bb99a 100644 --- a/.github/workflows/run_tests_ubuntu.yml +++ b/.github/workflows/run_tests_ubuntu.yml @@ -6,6 +6,10 @@ name: Run Ubuntu/Linux netCDF Tests on: [pull_request, workflow_dispatch] +concurrency: + group: ${{ github.workflow}}-${{ github.head_ref }} + cancel-in-progress: true + jobs: build-deps-serial: @@ -196,6 +200,19 @@ jobs: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check -j if: ${{ success() }} + - name: Create source distribution + shell: bash -l {0} + if: ${{ success() }} + run: make dist -j + + - uses: actions/upload-artifact@v3 + with: + name: netcdf-c-autotools-source-distribution + path: | + *.tar* + *.zip + *.tgz + ## # Parallel ## @@ -449,7 +466,28 @@ jobs: use_nczarr: [ nczarr_off, nczarr_on ] steps: - - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 + with: + name: netcdf-c-autotools-source-distribution + + - name: Unpack source distribution + shell: bash -l {0} + run: | + if [ -f *.zip ]; + then + unzip *.zip + else + tar xvzf $(ls *.tar* *.tgz *.zip | head -1) + fi + ls -d netcdf-c* + for name in netcdf-c*; + do + if [ -d ${name} ]; + then + cd ${name} + break + fi + done - name: Install System dependencies shell: bash -l {0} @@ -498,11 +536,28 @@ jobs: - name: Run autoconf shell: bash -l {0} - run: autoreconf -if + run: | + for name in netcdf-c*; + do + if [ -d ${name} ]; + then + cd ${name} + break + fi + done + autoreconf -if - name: Configure shell: bash -l {0} run: | + for name in netcdf-c*; + do + if [ -d ${name} ]; + then + cd ${name} + break + fi + done current_directory="$(pwd)" mkdir ../build cd ../build && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} "${current_directory}/configure" ${ENABLE_HDF5} ${ENABLE_DAP} ${ENABLE_NCZARR} @@ -510,29 +565,56 @@ jobs: - name: Look at config.log if error shell: bash -l {0} - run: cd ../build && cat config.log + run: | + if [ -d ../build ]; + then + cd ../build + else + cd build + fi && cat config.log if: ${{ failure() }} - name: Print Summary shell: bash -l {0} - run: cd ../build && cat libnetcdf.settings + run: | + if [ -d ../build ]; + then + cd ../build + else + cd build + fi && cat libnetcdf.settings - name: Build Library and Utilities shell: bash -l {0} run: | - cd ../build && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make -j + if [ -d ../build ]; + then + cd ../build + else + cd build + fi && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make -j if: ${{ success() }} - name: Build Tests shell: bash -l {0} run: | - cd ../build && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check TESTS="" -j + if [ -d ../build ]; + then + cd ../build + else + cd build + fi && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check TESTS="" -j if: ${{ success() }} - name: Run Tests shell: bash -l {0} run: | - cd ../build && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check -j + if [ -d ../build ]; + then + cd ../build + else + cd build + fi && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check -j if: ${{ success() }} nc-cmake: diff --git a/.github/workflows/run_tests_win_cygwin.yml b/.github/workflows/run_tests_win_cygwin.yml index bc084a401f..6e124448ed 100644 --- a/.github/workflows/run_tests_win_cygwin.yml +++ b/.github/workflows/run_tests_win_cygwin.yml @@ -2,6 +2,10 @@ name: Run Cygwin-based tests on: [pull_request,workflow_dispatch] +concurrency: + group: ${{ github.workflow}}-${{ github.head_ref }} + cancel-in-progress: true + env: SHELLOPTS: igncr CHERE_INVOKING: 1 diff --git a/.github/workflows/run_tests_win_mingw.yml b/.github/workflows/run_tests_win_mingw.yml index f48e81e7fa..978275cf6c 100644 --- a/.github/workflows/run_tests_win_mingw.yml +++ b/.github/workflows/run_tests_win_mingw.yml @@ -4,13 +4,17 @@ # for information related to github runners. ### -name: Run MSYS2, MinGW64-based Tests +name: Run MSYS2, MinGW64-based Tests (Not Visual Studio) env: CPPFLAGS: "-D_BSD_SOURCE" on: [pull_request,workflow_dispatch] +concurrency: + group: ${{ github.workflow}}-${{ github.head_ref }} + cancel-in-progress: true + jobs: build-and-test-autotools: diff --git a/CMakeInstallation.cmake b/CMakeInstallation.cmake index 7463420b81..5967889c95 100644 --- a/CMakeInstallation.cmake +++ b/CMakeInstallation.cmake @@ -74,7 +74,7 @@ FIND_PROGRAM(NC_DPKG NAMES dpkg) IF(NC_DPKG) # Define a macro for getting the dpkg architecture. MACRO(getdpkg_arch arch) - exec_program("${NC_DPKG}" ARGS "--print-architecture" OUTPUT_VARIABLE "${arch}") + execute_process(COMMAND "${NC_DPKG}" "--print-architecture" OUTPUT_VARIABLE "${arch}" OUTPUT_STRIP_TRAILING_WHITESPACE) ENDMACRO(getdpkg_arch) getdpkg_arch(dpkg_arch) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0191f1454c..189f6d87c2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,9 +6,7 @@ # Set Project Properties ################################## -#Minimum required CMake Version cmake_minimum_required(VERSION 3.12.0) -# CMake 3.12: Use libraries specified in CMAKE_REQUIRED_LIBRARIES for check include macros #Project Name project(netCDF @@ -18,6 +16,14 @@ DESCRIPTION "NetCDF is a set of software libraries and machine-independent data ) set(PACKAGE "netCDF" CACHE STRING "") +# Backport of built-in `PROJECT_IS_TOP_LEVEL` from CMake 3.21 +if (NOT DEFINED NETCDF_IS_TOP_LEVEL) + set(NETCDF_IS_TOP_LEVEL OFF) + if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) + set(NETCDF_IS_TOP_LEVEL ON) + endif () +endif () + ##### # Version Info: # @@ -51,7 +57,7 @@ SET(NC_DISPATCH_VERSION 5) find_program(UNAME NAMES uname) IF(UNAME) macro(getuname name flag) - exec_program("${UNAME}" ARGS "${flag}" OUTPUT_VARIABLE "${name}") + execute_process(COMMAND "${UNAME}" "${flag}" OUTPUT_VARIABLE "${name}" OUTPUT_STRIP_TRAILING_WHITESPACE) endmacro(getuname) getuname(osname -s) getuname(osrel -r) @@ -267,7 +273,8 @@ IF(CMAKE_COMPILER_IS_GNUCC OR APPLE) # Coverage tests need to have optimization turned off. IF(ENABLE_COVERAGE_TESTS) - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -coverage -fprofile-arcs -ftest-coverage") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0 -coverage -fprofile-arcs -ftest-coverage") SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage") MESSAGE(STATUS "Coverage Tests: On.") ENDIF() @@ -1404,7 +1411,7 @@ IF(ENABLE_TESTS) SET(HOSTNAME_ARG "-s") ENDIF() IF(HOSTNAME_CMD) - EXEC_PROGRAM(${HOSTNAME_CMD} ARGS "${HOSTNAME_ARG}" OUTPUT_VARIABLE HOSTNAME) + EXECUTE_PROCESS(COMMAND ${HOSTNAME_CMD} "${HOSTNAME_ARG}" OUTPUT_VARIABLE HOSTNAME OUTPUT_STRIP_TRAILING_WHITESPACE) SET(NC_CTEST_SITE "${HOSTNAME}" CACHE STRING "Hostname of test machine.") ENDIF() @@ -2789,5 +2796,8 @@ install( #### # CPack inclusion must come last. -# INCLUDE(CPack) -INCLUDE(CMakeInstallation.cmake) +option(NETCDF_PACKAGE "Create netCDF-C package " ${NETCDF_IS_TOP_LEVEL}) + +if (NETCDF_PACKAGE) + include(CMakeInstallation.cmake) +endif() diff --git a/Makefile.am b/Makefile.am index 64648d6180..55907f7ef5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -211,7 +211,11 @@ install-data-hook: all-local: liblib/libnetcdf.la echo ${PACKAGE_VERSION} > VERSION if ENABLE_S3_TESTALL + rm -f ${abs_top_builddir}/tmp_@PLATFORMUID@.uids echo "@TESTUID@" >> ${abs_top_builddir}/s3cleanup_@PLATFORMUID@.uids + cat ${abs_top_builddir}/s3cleanup_@PLATFORMUID@.uids | sort | uniq > ${abs_top_builddir}/tmp_@PLATFORMUID@.uids + rm -f ${abs_top_builddir}/s3cleanup_@PLATFORMUID@.uids + mv ${abs_top_builddir}/tmp_@PLATFORMUID@.uids ${abs_top_builddir}/s3cleanup_@PLATFORMUID@.uids endif if ENABLE_S3_TESTALL diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 4ff7515ada..2b5189252c 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -9,6 +9,9 @@ This file contains a high-level description of this package's evolution. Release * Update the internal copy of tinyxml2 to latest code. See [Github #2771](https://github.com/Unidata/netcdf-c/pull/2771). * Mitigate the problem of remote/nczarr-related test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). +* Fix DAP2 proxy problems. See [Github #2764](https://github.com/Unidata/netcdf-c/pull/2764). +* Cleanup a number of misc issues. See [Github #2763](https://github.com/Unidata/netcdf-c/pull/2763). +* Mitigate the problem of test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). * Extend NCZarr to support unlimited dimensions. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). * Fix significant bug in the NCZarr cache management. See [Github #2737](https://github.com/Unidata/netcdf-c/pull/2737). * Fix default parameters for caching of NCZarr. See [Github #2734](https://github.com/Unidata/netcdf-c/pull/2734). diff --git a/configure.ac b/configure.ac index 19386fe141..a35fd764d2 100644 --- a/configure.ac +++ b/configure.ac @@ -857,129 +857,136 @@ AC_MSG_RESULT([${have_sz}]) ########## -# See if we have libzip for NCZarr -AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll],[have_zip=yes],[have_zip=no]) -AC_MSG_CHECKING([whether libzip library is available]) -AC_MSG_RESULT([${have_zip}]) -enable_nczarr_zip=${have_zip} # alias +## +# Check to see if we're using NCZarr. If not, we don't need to check for dependencies and such. +## if test "x$enable_nczarr" = xno ; then enable_nczarr_zip=no -fi +else -AC_MSG_CHECKING([whether nczarr zip support is enabled]) -AC_MSG_RESULT([${enable_nczarr_zip}]) + # See if we have libzip for NCZarr + AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll],[have_zip=yes],[have_zip=no]) + AC_MSG_CHECKING([whether libzip library is available]) + AC_MSG_RESULT([${have_zip}]) + enable_nczarr_zip=${have_zip} # alias -if test "x$enable_nczarr_zip" = xyes ; then -AC_DEFINE([ENABLE_NCZARR_ZIP], [1], [If true, then libzip found]) -fi -# Check for enabling of S3 support -AC_MSG_CHECKING([whether netcdf S3 support should be enabled]) -AC_ARG_ENABLE([s3], - [AS_HELP_STRING([--enable-s3], - [enable netcdf S3 support])]) -test "x$enable_s3" = xyes || enable_s3=no -AC_MSG_RESULT($enable_s3) -if test "x$enable_remote_functionality" = xno ; then - AC_MSG_WARN([--disable-remote-functionality => --disable-s3]) - enable_s3=no -fi - -# --enable-nczarr-s3 is synonym for --enable-s3 (but...) -AC_MSG_CHECKING([whether netcdf NCZarr S3 support should be enabled]) -AC_ARG_ENABLE([nczarr-s3], - [AS_HELP_STRING([--enable-nczarr-s3], - [(Deprecated) enable netcdf NCZarr S3 support; Deprecated in favor of --enable-s3])]) -AC_MSG_RESULT([$enable_nczarr_s3 (Deprecated) Please use --enable-s3)]) - -# Set enable_s3 instead of enable_nczarr_s3 -if test "x$enable_s3" = xno && test "x$enable_nczarr_s3" = xyes && test "x$enable_remote_functionality" = xyes; then -enable_s3=yes # back compatibility -fi -unset enable_nczarr_s3 - -# Note we check for the library after checking for enable_s3 -# because for some reason this fails if we unconditionally test for sdk -# and it is not available. Fix someday -S3LIBS="" -if test "x$enable_s3" = xyes ; then -# See if we have the s3 aws library -# Check for the AWS S3 SDK library - AC_LANG_PUSH([C++]) - AC_CHECK_LIB([aws-c-common], [aws_string_destroy], [enable_s3_aws=yes],[enable_s3_aws=no]) - if test "x$enable_s3_aws" = "xyes" ; then - S3LIBS="-laws-cpp-sdk-core -laws-cpp-sdk-s3" + AC_MSG_CHECKING([whether nczarr zip support is enabled]) + AC_MSG_RESULT([${enable_nczarr_zip}]) + + if test "x$enable_nczarr_zip" = xyes ; then + AC_DEFINE([ENABLE_NCZARR_ZIP], [1], [If true, then libzip found]) fi - AC_LANG_POP -else - enable_s3_aws=no -fi -AC_MSG_CHECKING([whether AWS S3 SDK library is available]) -AC_MSG_RESULT([$enable_s3_aws]) + # Check for enabling of S3 support + AC_MSG_CHECKING([whether netcdf S3 support should be enabled]) + AC_ARG_ENABLE([s3], + [AS_HELP_STRING([--enable-s3], + [enable netcdf S3 support])]) + test "x$enable_s3" = xyes || enable_s3=no + AC_MSG_RESULT($enable_s3) + + if test "x$enable_remote_functionality" = xno ; then + AC_MSG_WARN([--disable-remote-functionality => --disable-s3]) + enable_s3=no + fi -# Check for enabling forced use of Internal S3 library -AC_MSG_CHECKING([whether internal S3 support should be used]) -AC_ARG_ENABLE([s3-internal], - [AS_HELP_STRING([--enable-s3-internal], - [enable internal S3 support])]) -test "x$enable_s3_internal" = xyes || enable_s3_internal=no -AC_MSG_RESULT($enable_s3_internal) + # --enable-nczarr-s3 is synonym for --enable-s3 (but...) + AC_MSG_CHECKING([whether netcdf NCZarr S3 support should be enabled]) + AC_ARG_ENABLE([nczarr-s3], + [AS_HELP_STRING([--enable-nczarr-s3], + [(Deprecated) enable netcdf NCZarr S3 support; Deprecated in favor of --enable-s3])]) + AC_MSG_RESULT([$enable_nczarr_s3 (Deprecated) Please use --enable-s3)]) -if test "x$enable_s3_aws" = xno && test "x$enable_s3_internal" = xno ; then -AC_MSG_WARN([No S3 library available => S3 support disabled]) -enable_s3=no -fi + # Set enable_s3 instead of enable_nczarr_s3 + if test "x$enable_s3" = xno && test "x$enable_nczarr_s3" = xyes && test "x$enable_remote_functionality" = xyes; then + enable_s3=yes # back compatibility + fi + unset enable_nczarr_s3 + + # Note we check for the library after checking for enable_s3 + # because for some reason this fails if we unconditionally test for sdk + # and it is not available. Fix someday + S3LIBS="" + if test "x$enable_s3" = xyes ; then + # See if we have the s3 aws library + # Check for the AWS S3 SDK library + AC_LANG_PUSH([C++]) + AC_CHECK_LIB([aws-c-common], [aws_string_destroy], [enable_s3_aws=yes],[enable_s3_aws=no]) + if test "x$enable_s3_aws" = "xyes" ; then + S3LIBS="-laws-cpp-sdk-core -laws-cpp-sdk-s3" + fi + AC_LANG_POP + else + enable_s3_aws=no + fi -if test "x$enable_s3_aws" = xyes && test "x$enable_s3_internal" = xyes ; then -AC_MSG_WARN([Both aws-sdk-cpp and s3-internal enabled => use s3-internal.]) -enable_s3_aws=no -fi + AC_MSG_CHECKING([whether AWS S3 SDK library is available]) + AC_MSG_RESULT([$enable_s3_aws]) -if test "x$enable_s3_internal" = xyes ; then -if test "x$ISOSX" != xyes && test "x$ISMINGW" != xyes && test "x$ISMSVC" != xyes ; then -# Find crypto libraries if using ssl -AC_CHECK_LIB([ssl],[ssl_create_cipher_list]) -AC_CHECK_LIB([crypto],[SHA256]) -fi -fi + # Check for enabling forced use of Internal S3 library + AC_MSG_CHECKING([whether internal S3 support should be used]) + AC_ARG_ENABLE([s3-internal], + [AS_HELP_STRING([--enable-s3-internal], + [enable internal S3 support])]) + test "x$enable_s3_internal" = xyes || enable_s3_internal=no + AC_MSG_RESULT($enable_s3_internal) -# Check for enabling S3 testing -AC_MSG_CHECKING([what level of netcdf S3 testing should be enabled]) -AC_ARG_WITH([s3-testing], - [AS_HELP_STRING([--with-s3-testing=yes|no|public], - [control netcdf S3 testing])], - [], [with_s3_testing=public]) -AC_MSG_RESULT($with_s3_testing) + if test "x$enable_s3_aws" = xno && test "x$enable_s3_internal" = xno ; then + AC_MSG_WARN([No S3 library available => S3 support disabled]) + enable_s3=no + fi -# Disable S3 tests if S3 support is disabled -if test "x$enable_s3" = xno ; then - if test "x$with_s3_testing" != xno ; then - AC_MSG_WARN([S3 support is disabled => no testing]) - with_s3_testing=no - fi -fi + if test "x$enable_s3_aws" = xyes && test "x$enable_s3_internal" = xyes ; then + AC_MSG_WARN([Both aws-sdk-cpp and s3-internal enabled => use s3-internal.]) + enable_s3_aws=no + fi -if test "x$enable_s3" = xyes ; then - AC_DEFINE([ENABLE_S3], [1], [if true, build netcdf-c with S3 support enabled]) -fi + if test "x$enable_s3_internal" = xyes ; then + if test "x$ISOSX" != xyes && test "x$ISMINGW" != xyes && test "x$ISMSVC" != xyes ; then + # Find crypto libraries if using ssl + AC_CHECK_LIB([ssl],[ssl_create_cipher_list]) + AC_CHECK_LIB([crypto],[SHA256]) + fi + fi -if test "x$enable_s3_aws" = xyes ; then - LIBS="$LIBS$S3LIBS" - AC_DEFINE([ENABLE_S3_AWS], [1], [If true, then use aws S3 library]) -fi + # Check for enabling S3 testing + AC_MSG_CHECKING([what level of netcdf S3 testing should be enabled]) + AC_ARG_WITH([s3-testing], + [AS_HELP_STRING([--with-s3-testing=yes|no|public], + [control netcdf S3 testing])], + [], [with_s3_testing=public]) + AC_MSG_RESULT($with_s3_testing) + + # Disable S3 tests if S3 support is disabled + if test "x$enable_s3" = xno ; then + if test "x$with_s3_testing" != xno ; then + AC_MSG_WARN([S3 support is disabled => no testing]) + with_s3_testing=no + fi + fi -if test "x$enable_s3_internal" = xyes ; then - AC_DEFINE([ENABLE_S3_INTERNAL], [1], [If true, then use internal S3 library]) -fi + if test "x$enable_s3" = xyes ; then + AC_DEFINE([ENABLE_S3], [1], [if true, build netcdf-c with S3 support enabled]) + fi -AC_DEFINE_UNQUOTED([WITH_S3_TESTING], [$with_s3_testing], [control S3 testing.]) + if test "x$enable_s3_aws" = xyes ; then + LIBS="$LIBS$S3LIBS" + AC_DEFINE([ENABLE_S3_AWS], [1], [If true, then use aws S3 library]) + fi -if test "x$with_s3_testing" = xyes ; then - AC_MSG_WARN([*** DO NOT SPECIFY WITH_S3_TESTING=YES UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***]) - AC_DEFINE([ENABLE_S3_TESTALL], [yes], [control S3 testing.]) + if test "x$enable_s3_internal" = xyes ; then + AC_DEFINE([ENABLE_S3_INTERNAL], [1], [If true, then use internal S3 library]) + fi + + AC_DEFINE_UNQUOTED([WITH_S3_TESTING], [$with_s3_testing], [control S3 testing.]) + + if test "x$with_s3_testing" = xyes ; then + AC_MSG_WARN([*** DO NOT SPECIFY WITH_S3_TESTING=YES UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***]) + AC_DEFINE([ENABLE_S3_TESTALL], [yes], [control S3 testing.]) + fi fi # Check whether we want to enable strict null byte header padding. diff --git a/ctest_scripts/ctest_mpicc.ctest b/ctest_scripts/ctest_mpicc.ctest index 9bca2038d8..617ea49a22 100644 --- a/ctest_scripts/ctest_mpicc.ctest +++ b/ctest_scripts/ctest_mpicc.ctest @@ -10,7 +10,11 @@ SET(CTEST_SOURCE_DIRECTORY "..") SET(CTEST_BINARY_DIRECTORY ".") set(CDASH_TOKEN $ENV{CDASH_TOKEN}) -MESSAGE("Using cdash token: ${CDASH_TOKEN}") +IF (CDASH_TOKEN) + MESSAGE("CDASH TOKEN FOUND") +ELSE (CDASH_TOKEN) + MESSAGE("NO CDASH TOKEN FOUND") +ENDIF (CDASH_TOKEN) SITE_NAME(local_site_name) diff --git a/ctest_scripts/ctest_serial.ctest b/ctest_scripts/ctest_serial.ctest index 522ca0b8c8..71a6eb6969 100644 --- a/ctest_scripts/ctest_serial.ctest +++ b/ctest_scripts/ctest_serial.ctest @@ -10,9 +10,13 @@ SET(CTEST_SOURCE_DIRECTORY "..") SET(CTEST_BINARY_DIRECTORY ".") set(CDASH_TOKEN $ENV{CDASH_TOKEN}) -MESSAGE("Using cdash token: ${CDASH_TOKEN}") - +IF (CDASH_TOKEN) + MESSAGE("CDASH TOKEN FOUND") +ELSE (CDASH_TOKEN) + MESSAGE("NO CDASH TOKEN FOUND") +ENDIF (CDASH_TOKEN) + SITE_NAME(local_site_name) set(CTEST_SITE ${local_site_name}) @@ -29,7 +33,7 @@ find_program(CTEST_GIT_COMMAND NAMES git) find_program(CTEST_COVERAGE_COMMAND NAMES gcov) find_program(CTEST_MEMORYCHECK_COMMAND NAMES valgrind) -set(CTEST_BUILD_OPTIONS "-DENABLE_COVERAGE_TESTS=TRUE -DENABLE_ERANGE_FILL=TRUE -DENABLE_LOGGING=TRUE -DENABLE_BYTERANGE=TRUE -DENABLE_LARGE_FILE_TESTS=FALSE") +set(CTEST_BUILD_OPTIONS "-DENABLE_COVERAGE_TESTS=FALSE -DENABLE_ERANGE_FILL=TRUE -DENABLE_LOGGING=TRUE -DENABLE_BYTERANGE=TRUE -DENABLE_LARGE_FILE_TESTS=FALSE") set(CTEST_DROP_METHOD https) @@ -42,7 +46,6 @@ ctest_start("Experimental") ctest_configure() ctest_build() ctest_test() -ctest_coverage() if (NOT "${CDASH_TOKEN}" STREQUAL "") ctest_submit(HTTPHEADER "Authorization: Bearer ${CDASH_TOKEN}") endif() diff --git a/ctest_scripts/ctest_serial_coverage.ctest b/ctest_scripts/ctest_serial_coverage.ctest new file mode 100644 index 0000000000..9b8f90a201 --- /dev/null +++ b/ctest_scripts/ctest_serial_coverage.ctest @@ -0,0 +1,52 @@ +### +# Standard CTest Script for testing netCDF. +# Requires a CDash Token. +# +# Set the CDASH_TOKEN environmental variable. +# +### + +SET(CTEST_SOURCE_DIRECTORY "..") +SET(CTEST_BINARY_DIRECTORY ".") + +set(CDASH_TOKEN $ENV{CDASH_TOKEN}) +IF (CDASH_TOKEN) + MESSAGE("CDASH TOKEN FOUND") +ELSE (CDASH_TOKEN) + MESSAGE("NO CDASH TOKEN FOUND") +ENDIF (CDASH_TOKEN) + + +SITE_NAME(local_site_name) +set(CTEST_SITE ${local_site_name}) + +set(CTEST_BUILD_CONFIGURATION "Profiling") +set(CTEST_CMAKE_GENERATOR "Unix Makefiles") +set(CTEST_BUILD_NAME "default") +set(CTEST_BUILD_CONFIGURATION "Profiling") +set(CTEST_DROP_METHOD "https") +set(CTEST_DROP_SITE_CDASH TRUE) +set(CTEST_PROJECT_NAME netcdf-c) + +find_program(CMAKE_COMMAND cmake) +find_program(CTEST_GIT_COMMAND NAMES git) +find_program(CTEST_COVERAGE_COMMAND NAMES gcov) +find_program(CTEST_MEMORYCHECK_COMMAND NAMES valgrind) + +set(CTEST_BUILD_OPTIONS "-DENABLE_COVERAGE_TESTS=TRUE -DENABLE_ERANGE_FILL=TRUE -DENABLE_LOGGING=TRUE -DENABLE_BYTERANGE=TRUE -DENABLE_LARGE_FILE_TESTS=FALSE") + + +set(CTEST_DROP_METHOD https) +set(CTEST_DROP_SITE "cdash.unidata.ucar.edu:443") +set(CTEST_DROP_LOCATION "/submit.php?project=netcdf-c") + +set(CTEST_CONFIGURE_COMMAND "${CMAKE_COMMAND} -DCMAKE_BUILD_TYPE:STRING=${CTEST_BUILD_CONFIGURATION} ${CTEST_BUILD_OPTIONS} ${CTEST_SOURCE_DIRECTORY}") + +ctest_start("Experimental") +ctest_configure() +ctest_build() +ctest_test() +ctest_coverage() +if (NOT "${CDASH_TOKEN}" STREQUAL "") + ctest_submit(HTTPHEADER "Authorization: Bearer ${CDASH_TOKEN}") +endif() diff --git a/ctest_scripts/ctest_serial_dap.ctest b/ctest_scripts/ctest_serial_dap.ctest index 9ea3333459..80fe31264d 100644 --- a/ctest_scripts/ctest_serial_dap.ctest +++ b/ctest_scripts/ctest_serial_dap.ctest @@ -10,7 +10,11 @@ SET(CTEST_SOURCE_DIRECTORY "..") SET(CTEST_BINARY_DIRECTORY ".") set(CDASH_TOKEN $ENV{CDASH_TOKEN}) -MESSAGE("Using cdash token: ${CDASH_TOKEN}") +IF (CDASH_TOKEN) + MESSAGE("CDASH TOKEN FOUND") +ELSE (CDASH_TOKEN) + MESSAGE("NO CDASH TOKEN FOUND") +ENDIF (CDASH_TOKEN) SITE_NAME(local_site_name) diff --git a/examples/C/format.c b/examples/C/format.c index 291aad65b6..40413dbe3f 100644 --- a/examples/C/format.c +++ b/examples/C/format.c @@ -45,7 +45,7 @@ main() /* Create a bunch of phoney data so we have something to write in the example file. */ for (fp=(float *)temp, i=0; i */ } NCRCinfo; +/* Opaque structures */ +struct NCS3INFO; + #if defined(__cplusplus) extern "C" { #endif @@ -94,7 +97,7 @@ EXTERNL int NC_getactives3profile(NCURI* uri, const char** profilep); EXTERNL int NC_s3profilelookup(const char* profile, const char* key, const char** valuep); EXTERNL int NC_authgets3profile(const char* profile, struct AWSprofile** profilep); EXTERNL int NC_iss3(NCURI* uri); -EXTERNL int NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** newurlp); +EXTERNL int NC_s3urlrebuild(NCURI* url, struct NCS3INFO* s3, NCURI** newurlp); #if defined(__cplusplus) } diff --git a/include/ncs3sdk.h b/include/ncs3sdk.h index c06f0e39f1..771faa6666 100644 --- a/include/ncs3sdk.h +++ b/include/ncs3sdk.h @@ -6,6 +6,11 @@ #ifndef NCS3SDK_H #define NCS3SDK_H 1 +/* Track the server type, if known */ +typedef enum NCS3SVC {NCS3UNK=0, /* unknown */ + NCS3=1, /* s3.amazon.aws */ + NCS3GS=0 /* storage.googleapis.com */ +} NCS3SVC; typedef struct NCS3INFO { char* host; /* non-null if other*/ @@ -13,6 +18,7 @@ typedef struct NCS3INFO { char* bucket; /* bucket name */ char* rootkey; char* profile; + NCS3SVC svc; } NCS3INFO; #ifdef __cplusplus diff --git a/libdap4/d4curlfunctions.c b/libdap4/d4curlfunctions.c index eb1fe9fc6d..ee06e4cacd 100644 --- a/libdap4/d4curlfunctions.c +++ b/libdap4/d4curlfunctions.c @@ -3,6 +3,11 @@ * See netcdf/COPYRIGHT file for copying and redistribution conditions. *********************************************************************/ +/* WARNING: oc2/occurlfunctions.c and libdap4/d4curlfunctions.c +should be merged since they are essentially the same file. +In the meantime, changes to one should be propagated to the other. +*/ + #include "d4includes.h" #include "d4curlfunctions.h" @@ -123,33 +128,43 @@ set_curlflag(NCD4INFO* state, int flag) } } break; - case CURLOPT_USE_SSL: - case CURLOPT_SSLCERT: case CURLOPT_SSLKEY: - case CURLOPT_SSL_VERIFYPEER: case CURLOPT_SSL_VERIFYHOST: - { - struct ssl* ssl = &state->auth->ssl; + case CURLOPT_SSL_VERIFYPEER: /* VERIFYPEER == 0 => VERIFYHOST == 0 */ /* We need to have 2 states: default and a set value */ - /* So -1 => default, >= 0 => use value; */ - if(ssl->verifypeer >= 0) - SETCURLOPT(state, CURLOPT_SSL_VERIFYPEER, (OPTARG)(ssl->verifypeer)); + /* So -1 => default >= 0 => use value */ + if(state->auth->ssl.verifypeer >= 0) { + SETCURLOPT(state, CURLOPT_SSL_VERIFYPEER, (OPTARG)(state->auth->ssl.verifypeer)); + if(state->auth->ssl.verifypeer == 0) state->auth->ssl.verifyhost = 0; + } + break; + case CURLOPT_SSL_VERIFYHOST: #ifdef HAVE_LIBCURL_766 - if(ssl->verifyhost >= 0) - SETCURLOPT(state, CURLOPT_SSL_VERIFYHOST, (OPTARG)(ssl->verifyhost)); + if(state->auth->ssl.verifyhost >= 0) { + SETCURLOPT(state, CURLOPT_SSL_VERIFYHOST, (OPTARG)(state->auth->ssl.verifyhost)); + } #endif - if(ssl->certificate) - SETCURLOPT(state, CURLOPT_SSLCERT, ssl->certificate); - if(ssl->key) - SETCURLOPT(state, CURLOPT_SSLKEY, ssl->key); - if(ssl->keypasswd) + break; + case CURLOPT_SSLCERT: + if(state->auth->ssl.certificate) + SETCURLOPT(state, CURLOPT_SSLCERT, state->auth->ssl.certificate); + break; + case CURLOPT_SSLKEY: + if(state->auth->ssl.key) + SETCURLOPT(state, CURLOPT_SSLKEY, state->auth->ssl.key); + if(state->auth->ssl.keypasswd) /* libcurl prior to 7.16.4 used 'CURLOPT_SSLKEYPASSWD' */ - SETCURLOPT(state, CURLOPT_KEYPASSWD, ssl->keypasswd); - if(ssl->cainfo) - SETCURLOPT(state, CURLOPT_CAINFO, ssl->cainfo); - if(ssl->capath) - SETCURLOPT(state, CURLOPT_CAPATH, ssl->capath); - } - break; + SETCURLOPT(state, CURLOPT_SSLKEYPASSWD, state->auth->ssl.keypasswd); + break; + case CURLOPT_CAINFO: + if(state->auth->ssl.cainfo) + SETCURLOPT(state, CURLOPT_CAINFO, state->auth->ssl.cainfo); + break; + case CURLOPT_CAPATH: + if(state->auth->ssl.capath) + SETCURLOPT(state, CURLOPT_CAPATH, state->auth->ssl.capath); + break; + case CURLOPT_USE_SSL: + break; #ifdef HAVE_CURLOPT_BUFFERSIZE case CURLOPT_BUFFERSIZE: @@ -200,6 +215,12 @@ NCD4_set_flags_perlink(NCD4INFO* state) if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_COOKIEJAR); if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_USERPWD); if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_PROXY); + if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_SSL_VERIFYPEER); + if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_SSL_VERIFYHOST); + if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_SSLCERT); + if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_SSLKEY); + if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_CAINFO); + if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_CAPATH); if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_USE_SSL); if(ret == NC_NOERR) ret = set_curlflag(state, CURLOPT_FOLLOWLOCATION); if(ret == NC_NOERR) ret = set_curlflag(state, CURLOPT_MAXREDIRS); diff --git a/libdispatch/dfile.c b/libdispatch/dfile.c index 7dba910f30..7447db3511 100644 --- a/libdispatch/dfile.c +++ b/libdispatch/dfile.c @@ -1770,7 +1770,7 @@ static int check_create_mode(int mode) { int mode_format; - int mmap = 0; + int use_mmap = 0; int inmemory = 0; int diskless = 0; @@ -1781,17 +1781,17 @@ check_create_mode(int mode) if (mode_format && (mode_format & (mode_format - 1))) return NC_EINVAL; - mmap = ((mode & NC_MMAP) == NC_MMAP); + use_mmap = ((mode & NC_MMAP) == NC_MMAP); inmemory = ((mode & NC_INMEMORY) == NC_INMEMORY); diskless = ((mode & NC_DISKLESS) == NC_DISKLESS); /* NC_INMEMORY and NC_DISKLESS and NC_MMAP are all mutually exclusive */ if(diskless && inmemory) return NC_EDISKLESS; - if(diskless && mmap) return NC_EDISKLESS; - if(inmemory && mmap) return NC_EINMEMORY; + if(diskless && use_mmap) return NC_EDISKLESS; + if(inmemory && use_mmap) return NC_EINMEMORY; /* mmap is not allowed for netcdf-4 */ - if(mmap && (mode & NC_NETCDF4)) return NC_EINVAL; + if(use_mmap && (mode & NC_NETCDF4)) return NC_EINVAL; #ifndef USE_NETCDF4 /* If the user asks for a netCDF-4 file, and the library was built @@ -1973,7 +1973,7 @@ NC_open(const char *path0, int omode, int basepe, size_t *chunksizehintp, const NC_Dispatch* dispatcher = NULL; int inmemory = 0; int diskless = 0; - int mmap = 0; + int use_mmap = 0; char* path = NULL; NCmodel model; char* newpath = NULL; @@ -1989,17 +1989,17 @@ NC_open(const char *path0, int omode, int basepe, size_t *chunksizehintp, {stat = NC_EINVAL; goto done;} /* Capture the inmemory related flags */ - mmap = ((omode & NC_MMAP) == NC_MMAP); + use_mmap = ((omode & NC_MMAP) == NC_MMAP); diskless = ((omode & NC_DISKLESS) == NC_DISKLESS); inmemory = ((omode & NC_INMEMORY) == NC_INMEMORY); /* NC_INMEMORY and NC_DISKLESS and NC_MMAP are all mutually exclusive */ if(diskless && inmemory) {stat = NC_EDISKLESS; goto done;} - if(diskless && mmap) {stat = NC_EDISKLESS; goto done;} - if(inmemory && mmap) {stat = NC_EINMEMORY; goto done;} + if(diskless && use_mmap) {stat = NC_EDISKLESS; goto done;} + if(inmemory && use_mmap) {stat = NC_EINMEMORY; goto done;} /* mmap is not allowed for netcdf-4 */ - if(mmap && (omode & NC_NETCDF4)) {stat = NC_EINVAL; goto done;} + if(use_mmap && (omode & NC_NETCDF4)) {stat = NC_EINVAL; goto done;} /* Attempt to do file path conversion: note that this will do nothing if path is a 'file:...' url, so it will need to be diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index 937fd8bd08..61941d1d71 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -136,6 +136,7 @@ static const struct MACRODEF { {"xarray","mode",{"zarr", NULL}}, {"noxarray","mode",{"nczarr", "noxarray", NULL}}, {"zarr","mode",{"nczarr","zarr", NULL}}, +{"gs3","mode",{"gs3","nczarr",NULL}}, /* Google S3 API */ {NULL,NULL,{NULL}} }; @@ -196,6 +197,7 @@ static struct NCPROTOCOLLIST { {"dods","http","mode=dap2"}, {"dap4","http","mode=dap4"}, {"s3","s3","mode=s3"}, + {"gs3","gs3","mode=gs3"}, {NULL,NULL,NULL} /* Terminate search */ }; @@ -914,7 +916,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void /* If s3, then rebuild the url */ if(NC_iss3(uri)) { NCURI* newuri = NULL; - if((stat = NC_s3urlrebuild(uri,NULL,NULL,&newuri))) goto done; + if((stat = NC_s3urlrebuild(uri,NULL,&newuri))) goto done; ncurifree(uri); uri = newuri; } else if(strcmp(uri->protocol,"file")==0) { diff --git a/libdispatch/drc.c b/libdispatch/drc.c index b896d90bfb..77922b752f 100644 --- a/libdispatch/drc.c +++ b/libdispatch/drc.c @@ -437,7 +437,9 @@ rccompile(const char* filepath) NCURI* uri = NULL; char* nextline = NULL; NCglobalstate* globalstate = NC_getglobalstate(); - char* bucket = NULL; + NCS3INFO s3; + + memset(&s3,0,sizeof(s3)); if((ret=NC_readfile(filepath,tmp))) { nclog(NCLOGWARN, "Could not open configuration file: %s",filepath); @@ -484,9 +486,8 @@ rccompile(const char* filepath) if(NC_iss3(uri)) { NCURI* newuri = NULL; /* Rebuild the url to S3 "path" format */ - nullfree(bucket); - bucket = NULL; - if((ret = NC_s3urlrebuild(uri,&bucket,NULL,&newuri))) goto done; + NC_s3clear(&s3); + if((ret = NC_s3urlrebuild(uri,&s3,&newuri))) goto done; ncurifree(uri); uri = newuri; newuri = NULL; @@ -546,6 +547,7 @@ rccompile(const char* filepath) rcorder(rc); done: + NC_s3clear(&s3); if(contents) free(contents); ncurifree(uri); ncbytesfree(tmp); diff --git a/libdispatch/ds3util.c b/libdispatch/ds3util.c index df001f8023..5091b1a24e 100644 --- a/libdispatch/ds3util.c +++ b/libdispatch/ds3util.c @@ -30,6 +30,7 @@ #undef AWSDEBUG #define AWSHOST ".amazonaws.com" +#define GOOGLEHOST "storage.googleapis.com" enum URLFORMAT {UF_NONE=0, UF_VIRTUAL=1, UF_PATH=2, UF_S3=3, UF_OTHER=4}; @@ -44,15 +45,12 @@ Rebuild an S3 url into a canonical path-style url. If region is not in the host, then use specified region if provided, otherwise us-east-1. @param url (in) the current url -@param region (in) region to use if needed; NULL => us-east-1 - (out) region from url or the input region -@param bucketp (in) bucket to use if needed - (out) bucket from url +@param s3 (in/out) NCS3INFO structure @param pathurlp (out) the resulting pathified url string */ int -NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** newurlp) +NC_s3urlrebuild(NCURI* url, NCS3INFO* s3, NCURI** newurlp) { int i,stat = NC_NOERR; NClist* hostsegments = NULL; @@ -63,6 +61,7 @@ NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** ne char* host = NULL; char* path = NULL; char* region = NULL; + NCS3SVC svc = NCS3UNK; if(url == NULL) {stat = NC_EURL; goto done;} @@ -83,14 +82,27 @@ NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** ne Path: https://s3..amazonaws.com// (3) or: https://s3.amazonaws.com// -- region defaults to us-east-1 (4) S3: s3:/// (5) - Other: https://// (6) + Google: https://storage.googleapis.com// (6) + or: gs3:/// (7) + Other: https://// (8) */ if(url->host == NULL || strlen(url->host) == 0) {stat = NC_EURL; goto done;} + + /* Reduce the host to standard form such as s3.amazonaws.com by pulling out the + region and bucket from the host */ if(strcmp(url->protocol,"s3")==0 && nclistlength(hostsegments)==1) { /* Format (5) */ bucket = nclistremove(hostsegments,0); /* region unknown at this point */ + /* Host will be set to canonical form later */ + svc = NCS3; + } else if(strcmp(url->protocol,"gs3")==0 && nclistlength(hostsegments)==1) { /* Format (7) */ + bucket = nclistremove(hostsegments,0); + /* region unknown at this point */ + /* Host will be set to canonical form later */ + svc = NCS3GS; } else if(endswith(url->host,AWSHOST)) { /* Virtual or path */ + svc = NCS3; /* If we find a bucket as part of the host, then remove it */ switch (nclistlength(hostsegments)) { default: stat = NC_EURL; goto done; @@ -99,11 +111,11 @@ NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** ne /* bucket unknown at this point */ break; case 4: /* Format (2) or (3) */ - if(strcasecmp(nclistget(hostsegments,1),"s3")==0) { /* Format (2) */ + if(strcasecmp(nclistget(hostsegments,0),"s3")!=0) { /* Presume format (2) */ /* region unknown at this point */ - bucket = nclistremove(hostsegments,0); /* Note removeal */ + bucket = nclistremove(hostsegments,0); /* Make canonical */ } else if(strcasecmp(nclistget(hostsegments,0),"s3")==0) { /* Format (3) */ - region = strdup(nclistget(hostsegments,1)); + region = nclistremove(hostsegments,1); /* Make canonical */ /* bucket unknown at this point */ } else /* ! Format (2) and ! Format (3) => error */ {stat = NC_EURL; goto done;} @@ -111,20 +123,27 @@ NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** ne case 5: /* Format (1) */ if(strcasecmp(nclistget(hostsegments,1),"s3")!=0) {stat = NC_EURL; goto done;} - region = strdup(nclistget(hostsegments,2)); - bucket = strdup(nclistremove(hostsegments,0)); + /* Make canonical */ + region = nclistremove(hostsegments,2); + bucket = nclistremove(hostsegments,0); break; } - } else { /* Presume Format (6) */ + } else if(strcasecmp(url->host,GOOGLEHOST)==0) { /* Google (6) */ + if((host = strdup(url->host))==NULL) + {stat = NC_ENOMEM; goto done;} + /* region is unknown */ + /* bucket is unknown at this point */ + svc = NCS3GS; + } else { /* Presume Format (8) */ if((host = strdup(url->host))==NULL) {stat = NC_ENOMEM; goto done;} /* region is unknown */ /* bucket is unknown */ } - /* region = (1) from url, (2) inoutregion, (3) default */ - if(region == NULL) - region = (inoutregionp?nulldup(*inoutregionp):NULL); + /* region = (1) from url, (2) s3->region, (3) default */ + if(region == NULL && s3 != NULL) + region = nulldup(s3->region); if(region == NULL) { const char* region0 = NULL; /* Get default region */ @@ -133,23 +152,30 @@ NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** ne } if(region == NULL) {stat = NC_ES3; goto done;} - /* bucket = (1) from url, (2) inoutbucket */ + /* bucket = (1) from url, (2) s3->bucket */ if(bucket == NULL && nclistlength(pathsegments) > 0) { bucket = nclistremove(pathsegments,0); /* Get from the URL path; will reinsert below */ } - if(bucket == NULL) - bucket = (inoutbucketp?nulldup(*inoutbucketp):NULL); + if(bucket == NULL && s3 != NULL) + bucket = nulldup(s3->bucket); if(bucket == NULL) {stat = NC_ES3; goto done;} - if(host == NULL) { /* Construct the revised host */ + if(svc == NCS3) { + /* Construct the revised host */ + ncbytesclear(buf); ncbytescat(buf,"s3."); ncbytescat(buf,region); ncbytescat(buf,AWSHOST); + nullfree(host); host = ncbytesextract(buf); + } else if(svc == NCS3GS) { + nullfree(host); + host = strdup(GOOGLEHOST); } - /* Construct the revised path */ ncbytesclear(buf); + + /* Construct the revised path */ if(bucket != NULL) { ncbytescat(buf,"/"); ncbytescat(buf,bucket); @@ -159,10 +185,13 @@ NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** ne ncbytescat(buf,nclistget(pathsegments,i)); } path = ncbytesextract(buf); + /* complete the new url */ if((newurl=ncuriclone(url))==NULL) {stat = NC_ENOMEM; goto done;} ncurisetprotocol(newurl,"https"); + assert(host != NULL); ncurisethost(newurl,host); + assert(path != NULL); ncurisetpath(newurl,path); /* Rebuild the url->url */ ncurirebuild(newurl); @@ -171,9 +200,11 @@ NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** ne fprintf(stderr,">>> NC_s3urlrebuild: final=%s bucket=%s region=%s\n",uri->uri,bucket,region); #endif if(newurlp) {*newurlp = newurl; newurl = NULL;} - if(inoutbucketp) {*inoutbucketp = bucket; bucket = NULL;} - if(inoutregionp) {*inoutregionp = region; region = NULL;} - + if(s3 != NULL) { + s3->bucket = bucket; bucket = NULL; + s3->region = region; region = NULL; + s3->svc = svc; + } done: nullfree(region); nullfree(bucket) @@ -218,7 +249,7 @@ NC_s3urlprocess(NCURI* url, NCS3INFO* s3, NCURI** newurlp) s3->profile = strdup(profile0); /* Rebuild the URL to path format and get a usable region and optional bucket*/ - if((stat = NC_s3urlrebuild(url,&s3->bucket,&s3->region,&url2))) goto done; + if((stat = NC_s3urlrebuild(url,s3,&url2))) goto done; s3->host = strdup(url2->host); /* construct the rootkey minus the leading bucket */ pathsegments = nclistnew(); @@ -268,7 +299,7 @@ NC_s3clear(NCS3INFO* s3) } /* -Check if a url has indicators that signal an S3 url. +Check if a url has indicators that signal an S3 or Google S3 url. */ int @@ -277,13 +308,17 @@ NC_iss3(NCURI* uri) int iss3 = 0; if(uri == NULL) goto done; /* not a uri */ - /* is the protocol "s3"? */ + /* is the protocol "s3" or "gs3" ? */ if(strcasecmp(uri->protocol,"s3")==0) {iss3 = 1; goto done;} - /* Is "s3" in the mode list? */ - if(NC_testmode(uri,"s3")) {iss3 = 1; goto done;} + if(strcasecmp(uri->protocol,"gs3")==0) {iss3 = 1; goto done;} + /* Is "s3" or "gs3" in the mode list? */ + if(NC_testmode(uri,"s3")) {iss3 = 1; goto done;} + if(NC_testmode(uri,"gs3")) {iss3 = 1; goto done;} /* Last chance; see if host looks s3'y */ - if(endswith(uri->host,AWSHOST)) {iss3 = 1; goto done;} - + if(uri->host != NULL) { + if(endswith(uri->host,AWSHOST)) {iss3 = 1; goto done;} + if(strcasecmp(uri->host,GOOGLEHOST)==0) {iss3 = 1; goto done;} + } done: return iss3; } diff --git a/libdispatch/ncexhash.c b/libdispatch/ncexhash.c index ea3957a054..26375e8d8f 100644 --- a/libdispatch/ncexhash.c +++ b/libdispatch/ncexhash.c @@ -111,7 +111,7 @@ ncexinit(void) int i; bitmasks[0] = 0; for(i=1;ileaflen, leafavg); fprintf(stderr," load=%g",leafload); fprintf(stderr,"]\n"); - dirsize = (1<<(map->depth))*((unsigned long long)sizeof(void*)); + dirsize = (1ULL<<(map->depth))*((unsigned long long)sizeof(void*)); leafsize = (nleaves)*((unsigned long long)sizeof(NCexleaf)); total = dirsize + leafsize; fprintf(stderr,"\tsizeof(directory)=%llu sizeof(leaves)=%lld total=%lld\n", diff --git a/libdispatch/nch5s3comms.c b/libdispatch/nch5s3comms.c index fd473f64e5..ba09b931b0 100644 --- a/libdispatch/nch5s3comms.c +++ b/libdispatch/nch5s3comms.c @@ -96,6 +96,7 @@ /*****************/ +#include "ncs3sdk.h" #include "nch5s3comms.h" /* S3 Communications */ /****************/ @@ -1063,7 +1064,7 @@ NCH5_s3comms_s3r_execute(s3r_t *handle, const char* url, *---------------------------------------------------------------------------- */ s3r_t * -NCH5_s3comms_s3r_open(const char* root, const char *region, const char *access_id, const char* access_key) +NCH5_s3comms_s3r_open(const char* root, NCS3SVC svc, const char *region, const char *access_id, const char* access_key) { int ret_value = SUCCEED; size_t tmplen = 0; @@ -1092,10 +1093,15 @@ NCH5_s3comms_s3r_open(const char* root, const char *region, const char *access_i * RECORD THE ROOT PATH *************************************/ - /* Verify that the region is a substring of root */ - if(region != NULL && region[0] != '\0') { - if(strstr(root,region) == NULL) - HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "region not present in root path."); + switch (svc) { + case NCS3: + /* Verify that the region is a substring of root */ + if(region != NULL && region[0] != '\0') { + if(strstr(root,region) == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "region not present in root path."); + } + break; + default: break; } handle->rootpath = nulldup(root); diff --git a/libdispatch/nch5s3comms.h b/libdispatch/nch5s3comms.h index acfffb2858..7cc482df66 100644 --- a/libdispatch/nch5s3comms.h +++ b/libdispatch/nch5s3comms.h @@ -502,7 +502,7 @@ EXTERNL hrb_t *NCH5_s3comms_hrb_init_request(const char *resource, const char *h * DECLARATION OF S3REQUEST ROUTINES * *************************************/ -EXTERNL s3r_t *NCH5_s3comms_s3r_open(const char* root, const char* region, const char* id, const char* access_key); +EXTERNL s3r_t *NCH5_s3comms_s3r_open(const char* root, NCS3SVC svc, const char* region, const char* id, const char* access_key); EXTERNL int NCH5_s3comms_s3r_close(s3r_t *handle); diff --git a/libdispatch/nclistmgr.c b/libdispatch/nclistmgr.c index 48b4eeb584..4069e87698 100644 --- a/libdispatch/nclistmgr.c +++ b/libdispatch/nclistmgr.c @@ -81,8 +81,8 @@ free_NCList(void) int add_to_NCList(NC* ncp) { - int i; - int new_id; + unsigned int i; + unsigned int new_id; if(nc_filelist == NULL) { if (!(nc_filelist = calloc(1, sizeof(NC*)*NCFILELISTLENGTH))) return NC_ENOMEM; @@ -96,7 +96,7 @@ add_to_NCList(NC* ncp) if(new_id == 0) return NC_ENOMEM; /* no more slots */ nc_filelist[new_id] = ncp; numfiles++; - ncp->ext_ncid = (new_id << ID_SHIFT); + ncp->ext_ncid = (int)(new_id << ID_SHIFT); return NC_NOERR; } diff --git a/libdispatch/nclog.c b/libdispatch/nclog.c index 26472fdfe0..5f6b8be7d8 100644 --- a/libdispatch/nclog.c +++ b/libdispatch/nclog.c @@ -82,9 +82,11 @@ ncloginit(void) } /*! -Enable/Disable logging. +Enable logging messages to a given level. Set to NCLOGOFF to disable +all messages, NCLOGERR for errors only, NCLOGWARN for warnings and +errors, and so on -\param[in] tf If 1, then turn on logging, if 0, then turn off logging. +\param[in] level Messages above this level are ignored \return The previous value of the logging flag. */ @@ -136,8 +138,11 @@ ncvlog(int level, const char* fmt, va_list ap) const char* prefix; if(!nclogginginitialized) ncloginit(); - if(nclog_global.loglevel > level || nclog_global.nclogstream == NULL) - return; + + if(nclog_global.loglevel < level || nclog_global.nclogstream == NULL) { + return; + } + prefix = nctagname(level); fprintf(nclog_global.nclogstream,"%s: ",prefix); if(fmt != NULL) { diff --git a/libdispatch/ncs3sdk_h5.c b/libdispatch/ncs3sdk_h5.c index e9da587f11..e51f8d8110 100644 --- a/libdispatch/ncs3sdk_h5.c +++ b/libdispatch/ncs3sdk_h5.c @@ -15,8 +15,8 @@ #include "ncrc.h" #include "ncxml.h" -#include "nch5s3comms.h" #include "ncs3sdk.h" +#include "nch5s3comms.h" #define NCTRACING #ifdef NCTRACING @@ -181,7 +181,7 @@ NC_s3sdkcreateclient(NCS3INFO* info) if((stat = NC_s3profilelookup(info->profile, "aws_secret_access_key", &accesskey))) goto done; } if((s3client->rooturl = makes3rooturl(info))==NULL) {stat = NC_ENOMEM; goto done;} - s3client->h5s3client = NCH5_s3comms_s3r_open(s3client->rooturl,info->region,accessid,accesskey); + s3client->h5s3client = NCH5_s3comms_s3r_open(s3client->rooturl,info->svc,info->region,accessid,accesskey); if(s3client->h5s3client == NULL) {stat = NC_ES3; goto done;} done: @@ -613,6 +613,10 @@ HTTP/1.1 200 string string +#ifdef GOOGLES3 + string + string +#endif ... ... @@ -679,6 +683,8 @@ parse_listbucketresult(char* xml, unsigned long long xmllen, struct LISTOBJECTSV result->nextcontinuationtoken = trim(ncxml_text(x),RECLAIM); } else if(strcmp(elem,"StartAfter")==0) { result->startafter = trim(ncxml_text(x),RECLAIM); + } else if(strcmp(elem,"StartAfter")==0) { + result->startafter = trim(ncxml_text(x),RECLAIM); } else { nclog(NCLOGERR,"Unexpected Element: <%s>",elem); stat = NC_ES3; @@ -711,7 +717,7 @@ parse_object(ncxml_t root, NClist* objects) for(x=ncxml_child_first(root);x != NULL;x=ncxml_child_next(x)) { const char* elem = ncxml_name(x); - if(strcmp(elem,"ChecksumAlorithm")==0) { + if(strcmp(elem,"ChecksumAlgorithm")==0) { if((stat = parse_checksumalgorithm(x,object->checksumalgorithms))) goto done; } else if(strcmp(elem,"ETag")==0) { object->etag = trim(ncxml_text(x),RECLAIM); @@ -725,6 +731,10 @@ parse_object(ncxml_t root, NClist* objects) object->size = trim(ncxml_text(x),RECLAIM); } else if(strcmp(elem,"StorageClass")==0) { object->storageclass = trim(ncxml_text(x),RECLAIM); + } else if(strcmp(elem,"Generation")==0) { + /* Ignore */ + } else if(strcmp(elem,"MetaGeneration")==0) { + /* Ignore */ } else { nclog(NCLOGERR,"Unexpected Element: <%s>",elem); stat = NC_ES3; diff --git a/libdispatch/ncuri.c b/libdispatch/ncuri.c index 7bca11a228..efd6fa4492 100644 --- a/libdispatch/ncuri.c +++ b/libdispatch/ncuri.c @@ -1237,7 +1237,7 @@ removedups(NClist* list) /* look for dups for this entry */ for(j=nclistlength(list)-2;j>i;j-=2) { if(strcasecmp(nclistget(list,i),nclistget(list,j))==0 - && strcasecmp(nclistget(list,i+1),nclistget(list,j+1))) { + && strcasecmp(nclistget(list,i+1),nclistget(list,j+1))==0) { nclistremove(list,j+1); nclistremove(list,j); } } diff --git a/libhdf5/hdf5internal.c b/libhdf5/hdf5internal.c index 0f21ddf65d..5532ed0c1d 100644 --- a/libhdf5/hdf5internal.c +++ b/libhdf5/hdf5internal.c @@ -995,7 +995,7 @@ nc4_hdf5_get_chunk_cache(int ncid, size_t *sizep, size_t *nelemsp, if (H5Pget_cache(plistid, NULL, nelemsp, sizep, &dpreemption) < 0) return NC_EHDFERR; if (preemptionp) - *preemptionp = dpreemption; + *preemptionp = (float)dpreemption; return NC_NOERR; } diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 514d1e2ffb..0c2f5b5234 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -25,6 +25,7 @@ #ifdef ENABLE_HDF5_ROS3 #include +#include "ncs3sdk.h" #endif /*Nemonic */ @@ -883,12 +884,11 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) #ifdef ENABLE_BYTERANGE else if(h5->byterange) { /* Arrange to use the byte-range drivers */ char* newpath = NULL; - char* awsregion0 = NULL; #ifdef ENABLE_HDF5_ROS3 H5FD_ros3_fapl_t fa; - const char* profile0 = NULL; const char* awsaccessid0 = NULL; const char* awssecretkey0 = NULL; + const char* profile0 = NULL; int iss3 = NC_iss3(h5->uri); fa.version = H5FD_CURR_ROS3_FAPL_T_VERSION; @@ -898,9 +898,11 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) fa.secret_key[0] = '\0'; if(iss3) { - /* Rebuild the URL */ + NCS3INFO s3; NCURI* newuri = NULL; - if((retval = NC_s3urlrebuild(h5->uri,NULL,&awsregion0,&newuri))) goto exit; + /* Rebuild the URL */ + memset(&s3,0,sizeof(s3)); + if((retval = NC_s3urlrebuild(h5->uri,&s3,&newuri))) goto exit; if((newpath = ncuribuild(newuri,NULL,NULL,NCURISVC))==NULL) {retval = NC_EURL; goto exit;} ncurifree(h5->uri); @@ -909,22 +911,23 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) BAIL(retval); if((retval = NC_s3profilelookup(profile0,AWS_ACCESS_KEY_ID,&awsaccessid0))) BAIL(retval); - if((retval = NC_s3profilelookup(profile0,AWS_SECRET_ACCESS_KEY,&awssecretkey0))) + if((retval = NC_s3profilelookup(profile0,AWS_SECRET_ACCESS_KEY,&awssecretkey0))) BAIL(retval); - if(awsregion0 == NULL) - awsregion0 = strdup(S3_REGION_DEFAULT); + if(s3.region == NULL) + s3.region = strdup(S3_REGION_DEFAULT); if(awsaccessid0 == NULL || awssecretkey0 == NULL ) { /* default, non-authenticating, "anonymous" fapl configuration */ fa.authenticate = (hbool_t)0; } else { fa.authenticate = (hbool_t)1; - assert(awsregion0 != NULL && strlen(awsregion0) > 0); + assert(s3.region != NULL && strlen(s3.region) > 0); assert(awsaccessid0 != NULL && strlen(awsaccessid0) > 0); assert(awssecretkey0 != NULL && strlen(awssecretkey0) > 0); - strlcat(fa.aws_region,awsregion0,H5FD_ROS3_MAX_REGION_LEN); + strlcat(fa.aws_region,s3.region,H5FD_ROS3_MAX_REGION_LEN); strlcat(fa.secret_id, awsaccessid0, H5FD_ROS3_MAX_SECRET_ID_LEN); strlcat(fa.secret_key, awssecretkey0, H5FD_ROS3_MAX_SECRET_KEY_LEN); } + NC_s3clear(&s3); /* create and set fapl entry */ if(H5Pset_fapl_ros3(fapl_id, &fa) < 0) BAIL(NC_EHDFERR); @@ -938,7 +941,6 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) if ((h5->hdfid = nc4_H5Fopen((newpath?newpath:path), flags, fapl_id)) < 0) BAIL(NC_EHDFERR); nullfree(newpath); - nullfree(awsregion0); } #endif else { @@ -1481,7 +1483,7 @@ nc4_get_var_meta(NC_VAR_INFO_T *var) if ((H5Pget_chunk_cache(access_pid, &(var->chunkcache.nelems), &(var->chunkcache.size), &rdcc_w0)) < 0) BAIL(NC_EHDFERR); - var->chunkcache.preemption = rdcc_w0; + var->chunkcache.preemption = (float)rdcc_w0; /* Get the dataset creation properties. */ if ((propid = H5Dget_create_plist(hdf5_var->hdf_datasetid)) < 0) diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index 0a1273250b..6d37fe9099 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -2391,7 +2391,7 @@ nc_set_var_chunk_cache_ints(int ncid, int varid, int size, int nelems, real_nelems = nelems; if (preemption >= 0) - real_preemption = preemption / 100.; + real_preemption = (float)(preemption / 100.); return NC4_HDF5_set_var_chunk_cache(ncid, varid, real_size, real_nelems, real_preemption); diff --git a/libnczarr/zclose.c b/libnczarr/zclose.c index b70e86f794..fdef3f5c16 100644 --- a/libnczarr/zclose.c +++ b/libnczarr/zclose.c @@ -179,7 +179,7 @@ NCZ_zclose_var1(NC_VAR_INFO_T* var) } /** - * @internal Close resources for vars in a group. + * @internal Close nczarr resources for vars in a group. * * @param grp Pointer to group info struct. * diff --git a/libnczarr/zdebug.c b/libnczarr/zdebug.c index 2a1ad4efaf..3b1e0e9217 100644 --- a/libnczarr/zdebug.c +++ b/libnczarr/zdebug.c @@ -34,7 +34,7 @@ int zreport(int err, const char* msg, const char* file, const char* fcn, int line) { if(err == 0) return err; - ZLOG(NCLOGWARN,"!!! zreport: err=%d msg=%s",err,msg); + ZLOG(NCLOGWARN,"!!! zreport: err=%d msg=%s @ %s#%s:%d",err,msg,file,fcn,line); ncbacktrace(); return zbreakpoint(err); } diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c index 0d2596890c..8a14cdc694 100644 --- a/libnczarr/zsync.c +++ b/libnczarr/zsync.c @@ -1448,29 +1448,11 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) { int stat = NC_NOERR; int i,j; - char* varpath = NULL; - char* key = NULL; NCZ_FILE_INFO_T* zinfo = NULL; - NC_VAR_INFO_T* var = NULL; - NCZ_VAR_INFO_T* zvar = NULL; NCZMAP* map = NULL; - NCjson* jvar = NULL; - NCjson* jncvar = NULL; - NCjson* jdimrefs = NULL; - NCjson* jvalue = NULL; int purezarr = 0; int xarray = 0; int formatv1 = 0; - nc_type vtype; - int vtypelen; - size64_t* shapes = NULL; - int rank = 0; - int zarr_rank = 1; /* Need to watch out for scalars */ - NClist* dimnames = nclistnew(); -#ifdef ENABLE_NCZARR_FILTERS - NCjson* jfilter = NULL; - int chainindex; -#endif ZTRACE(3,"file=%s grp=%s |varnames|=%u",file->controller->path,grp->hdr.name,nclistlength(varnames)); @@ -1483,7 +1465,32 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) /* Load each var in turn */ for(i = 0; i < nclistlength(varnames); i++) { - const char* varname = nclistget(varnames,i); + /* per-variable info */ + NC_VAR_INFO_T* var = NULL; + NCZ_VAR_INFO_T* zvar = NULL; + NCjson* jvar = NULL; + NCjson* jncvar = NULL; + NCjson* jdimrefs = NULL; + NCjson* jvalue = NULL; + char* varpath = NULL; + char* key = NULL; + const char* varname = NULL; + size64_t* shapes = NULL; + NClist* dimnames = NULL; + int varsized = 0; + int suppress = 0; /* Abort processing of this variable */ + nc_type vtype = NC_NAT; + int vtypelen = 0; + int rank = 0; + int zarr_rank = 0; /* Need to watch out for scalars */ +#ifdef ENABLE_NCZARR_FILTERS + NCjson* jfilter = NULL; + int chainindex = 0; +#endif + + dimnames = nclistnew(); + varname = nclistget(varnames,i); + if((stat = nc4_var_list_add2(grp, varname, &var))) goto done; @@ -1522,6 +1529,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) if(version != zinfo->zarr.zarr_version) {stat = (THROW(NC_ENCZARR)); goto done;} } + /* Set the type and endianness of the variable */ { int endianness; @@ -1609,23 +1617,6 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) jdimrefs = NULL; } - /* shape */ - { - if((stat = NCJdictget(jvar,"shape",&jvalue))) goto done; - if(NCJsort(jvalue) != NCJ_ARRAY) {stat = (THROW(NC_ENCZARR)); goto done;} - if(zvar->scalar) { - rank = 0; - zarr_rank = 1; /* Zarr does not support scalars */ - } else - rank = (zarr_rank = NCJlength(jvalue)); - /* Save the rank of the variable */ - if((stat = nc4_var_set_ndims(var, rank))) goto done; - /* extract the shapes */ - if((shapes = (size64_t*)malloc(sizeof(size64_t)*zarr_rank)) == NULL) - {stat = (THROW(NC_ENOMEM)); goto done;} - if((stat = decodeints(jvalue, shapes))) goto done; - } - /* Capture dimension_separator (must precede chunk cache creation) */ { NCglobalstate* ngs = NC_getglobalstate(); @@ -1661,6 +1652,36 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) } } + /* shape */ + { + if((stat = NCJdictget(jvar,"shape",&jvalue))) goto done; + if(NCJsort(jvalue) != NCJ_ARRAY) {stat = (THROW(NC_ENCZARR)); goto done;} + + /* Process the rank */ + zarr_rank = NCJlength(jvalue); + if(zarr_rank == 0) { + /* suppress variable */ + ZLOG(NCLOGWARN,"Empty shape for variable %s suppressed",var->hdr.name); + suppress = 1; + goto suppressvar; + } + + if(zvar->scalar) { + rank = 0; + zarr_rank = 1; /* Zarr does not support scalars */ + } else + rank = (zarr_rank = NCJlength(jvalue)); + + if(zarr_rank > 0) { + /* Save the rank of the variable */ + if((stat = nc4_var_set_ndims(var, rank))) goto done; + /* extract the shapes */ + if((shapes = (size64_t*)malloc(sizeof(size64_t)*zarr_rank)) == NULL) + {stat = (THROW(NC_ENOMEM)); goto done;} + if((stat = decodeints(jvalue, shapes))) goto done; + } + } + /* chunks */ { size64_t chunks[NC_MAX_VAR_DIMS]; @@ -1668,8 +1689,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) if(jvalue != NULL && NCJsort(jvalue) != NCJ_ARRAY) {stat = (THROW(NC_ENCZARR)); goto done;} /* Verify the rank */ - assert (zarr_rank == NCJlength(jvalue)); - if(zvar->scalar) { + if(zvar->scalar || zarr_rank == 0) { if(var->ndims != 0) {stat = (THROW(NC_ENCZARR)); goto done;} zvar->chunkproduct = 1; @@ -1746,37 +1766,47 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) if((stat = NCZ_filter_build(file,var,jfilter,chainindex++))) goto done; } } + /* Suppress variable if there are filters and var is not fixed-size */ + if(varsized && nclistlength((NClist*)var->filters) > 0) + suppress = 1; #endif - if((stat = computedimrefs(file, var, purezarr, xarray, rank, dimnames, shapes, var->dim))) - goto done; - - if(!zvar->scalar) { - /* Extract the dimids */ - for(j=0;jdimids[j] = var->dim[j]->hdr.id; + if(zarr_rank > 0) { + if((stat = computedimrefs(file, var, purezarr, xarray, rank, dimnames, shapes, var->dim))) + goto done; + if(!zvar->scalar) { + /* Extract the dimids */ + for(j=0;jdimids[j] = var->dim[j]->hdr.id; + } } #ifdef ENABLE_NCZARR_FILTERS - /* At this point, we can finalize the filters */ - if((stat = NCZ_filter_setup(var))) goto done; + if(!suppress) { + /* At this point, we can finalize the filters */ + if((stat = NCZ_filter_setup(var))) goto done; + } #endif +suppressvar: + if(suppress) { + /* Reclaim NCZarr variable specific info */ + (void)NCZ_zclose_var1(var); + /* Remove from list of variables and reclaim the top level var object */ + (void)nc4_var_list_del(grp, var); + var = NULL; + } + /* Clean up from last cycle */ - nclistfreeall(dimnames); dimnames = nclistnew(); + nclistfreeall(dimnames); dimnames = NULL; nullfree(varpath); varpath = NULL; nullfree(shapes); shapes = NULL; + nullfree(key); key = NULL; if(formatv1) {NCJreclaim(jncvar); jncvar = NULL;} NCJreclaim(jvar); jvar = NULL; var = NULL; } done: - nullfree(shapes); - nullfree(varpath); - nullfree(key); - nclistfreeall(dimnames); - NCJreclaim(jvar); - if(formatv1) NCJreclaim(jncvar); return ZUNTRACE(THROW(stat)); } diff --git a/nc_test/tst_diskless.c b/nc_test/tst_diskless.c index 13ae21b354..d252de6d57 100644 --- a/nc_test/tst_diskless.c +++ b/nc_test/tst_diskless.c @@ -46,7 +46,7 @@ void fail(int line) { #endif /* Control flags */ -static int flags, persist, usenetcdf4, mmap, diskless; +static int flags, persist, usenetcdf4, use_mmap, diskless; char* smode(int mode) @@ -90,13 +90,13 @@ main(int argc, char **argv) /* Set defaults */ persist = 0; usenetcdf4 = 0; - mmap = 0; + use_mmap = 0; diskless = 0; for(i=1;i= min && value <= max)) { diff --git a/nc_test4/cdm_sea_soundings.c b/nc_test4/cdm_sea_soundings.c index f7da504ce9..b2e2f681cd 100644 --- a/nc_test4/cdm_sea_soundings.c +++ b/nc_test4/cdm_sea_soundings.c @@ -28,11 +28,11 @@ main(int argc, char **argv) /* Create phony data. */ for (i = 0; i < DIM_LEN; i++) { - if (!(data[i].temp_vl.p = malloc(sizeof(float) * (i + 1)))) + if (!(data[i].temp_vl.p = malloc(sizeof(float) * (size_t)(i + 1)))) return NC_ENOMEM; for (j = 0; j < i + 1; j++) - ((float *)(data[i].temp_vl.p))[j] = 23.5 - j; - data[i].temp_vl.len = i + 1; + ((float *)(data[i].temp_vl.p))[j] = 23.5f - (float)j; + data[i].temp_vl.len = (size_t)i + 1; } printf("\n*** Testing netcdf-4 CDM compliance: sea soundings.\n"); diff --git a/nc_test4/tst_atts3.c b/nc_test4/tst_atts3.c index 0c0bd621b2..84726014d4 100644 --- a/nc_test4/tst_atts3.c +++ b/nc_test4/tst_atts3.c @@ -1215,7 +1215,7 @@ create_file() check_err(stat,__LINE__,__FILE__); } { /* Gf */ - static const float Gf_att[5] = {-3.4028231e+38, 3.4028231e+38, -9.96921e+36, 9.96921e+36, 531} ; + static const float Gf_att[5] = {-3.4028231e+38f, 3.4028231e+38f, -9.96921e+36f, 9.96921e+36f, 531} ; stat = nc_put_att_float(ncid, NC_GLOBAL, "Gf", NC_FLOAT, 5, Gf_att); check_err(stat,__LINE__,__FILE__); } @@ -1247,7 +1247,7 @@ create_file() check_err(stat,__LINE__,__FILE__); } { /* f */ - static const float i_f_att[4] = {-3.4028231e+38, 3.4028231e+38, -9.96921e+36, 9.96921e+36} ; + static const float i_f_att[4] = {-3.4028231e+38f, 3.4028231e+38f, -9.96921e+36f, 9.96921e+36f} ; stat = nc_put_att_float(ncid, i_id, "f", NC_FLOAT, 4, i_f_att); check_err(stat,__LINE__,__FILE__); } @@ -1330,7 +1330,7 @@ create_file() } { - float fr_data[2] = {-3.4028231e+38, 3.4028231e+38} ; + float fr_data[2] = {-3.4028231e+38f, 3.4028231e+38f} ; size_t fr_startset[1] = {0} ; size_t fr_countset[1] = {2} ; stat = nc_put_vara(ncid, fr_id, fr_startset, fr_countset, fr_data); @@ -1378,7 +1378,7 @@ create_file() } { - float f1_data[1] = {-3.4028231e+38} ; + float f1_data[1] = {-3.4028231e+38f} ; size_t f1_startset[1] = {0} ; size_t f1_countset[1] = {1} ; stat = nc_put_vara(ncid, f1_id, f1_startset, f1_countset, f1_data); @@ -1426,7 +1426,7 @@ create_file() } { - float f2_data[2] = {-3.4028231e+38, 3.4028231e+38} ; + float f2_data[2] = {-3.4028231e+38f, 3.4028231e+38f} ; size_t f2_startset[1] = {0} ; size_t f2_countset[1] = {2} ; stat = nc_put_vara(ncid, f2_id, f2_startset, f2_countset, f2_data); @@ -1474,7 +1474,7 @@ create_file() } { - float f3_data[3] = {-3.4028231e+38, 3.4028231e+38, -9.96921e+36} ; + float f3_data[3] = {-3.4028231e+38f, 3.4028231e+38f, -9.96921e+36f} ; size_t f3_startset[1] = {0} ; size_t f3_countset[1] = {3} ; stat = nc_put_vara(ncid, f3_id, f3_startset, f3_countset, f3_data); @@ -1522,7 +1522,7 @@ create_file() } { - float f4_data[4] = {-3.4028231e+38, 3.4028231e+38, -9.96921e+36, 9.96921e+36} ; + float f4_data[4] = {-3.4028231e+38f, 3.4028231e+38f, -9.96921e+36f, 9.96921e+36f} ; size_t f4_startset[1] = {0} ; size_t f4_countset[1] = {4} ; stat = nc_put_vara(ncid, f4_id, f4_startset, f4_countset, f4_data); diff --git a/nc_test4/tst_bloscfail.sh b/nc_test4/tst_bloscfail.sh index a74a3f63e5..14dfb42d1c 100755 --- a/nc_test4/tst_bloscfail.sh +++ b/nc_test4/tst_bloscfail.sh @@ -60,7 +60,6 @@ ${NCCOPY} -4 -V three_dmn_rec_var -F *,32001,0,0,0,0,1,1,0 ./tmp_bloscx3.nc ./tm # This should fail because shuffle is off if ${NCCOPY} -4 -V three_dmn_rec_var -F *,32001,0,0,0,0,1,0,0 ./tmp_bloscx3.nc ./tmp_bloscx4_fail.nc ; then echo "*** not xfail: nccopy " - exit 1; else echo "*** xfail: nccopy " fi diff --git a/nc_test4/tst_chunks2.c b/nc_test4/tst_chunks2.c index f166a500eb..0499d6f392 100644 --- a/nc_test4/tst_chunks2.c +++ b/nc_test4/tst_chunks2.c @@ -50,7 +50,7 @@ calculate_waste(int ndims, size_t *dimlen, size_t *chunksize, float *waste) #endif /* Percent of the chunked file that is wasted space. */ - *waste = ((float)(chunked - unchunked) / (float)chunked) * 100.0; + *waste = ((float)(chunked - unchunked) / (float)chunked) * 100.0f; #ifdef PRINT_CHUNK_WASTE_REPORT printf("\ndimlen\tchunksize\tnum_chunks\n"); diff --git a/nc_test4/tst_compounds.c b/nc_test4/tst_compounds.c index f91c210021..731ed8bc4f 100644 --- a/nc_test4/tst_compounds.c +++ b/nc_test4/tst_compounds.c @@ -546,8 +546,8 @@ main(int argc, char **argv) hr_data_out[i].starfleet_id = i; hr_data_out[i].svc_rec = data[i]; if (sprintf(hr_data_out[i].name, "alien_%d", i) < 0) ERR; - hr_data_out[i].max_temp = 99.99; - hr_data_out[i].min_temp = -9.99; + hr_data_out[i].max_temp = 99.99f; + hr_data_out[i].min_temp = -9.99f; hr_data_out[i].percent_transporter_errosion = .030303; } @@ -659,10 +659,10 @@ main(int argc, char **argv) /* Initialize data. */ for (i = 0; i < DIM6_LEN; i++) { - obsdata[i].day = 15 * i + 1; - obsdata[i].elev = 2 * i + 1; + obsdata[i].day = 15 * (char)i + 1; + obsdata[i].elev = 2 * (short)i + 1; obsdata[i].count = 2 * i + 1; - obsdata[i].relhum = 2.0 * i + 1; + obsdata[i].relhum = 2.0f * (float)i + 1; obsdata[i].time = 2.0 * i + 1; } missing_val.day = 99; @@ -863,7 +863,7 @@ main(int argc, char **argv) for (i = 0; i < DIM1_LEN; i++) { if (sprintf(hr_data_out[i].name, "alien_%d", i) < 0) ERR; - hr_data_out[i].max_temp = 99.99; + hr_data_out[i].max_temp = 99.99f; } /* Create a file with a nested compound type attribute and variable. */ @@ -927,7 +927,7 @@ main(int argc, char **argv) /* Create some phony data. */ for (i = 0; i < DIM1_LEN; i++) { - hr_data_out[i].max_temp = 99.99; + hr_data_out[i].max_temp = 99.99f; for (j = 0; j < ARRAY_LEN; j++) hr_data_out[i].name[j] = j; } diff --git a/nc_test4/tst_compounds2.c b/nc_test4/tst_compounds2.c index 7b1af1de96..8a9fadb4db 100644 --- a/nc_test4/tst_compounds2.c +++ b/nc_test4/tst_compounds2.c @@ -49,7 +49,7 @@ main(int argc, char **argv) /* Create some phony data. */ for (i = 0; i < DIM1_LEN; i++) { - data_out[i].y = 99.99; + data_out[i].y = 99.99f; for (j = 0; j < ARRAY_LEN; j++) data_out[i].x[j] = j; } diff --git a/nc_test4/tst_compounds3.c b/nc_test4/tst_compounds3.c index c09cb0b448..d388658bbb 100644 --- a/nc_test4/tst_compounds3.c +++ b/nc_test4/tst_compounds3.c @@ -136,7 +136,7 @@ main(int argc, char **argv) for (i = 0; i < ATT_LEN; i++) { - a1_att[i].s1.x = 13.3; + a1_att[i].s1.x = 13.3f; a1_att[i].s1.y = 13.3; } /* Create a file with two groups, define a type in each group, diff --git a/nc_test4/tst_converts2.c b/nc_test4/tst_converts2.c index 0f3bd92258..da6006a163 100644 --- a/nc_test4/tst_converts2.c +++ b/nc_test4/tst_converts2.c @@ -215,7 +215,7 @@ main(int argc, char **argv) if (uint64_in != ivalue) ERR; if (nc_get_var_float(ncid, varid, &float_in)) ERR; - if(fabs( (float_in-X_INT_MAX) - ((float)ivalue - X_INT_MAX)) > 1) ERR; + if(fabs( (float_in-(float)X_INT_MAX) - ((float)ivalue - (float)X_INT_MAX)) > 1) ERR; if (nc_get_var_double(ncid, varid, &double_in)) ERR; if (double_in != (double)ivalue) ERR; diff --git a/nc_test4/tst_coords.c b/nc_test4/tst_coords.c index 000b8a632b..f7133a57f9 100644 --- a/nc_test4/tst_coords.c +++ b/nc_test4/tst_coords.c @@ -38,8 +38,8 @@ main(int argc, char **argv) #define LAT_NAME "lat" #define LON_NAME "lon" #define NVARS 2 -#define START_LAT 25.0 -#define START_LON -125.0 +#define START_LAT 25.0f +#define START_LON -125.0f { int ncid, lon_dimid, lat_dimid; int lat_varid, lon_varid; @@ -52,9 +52,9 @@ main(int argc, char **argv) /* Initialize coord data. */ for (lat = 0; lat < NLAT; lat++) - lats[lat] = START_LAT + 5. * lat; + lats[lat] = START_LAT + 5.f * (float)lat; for (lon = 0; lon < NLON; lon++) - lons[lon] = START_LON + 5. * lon; + lons[lon] = START_LON + 5.f * (float)lon; /* Create file with two dimensions. */ if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; @@ -103,7 +103,7 @@ main(int argc, char **argv) #define DIM0_NAME "d0" #define CACHE_SIZE 1000000 #define CACHE_NELEMS 1009 -#define CACHE_PREEMPTION .90 +#define CACHE_PREEMPTION .90f int ncid, dimid, varid; char name_in[NC_MAX_NAME + 1]; @@ -413,12 +413,12 @@ main(int argc, char **argv) check_err(stat,__LINE__,__FILE__); } { /* missing_value */ - static const float pr_missing_value_att[1] = {1e+20} ; + static const float pr_missing_value_att[1] = {1e+20f} ; stat = nc_put_att_float(root_grp, pr_id, "missing_value", NC_FLOAT, 1, pr_missing_value_att); check_err(stat,__LINE__,__FILE__); } { /* _FillValue */ - static const float pr_FillValue_att[1] = {1e+20} ; + static const float pr_FillValue_att[1] = {1e+20f} ; stat = nc_put_att_float(root_grp, pr_id, "_FillValue", NC_FLOAT, 1, pr_FillValue_att); check_err(stat,__LINE__,__FILE__); } diff --git a/nc_test4/tst_dims.c b/nc_test4/tst_dims.c index d821e20f73..f64ba66d59 100644 --- a/nc_test4/tst_dims.c +++ b/nc_test4/tst_dims.c @@ -849,9 +849,9 @@ main(int argc, char **argv) /* Lats and lons suitable for some South American data. */ for (lat[0] = 40.0, i = 1; i < LAT_LEN; i++) - lat[i] = lat[i - 1] + .5; + lat[i] = (float)lat[i - 1] + .5f; for (lon[0] = 20.0, i = 1; i < LON_LEN; i++) - lon[i] = lon[i - 1] + 1.5; + lon[i] = (float)lon[i - 1] + 1.5f; /* Some phoney 2D pressure data. */ for (i = 0; i < LAT_LEN; i++) @@ -1002,9 +1002,9 @@ main(int argc, char **argv) /* Some phony 1D lats and lons. */ for (i = 0; i < LAT_LEN; i++) - lat[i] = i * 5.; + lat[i] = (float)i * 5.f; for (i = 0; i < LON_LEN; i++) - lon[i] = i * 5.; + lon[i] = (float)i * 5.f; if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR; diff --git a/nc_test4/tst_files.c b/nc_test4/tst_files.c index e0f7ddf302..f2ad31bedd 100644 --- a/nc_test4/tst_files.c +++ b/nc_test4/tst_files.c @@ -209,7 +209,7 @@ main(int argc, char **argv) int dimids_var[2], var_type; size_t dim_len; char dim_name[NC_MAX_NAME+1], var_name[NC_MAX_NAME+1]; - float float_in, float_out = 99.99; + float float_in, float_out = 99.99f; int int_in, int_out = -9999; /* Create a file, this time with attributes. */ diff --git a/nc_test4/tst_put_vars.c b/nc_test4/tst_put_vars.c index 62f59ff80e..1ce011c2a3 100644 --- a/nc_test4/tst_put_vars.c +++ b/nc_test4/tst_put_vars.c @@ -42,7 +42,7 @@ main() int dimids[NDIMS]; for (i = 0; i < (X_LEN * Y_LEN * Z_LEN); i++) - mydata[i] = i; + mydata[i] = (float)i; /* create the file in NetCDF-4 format */ if ((retval = nc_create(FILE_NAME, NC_NETCDF4, &ncid))) diff --git a/nc_test4/tst_quantize.c b/nc_test4/tst_quantize.c index 47c8ce2eef..eb43c30437 100644 --- a/nc_test4/tst_quantize.c +++ b/nc_test4/tst_quantize.c @@ -277,7 +277,7 @@ main(int argc, char **argv) { int ncid, varid1, varid2; int quantize_mode_in, nsd_in; - float float_data[DIM_LEN_1] = {1.1111111}; + float float_data[DIM_LEN_1] = {1.1111111f}; double double_data[DIM_LEN_1] = {1.111111111111}; printf("\t\t**** testing quantize algorithm %d...\n", quantize_mode[q]); @@ -394,7 +394,7 @@ main(int argc, char **argv) { int ncid, dimid, varid1, varid2; int quantize_mode_in, nsd_in; - float float_data[DIM_LEN_1] = {1.1111111}; + float float_data[DIM_LEN_1] = {1.1111111f}; double double_data[DIM_LEN_1] = {1.111111111111}; printf("\t\t**** testing quantize algorithm %d...\n", quantize_mode[q]); @@ -485,7 +485,7 @@ main(int argc, char **argv) { int ncid, dimid, varid1, varid2; int quantize_mode_in, nsd_in; - float float_data[DIM_LEN_5] = {1.11111111, 1.0, 9.99999999, 12345.67, .1234567}; + float float_data[DIM_LEN_5] = {1.11111111f, 1.0f, 9.99999999f, 12345.67f, .1234567f}; double double_data[DIM_LEN_5] = {1.1111111, 1.0, 9.999999999, 1234567890.12345, 123456789012345.0}; int x; @@ -616,7 +616,7 @@ main(int argc, char **argv) { int ncid, dimid, varid1, varid2; int quantize_mode_in, nsd_in; - float float_data[DIM_LEN_1] = {1.1111111}; + float float_data[DIM_LEN_1] = {1.1111111f}; double double_data[DIM_LEN_1] = {1.111111111111}; printf("\t\t**** testing quantize algorithm %d...\n", quantize_mode[q]); @@ -733,7 +733,7 @@ main(int argc, char **argv) { int ncid, dimid, varid1, varid2; int quantize_mode_in, nsd_in; - float float_data[DIM_LEN_5] = {1.11111111, 1.0, 9.99999999, 12345.67, .1234567}; + float float_data[DIM_LEN_5] = {1.11111111f, 1.0, 9.99999999f, 12345.67f, .1234567f}; double double_data[DIM_LEN_5] = {1.1111111, 1.0, 9.999999999, 1234567890.12345, 123456789012345.0}; int x; @@ -862,7 +862,7 @@ main(int argc, char **argv) { int ncid, dimid, varid1, varid2; int quantize_mode_in, nsd_in; - float float_data[DIM_LEN_5] = {1.11111111, NC_FILL_FLOAT, 9.99999999, 12345.67, NC_FILL_FLOAT}; + float float_data[DIM_LEN_5] = {1.11111111f, NC_FILL_FLOAT, 9.99999999f, 12345.67f, NC_FILL_FLOAT}; double double_data[DIM_LEN_5] = {1.1111111, NC_FILL_DOUBLE, 9.999999999, 1234567890.12345, NC_FILL_DOUBLE}; int x; @@ -987,13 +987,13 @@ main(int argc, char **argv) SUMMARIZE_ERR; printf("\t**** testing more quantization values with custom fill values...\n"); { -#define CUSTOM_FILL_FLOAT 99.99999 +#define CUSTOM_FILL_FLOAT 99.99999f #define CUSTOM_FILL_DOUBLE -99999.99999 for (q = 0; q < NUM_QUANTIZE_MODES; q++) { int ncid, dimid, varid1, varid2; int quantize_mode_in, nsd_in; - float float_data[DIM_LEN_5] = {1.11111111, CUSTOM_FILL_FLOAT, 9.99999999, 12345.67, CUSTOM_FILL_FLOAT}; + float float_data[DIM_LEN_5] = {1.11111111f, CUSTOM_FILL_FLOAT, 9.99999999f, 12345.67f, CUSTOM_FILL_FLOAT}; double double_data[DIM_LEN_5] = {1.1111111, CUSTOM_FILL_DOUBLE, 9.999999999, 1234567890.12345, CUSTOM_FILL_DOUBLE}; float custom_fill_float = CUSTOM_FILL_FLOAT; double custom_fill_double = CUSTOM_FILL_DOUBLE; @@ -1311,8 +1311,8 @@ main(int argc, char **argv) ERR; for (i = 0; i < DIM_LEN_SIMPLE; i++) { - float_data[i] = 1.5 * i; - double_data[i] = 1.5 * i; + float_data[i] = 1.5f * (float)i; + double_data[i] = 1.5 * (double)i; } /* Create the file. */ diff --git a/nc_test4/tst_unlim_vars.c b/nc_test4/tst_unlim_vars.c index e6f157a1ce..e488f36ad6 100644 --- a/nc_test4/tst_unlim_vars.c +++ b/nc_test4/tst_unlim_vars.c @@ -51,7 +51,7 @@ main(int argc, char **argv) for (time = 0; time < NUM_TIMESTEPS; time++) for (lat = 0; lat < LAT_LEN; lat++) for (lon = 0; lon < LON_LEN; lon++) - data_out[time][lat][lon] = 25.5 + lat + lon + time; + data_out[time][lat][lon] = 25.5f + (float)(lat + lon + time); /* Create a file with a 3D surface temp variable. */ if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR; diff --git a/nc_test4/tst_vars.c b/nc_test4/tst_vars.c index a605da71ef..f94fc0dcbb 100644 --- a/nc_test4/tst_vars.c +++ b/nc_test4/tst_vars.c @@ -56,10 +56,10 @@ #define NVARS_EX 4 /* These are used to construct some example data. */ -#define SAMPLE_PRESSURE 900 -#define SAMPLE_TEMP 9.0 -#define START_LAT 25.0 -#define START_LON -125.0 +#define SAMPLE_PRESSURE 900.f +#define SAMPLE_TEMP 9.0f +#define START_LAT 25.0f +#define START_LON -125.0f /* For the units attributes. */ #define UNITS "units" @@ -97,16 +97,16 @@ create_4D_example(char *file_name, int cmode) * would have some real data to write, for example, model * output. */ for (lat = 0; lat < NLAT; lat++) - lats[lat] = START_LAT + 5.*lat; + lats[lat] = START_LAT + 5.f*(float)lat; for (lon = 0; lon < NLON; lon++) - lons[lon] = START_LON + 5.*lon; + lons[lon] = START_LON + 5.f*(float)lon; for (lvl = 0; lvl < NLVL; lvl++) for (lat = 0; lat < NLAT; lat++) for (lon = 0; lon < NLON; lon++) { - pres_out[lvl][lat][lon] = SAMPLE_PRESSURE + i; - temp_out[lvl][lat][lon] = SAMPLE_TEMP + i++; + pres_out[lvl][lat][lon] = SAMPLE_PRESSURE + (float)i; + temp_out[lvl][lat][lon] = SAMPLE_TEMP + (float)i++; } /* Create the file. */ @@ -307,7 +307,7 @@ main(int argc, char **argv) unsigned short ushort_out[DIM1_LEN][DIM2_LEN] = {{110, 128, 255},{110, 128, 255}}; short short_in[DIM1_LEN][DIM2_LEN], short_out[DIM1_LEN][DIM2_LEN] = {{-110, -128, 255},{-110, -128, 255}}; int int_in[DIM1_LEN][DIM2_LEN], int_out[DIM1_LEN][DIM2_LEN] = {{0, 128, 255},{0, 128, 255}}; - float float_in[DIM1_LEN][DIM2_LEN], float_out[DIM1_LEN][DIM2_LEN] = {{-.1, 9999.99, 100.001},{-.1, 9999.99, 100.001}}; + float float_in[DIM1_LEN][DIM2_LEN], float_out[DIM1_LEN][DIM2_LEN] = {{-.1f, 9999.99f, 100.001f},{-.1f, 9999.99f, 100.001f}}; double double_in[DIM1_LEN][DIM2_LEN], double_out[DIM1_LEN][DIM2_LEN] = {{0.02, .1128, 1090.1},{0.02, .1128, 1090.1}}; unsigned int uint_in[DIM1_LEN][DIM2_LEN], uint_out[DIM1_LEN][DIM2_LEN] = {{0, 128, 255},{0, 128, 255}}; long long int64_in[DIM1_LEN][DIM2_LEN], int64_out[DIM1_LEN][DIM2_LEN] = {{-111, 777, 100},{-111, 777, 100}}; diff --git a/nc_test4/tst_xplatform2.c b/nc_test4/tst_xplatform2.c index 02b7dc7067..6d2043d9f4 100644 --- a/nc_test4/tst_xplatform2.c +++ b/nc_test4/tst_xplatform2.c @@ -312,7 +312,7 @@ main(int argc, char **argv) return NC_ENOMEM; for (j = 0; j < vlen_of_comp_out[i].len; j++) { - ((struct s1 *)vlen_of_comp_out[i].p)[j].x = 42.42; + ((struct s1 *)vlen_of_comp_out[i].p)[j].x = 42.42f; ((struct s1 *)vlen_of_comp_out[i].p)[j].y = 2.0; } } @@ -322,7 +322,7 @@ main(int argc, char **argv) for (i = 0; i < DIM2_LEN; i++) for (j = 0; j < NUM_S1; j++) { - comp_array_of_comp_out[i].data[j].x = 42.42; + comp_array_of_comp_out[i].data[j].x = 42.42f; comp_array_of_comp_out[i].data[j].y = 2.0; } @@ -336,7 +336,7 @@ main(int argc, char **argv) return NC_ENOMEM; for (k = 0; k < comp_array_of_vlen_of_comp_out[i].data[j].len; k++) { - ((struct s1 *)comp_array_of_vlen_of_comp_out[i].data[j].p)[k].x = 42.42; + ((struct s1 *)comp_array_of_vlen_of_comp_out[i].data[j].p)[k].x = 42.42f; ((struct s1 *)comp_array_of_vlen_of_comp_out[i].data[j].p)[k].y = 2.0; } } diff --git a/ncdap_test/t_dap3a.c b/ncdap_test/t_dap3a.c index bbfd36252e..4ededc15c8 100644 --- a/ncdap_test/t_dap3a.c +++ b/ncdap_test/t_dap3a.c @@ -317,7 +317,7 @@ static float int162float32_data[DIMSIZE]={0.000,256.000,512.000,768.000,1024.000 static int int32_data[DIMSIZE]={0,2048,4096,6144,8192,10240,12288,14336,16384,18432,20480,22528,24576,26624,28672,30720,32768,34816,36864,38912,40960,43008,45056,47104,49152}; static float int32tofloat32_data[DIMSIZE]={0.000,2048.000,4096.000,6144.000,8192.000,10240.000,12288.000,14336.000,16384.000,18432.000,20480.000,22528.000,24576.000,26624.000,28672.000,30720.000,32768.000,34816.000,36864.000,38912.000,40960.000,43008.000,45056.000,47104.000,49152.000}; static long int32toilong_data[DIMSIZE]={0,2048,4096,6144,8192,10240,12288,14336,16384,18432,20480,22528,24576,26624,28672,30720,32768,34816,36864,38912,40960,43008,45056,47104,49152}; -static float float32_data[DIMSIZE]={0.000,0.010,0.020,0.030,0.040,0.050,0.060,0.070,0.080,0.090,0.100,0.110,0.120,0.130,0.140,0.149,0.159,0.169,0.179,0.189,0.199,0.208,0.218,0.228,0.238}; +static float float32_data[DIMSIZE]={0.000f,0.010f,0.020f,0.030f,0.040f,0.050f,0.060f,0.070f,0.080f,0.090f,0.100f,0.110f,0.120f,0.130f,0.140f,0.149f,0.159f,0.169f,0.179f,0.189f,0.199f,0.208f,0.218f,0.228f,0.238f}; static double float64_data[DIMSIZE]={1.000,1.000,1.000,1.000,0.999,0.999,0.998,0.998,0.997,0.996,0.995,0.994,0.993,0.992,0.990,0.989,0.987,0.986,0.984,0.982,0.980,0.978,0.976,0.974,0.971}; #ifndef USE_NETCDF4 diff --git a/ncdap_test/test_cvt.c b/ncdap_test/test_cvt.c index 46f52de2a0..3485dd4f7a 100644 --- a/ncdap_test/test_cvt.c +++ b/ncdap_test/test_cvt.c @@ -316,7 +316,7 @@ static float int162float32_data[DIMSIZE]={0.000,256.000,512.000,768.000,1024.000 static int int32_data[DIMSIZE]={0,2048,4096,6144,8192,10240,12288,14336,16384,18432,20480,22528,24576,26624,28672,30720,32768,34816,36864,38912,40960,43008,45056,47104,49152}; static float int32tofloat32_data[DIMSIZE]={0.000,2048.000,4096.000,6144.000,8192.000,10240.000,12288.000,14336.000,16384.000,18432.000,20480.000,22528.000,24576.000,26624.000,28672.000,30720.000,32768.000,34816.000,36864.000,38912.000,40960.000,43008.000,45056.000,47104.000,49152.000}; static long int32toilong_data[DIMSIZE]={0,2048,4096,6144,8192,10240,12288,14336,16384,18432,20480,22528,24576,26624,28672,30720,32768,34816,36864,38912,40960,43008,45056,47104,49152}; -static float float32_data[DIMSIZE]={0.000,0.010,0.020,0.030,0.040,0.050,0.060,0.070,0.080,0.090,0.100,0.110,0.120,0.130,0.140,0.149,0.159,0.169,0.179,0.189,0.199,0.208,0.218,0.228,0.238}; +static float float32_data[DIMSIZE]={0.000f,0.010f,0.020f,0.030f,0.040f,0.050f,0.060f,0.070f,0.080f,0.090f,0.100f,0.110f,0.120f,0.130f,0.140f,0.149f,0.159f,0.169f,0.179f,0.189f,0.199f,0.208f,0.218f,0.228f,0.238f}; static double float64_data[DIMSIZE]={1.000,1.000,1.000,1.000,0.999,0.999,0.998,0.998,0.997,0.996,0.995,0.994,0.993,0.992,0.990,0.989,0.987,0.986,0.984,0.982,0.980,0.978,0.976,0.974,0.971}; #ifndef USE_NETCDF4 diff --git a/ncdap_test/test_varm3.c b/ncdap_test/test_varm3.c index 707bf53011..740d6fa6f6 100644 --- a/ncdap_test/test_varm3.c +++ b/ncdap_test/test_varm3.c @@ -33,33 +33,33 @@ TODO: Note that this test uses thredds server #define VAR "SST" static float expected_stride1[12] = { -29.430857, -29.403780, -29.325428, -29.578333, -29.660833, -29.378437, -29.151943, -29.109715, -29.114864, -29.550285, -29.542500, -29.500286 +29.430857f, +29.403780f, +29.325428f, +29.578333f, +29.660833f, +29.378437f, +29.151943f, +29.109715f, +29.114864f, +29.550285f, +29.542500f, +29.500286f }; static float expected_stride2[6] = { -29.430857, -29.325428, -29.660833, -29.151943, -29.114864, -29.542500 +29.430857f, +29.325428f, +29.660833f, +29.151943f, +29.114864f, +29.542500f }; static float expected_stride3[3] = { -29.430857, -29.378437, -29.542500 +29.430857f, +29.378437f, +29.542500f }; void diff --git a/ncdap_test/tst_encode.sh b/ncdap_test/tst_encode.sh index e0de50b77a..54e0a20bed 100755 --- a/ncdap_test/tst_encode.sh +++ b/ncdap_test/tst_encode.sh @@ -8,6 +8,5 @@ set -e echo "" echo "*** Testing #encode=" mechanism -#${NCDUMP} -h 'http://opendap2.oceanbrowser.net/thredds/dodsC/data/emodnet1-domains/tmp%20test.nc?lon[0:8]#encode=none' # raw: http://iridl.ldeo.columbia.edu/SOURCES/.Indices/.soi/.c8110/.anomaly/T/(Jan 1979)/VALUE/dods ${NCDUMP} -h 'http://iridl.ldeo.columbia.edu/SOURCES/.Indices/.soi/.c8110/.anomaly/T/%28Jan%201979%29/VALUE/dods?anomaly[0]' diff --git a/ncdap_test/tst_remote.sh b/ncdap_test/tst_remote.sh index d7cc2a636f..2a4dedf72c 100755 --- a/ncdap_test/tst_remote.sh +++ b/ncdap_test/tst_remote.sh @@ -1,6 +1,10 @@ #!/bin/sh +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + if test "x$SETX" != x ; then set -x ; fi + set -e quiet=0 diff --git a/ncdump/nccopy.c b/ncdump/nccopy.c index 2ac4a73d9c..ae041547e4 100644 --- a/ncdump/nccopy.c +++ b/ncdump/nccopy.c @@ -7,6 +7,7 @@ *********************************************************************/ #include "config.h" /* for USE_NETCDF4 macro */ +#include #include #include #ifdef HAVE_GETOPT_H @@ -2310,7 +2311,7 @@ main(int argc, char**argv) double dval = double_with_suffix(optarg); /* "K" for kilobytes. "M" for megabytes, ... */ if(dval < 0) error("Suffix used for '-m' option value must be K, M, G, T, or P"); - option_copy_buffer_size = dval; + option_copy_buffer_size = (size_t)dval; break; } case 'h': /* non-default size of chunk cache */ @@ -2318,7 +2319,7 @@ main(int argc, char**argv) double dval = double_with_suffix(optarg); /* "K" for kilobytes. "M" for megabytes, ... */ if(dval < 0) error("Suffix used for '-h' option value must be K, M, G, T, or P"); - option_chunk_cache_size = dval; + option_chunk_cache_size = (size_t)dval; break; } case 'e': /* number of elements chunk cache can hold */ @@ -2326,7 +2327,7 @@ main(int argc, char**argv) double dval = double_with_suffix(optarg); /* "K" for kilobytes. "M" for megabytes, ... */ if(dval < 0 ) error("Suffix used for '-e' option value must be K, M, G, T, or P"); - option_chunk_cache_nelems = (long)dval; + option_chunk_cache_nelems = (size_t)dval; break; } case 'r': diff --git a/ncdump/ncdump.c b/ncdump/ncdump.c index f577079813..abaa20e387 100644 --- a/ncdump/ncdump.c +++ b/ncdump/ncdump.c @@ -699,7 +699,7 @@ pr_att_valsx( case NC_BYTE: case NC_SHORT: case NC_INT: - ii = vals[iel]; + ii = (int)vals[iel]; res = snprintf(gps, PRIM_LEN, "%d", ii); assert(res < PRIM_LEN); (void) strlcat(attvals, gps, attvalslen); @@ -708,28 +708,28 @@ pr_att_valsx( case NC_UBYTE: case NC_USHORT: case NC_UINT: - ui = vals[iel]; + ui = (unsigned int)vals[iel]; res = snprintf(gps, PRIM_LEN, "%u", ui); assert(res < PRIM_LEN); (void) strlcat(attvals, gps, attvalslen); (void) strlcat(attvals, iel < len-1 ? " " : "", attvalslen); break; case NC_INT64: - i64 = vals[iel]; + i64 = (int64_t)vals[iel]; res = snprintf(gps, PRIM_LEN, "%lld", i64); assert(res < PRIM_LEN); (void) strlcat(attvals, gps, attvalslen); (void) strlcat(attvals, iel < len-1 ? " " : "", attvalslen); break; case NC_UINT64: - ui64 = vals[iel]; + ui64 = (uint64_t)vals[iel]; res = snprintf(gps, PRIM_LEN, "%llu", ui64); assert(res < PRIM_LEN); (void) strlcat(attvals, gps, attvalslen); (void) strlcat(attvals, iel < len-1 ? " " : "", attvalslen); break; case NC_FLOAT: - ff = vals[iel]; + ff = (float)vals[iel]; res = snprintf(gps, PRIM_LEN, float_attx_fmt, ff); assert(res < PRIM_LEN); tztrim(gps); /* trim trailing 0's after '.' */ @@ -847,7 +847,7 @@ pr_att( case NC_VLEN: /* because size returned for vlen is base type size, but we * need space to read array of vlen structs into ... */ - data = emalloc((att.len + 1) * sizeof(nc_vlen_t)); + data = emalloc((att.len + 1) * sizeof(nc_vlen_t)); break; case NC_OPAQUE: data = emalloc((att.len + 1) * type_size); diff --git a/ncdump/tst_create_files.c b/ncdump/tst_create_files.c index c8122042a9..e4cee78e40 100644 --- a/ncdump/tst_create_files.c +++ b/ncdump/tst_create_files.c @@ -226,8 +226,8 @@ main(int argc, char **argv) #define ATT_NAME_CMP "my_favorite_wind_speeds" #define COMPOUND_NAME "wind_vector" #define NUM_FAVS 3 -#define U_VALUE 13.3 -#define V_VALUE 12.2 +#define U_VALUE 13.3f +#define V_VALUE 12.2f { int ncid; diff --git a/ncdump/tst_vlen_data.c b/ncdump/tst_vlen_data.c index ad5d18388e..f11b3932b7 100644 --- a/ncdump/tst_vlen_data.c +++ b/ncdump/tst_vlen_data.c @@ -91,7 +91,7 @@ main(int argc, char **argv) array[i] = (float *) malloc(ncolumns * sizeof(float)); if(array[i] == NULL) ERR; for (j = 0; j < ncolumns; j++) { - array[i][j] = 10.0 * (i + 1) + j; + array[i][j] = 10.0f * (float)(i + 1) + (float)j; } } array[4][0] = missing_value; /* overwrite last row with missing for equality test */ diff --git a/nctest/CMakeLists.txt b/nctest/CMakeLists.txt index d7de84a6cd..ada220f2ab 100644 --- a/nctest/CMakeLists.txt +++ b/nctest/CMakeLists.txt @@ -18,7 +18,9 @@ TARGET_LINK_LIBRARIES(nctest netcdf) ADD_TEST(nctest ${EXECUTABLE_OUTPUT_PATH}/nctest) add_bin_test_no_prefix(tst_rename) -add_sh_test(nctest compare_test_files) -IF(HAVE_BASH) - SET_TESTS_PROPERTIES(nctest_compare_test_files PROPERTIES DEPENDS nctest) -ENDIF(HAVE_BASH) +IF(BUILD_UTILITIES) + add_sh_test(nctest compare_test_files) + IF(HAVE_BASH) + SET_TESTS_PROPERTIES(nctest_compare_test_files PROPERTIES DEPENDS nctest) + ENDIF(HAVE_BASH) +ENDIF(BUILD_UTILITIES) diff --git a/nczarr_test/CMakeLists.txt b/nczarr_test/CMakeLists.txt index 09b013823b..e58add623a 100644 --- a/nczarr_test/CMakeLists.txt +++ b/nczarr_test/CMakeLists.txt @@ -143,8 +143,8 @@ IF(ENABLE_TESTS) add_sh_test(nczarr_test run_ut_misc) add_sh_test(nczarr_test run_ncgen4) if(LARGE_FILE_TESTS) - BUILD_BIN_TEST(test_readcaching}) - BUILD_BIN_TEST(test_writecaching}) + BUILD_BIN_TEST(test_readcaching) + BUILD_BIN_TEST(test_writecaching) BUILD_BIN_TEST(test_chunkcases ${TSTCOMMONSRC}) add_sh_test(nczarr_test run_cachetest) add_sh_test(nczarr_test run_chunkcases) @@ -193,6 +193,8 @@ IF(ENABLE_TESTS) if(ENABLE_NCZARR_ZIP) add_sh_test(nczarr_test run_newformat) + # Test various corrupted files + ADD_SH_TEST(nczarr_test run_corrupt.sh) endif() IF(FALSE) # Obsolete tests diff --git a/nczarr_test/Makefile.am b/nczarr_test/Makefile.am index a42248da01..d8cb5d4a54 100644 --- a/nczarr_test/Makefile.am +++ b/nczarr_test/Makefile.am @@ -161,6 +161,9 @@ endif # ISMINGW endif #ENABLE_FILTER_TESTING endif #ENABLE_NCZARR_FILTERS +# Test various corrupted files +TESTS += run_corrupt.sh + endif #BUILD_UTILITIES # These programs are used by the test cases @@ -200,7 +203,8 @@ run_purezarr.sh run_interop.sh run_misc.sh \ run_filter.sh \ run_newformat.sh run_nczarr_fill.sh run_quantize.sh \ run_jsonconvention.sh run_nczfilter.sh run_unknown.sh \ -run_scalar.sh run_strings.sh run_nulls.sh run_notzarr.sh run_external.sh run_unlim_io.sh +run_scalar.sh run_strings.sh run_nulls.sh run_notzarr.sh run_external.sh \ +run_unlim_io.sh run_corrupt.sh EXTRA_DIST += \ ref_ut_map_create.cdl ref_ut_map_writedata.cdl ref_ut_map_writemeta2.cdl ref_ut_map_writemeta.cdl \ @@ -228,6 +232,9 @@ ref_nulls_nczarr.baseline ref_nulls_zarr.baseline ref_nulls.cdl ref_notzarr.tar. EXTRA_DIST += ref_power_901_constants_orig.zip ref_power_901_constants.cdl ref_quotes_orig.zip ref_quotes.cdl \ ref_zarr_test_data.cdl.gz ref_zarr_test_data_2d.cdl.gz +# Additional Files +EXTRA_DIST += ref_noshape.file.zip + CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp*.zmap tmp_ngc.c ref_zarr_test_data.cdl tst_*.nc.zip ref_quotes.zip ref_power_901_constants.zip BUILT_SOURCES = test_quantize.c test_filter_vlen.c test_unlim_vars.c test_endians.c \ diff --git a/nczarr_test/ref_noshape.file.zip b/nczarr_test/ref_noshape.file.zip new file mode 100644 index 0000000000..6f4d9c1503 Binary files /dev/null and b/nczarr_test/ref_noshape.file.zip differ diff --git a/nczarr_test/run_corrupt.sh b/nczarr_test/run_corrupt.sh new file mode 100755 index 0000000000..e9ddc67d57 --- /dev/null +++ b/nczarr_test/run_corrupt.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +# Test various kinds of corrupted files + + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "$srcdir/test_nczarr.sh" + +set -e + +s3isolate "testdir_corrupt" +THISDIR=`pwd` +cd $ISOPATH + +export NCLOGGING=WARN + +testnoshape1() { + zext=file + unzip ${srcdir}/ref_noshape.file.zip + fileargs ${ISOPATH}/ref_noshape "mode=zarr,$zext" + rm -f tmp_noshape1_${zext}.cdl + ${NCDUMP} $flags $fileurl > tmp_noshape1_${zext}.cdl +} + +testnoshape2() { + # Test against the original issue URL + rm -f tmp_noshape2_gs.cdl + fileurl="https://storage.googleapis.com/cmip6/CMIP6/CMIP/NASA-GISS/GISS-E2-1-G/historical/r1i1p1f1/day/tasmin/gn/v20181015/#mode=zarr,s3&aws.profile=no" + ${NCDUMP} -h $flags $fileurl > tmp_noshape2_gs.cdl +} + +testnoshape1 +if test "x$FEATURE_S3TESTS" = xyes ; then testnoshape2; fi diff --git a/nczarr_test/run_scalar.sh b/nczarr_test/run_scalar.sh index c6de0ebc81..b7c268ee5b 100755 --- a/nczarr_test/run_scalar.sh +++ b/nczarr_test/run_scalar.sh @@ -7,7 +7,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e -s3isolate "testdir_nczarr" +s3isolate "testdir_scalar" THISDIR=`pwd` cd $ISOPATH @@ -43,7 +43,7 @@ echo "*** create nczarr file" ${NCGEN} -4 -b -o "$nczarrurl" $top_srcdir/nczarr_test/ref_scalar.cdl echo "*** read purezarr" -${NCDUMP} -n ref_scalar $zarrurl > tmp_scalar_zarr0_${zext}.cdl +${NCDUMP} -n ref_scalar $zarrurl > tmp_scalar_zarr_${zext}.cdl ${ZMD} -h $zarrurl > tmp_scalar_zarr_${zext}.txt echo "*** read nczarr" ${NCDUMP} -n ref_scalar $nczarrurl > tmp_scalar_nczarr_${zext}.cdl @@ -53,8 +53,8 @@ echo "*** verify" diff -bw $top_srcdir/nczarr_test/ref_scalar.cdl tmp_scalar_nczarr_${zext}.cdl # Fixup -zarrscalar tmp_scalar_zarr0_${zext}.cdl tmp_scalar_zarr_${zext}.cdl -diff -bw $top_srcdir/nczarr_test/ref_scalar.cdl tmp_scalar_zarr_${zext}.cdl +zarrscalar tmp_scalar_zarr_${zext}.cdl tmp_rescale_zarr_${zext}.cdl +diff -bw $top_srcdir/nczarr_test/ref_scalar.cdl tmp_rescale_zarr_${zext}.cdl } testcase file diff --git a/nczarr_test/test_zchunks2.c b/nczarr_test/test_zchunks2.c index 7b471a2b9f..54963761c8 100644 --- a/nczarr_test/test_zchunks2.c +++ b/nczarr_test/test_zchunks2.c @@ -52,7 +52,7 @@ calculate_waste(int ndims, size_t *dimlen, size_t *chunksize, float *waste) #endif /* Percent of the chunked file that is wasted space. */ - *waste = ((float)(chunked - unchunked) / (float)chunked) * 100.0; + *waste = ((float)(chunked - unchunked) / (float)chunked) * 100.0f; #ifdef PRINT_CHUNK_WASTE_REPORT printf("\ndimlen\tchunksize\tnum_chunks\n"); diff --git a/netCDFConfig.cmake.in b/netCDFConfig.cmake.in index 3146bb7d7c..db7bb823e6 100644 --- a/netCDFConfig.cmake.in +++ b/netCDFConfig.cmake.in @@ -40,6 +40,10 @@ set(netCDF_HAS_DAP4 @HAS_DAP4@) set(netCDF_HAS_DISKLESS @HAS_DISKLESS@) set(netCDF_HAS_MMAP @HAS_MMAP@) set(netCDF_HAS_JNA @HAS_JNA@) +if (netCDF_HAS_HDF4 OR netCDF_HAS_HDF5) + include(CMakeFindDependencyMacro) + find_dependency(HDF5) +endif () if (@HAS_PARALLEL@) include(CMakeFindDependencyMacro) diff --git a/oc2/occurlfunctions.c b/oc2/occurlfunctions.c index 06b3fd352e..275d42eb4d 100644 --- a/oc2/occurlfunctions.c +++ b/oc2/occurlfunctions.c @@ -1,6 +1,11 @@ /* Copyright 2018, UCAR/Unidata and OPeNDAP, Inc. See the COPYRIGHT file for more information. */ +/* WARNING: oc2/occurlfunctions.c and libdap4/d4curlfunctions.c +should be merged since they are essentially the same file. +In the meantime, changes to one should be propagated to the other. +*/ + #include "config.h" #include #ifdef HAVE_STDINT_H @@ -127,36 +132,43 @@ ocset_curlflag(OCstate* state, int flag) } break; - case CURLOPT_USE_SSL: - case CURLOPT_SSLCERT: case CURLOPT_SSLKEY: - case CURLOPT_SSL_VERIFYPEER: case CURLOPT_SSL_VERIFYHOST: - case CURLOPT_CAINFO: case CURLOPT_CAPATH: - { - struct ssl* ssl = &state->auth->ssl; + case CURLOPT_SSL_VERIFYPEER: /* VERIFYPEER == 0 => VERIFYHOST == 0 */ /* We need to have 2 states: default and a set value */ /* So -1 => default >= 0 => use value */ - if(ssl->verifypeer >= 0) { - SETCURLOPT(state, CURLOPT_SSL_VERIFYPEER, (OPTARG)(ssl->verifypeer)); - } + if(state->auth->ssl.verifypeer >= 0) { + SETCURLOPT(state, CURLOPT_SSL_VERIFYPEER, (OPTARG)(state->auth->ssl.verifypeer)); + if(state->auth->ssl.verifypeer == 0) state->auth->ssl.verifyhost = 0; + } + break; + case CURLOPT_SSL_VERIFYHOST: #ifdef HAVE_LIBCURL_766 - if(ssl->verifyhost >= 0) { - SETCURLOPT(state, CURLOPT_SSL_VERIFYHOST, (OPTARG)(ssl->verifyhost)); + if(state->auth->ssl.verifyhost >= 0) { + SETCURLOPT(state, CURLOPT_SSL_VERIFYHOST, (OPTARG)(state->auth->ssl.verifyhost)); } #endif - if(ssl->certificate) - SETCURLOPT(state, CURLOPT_SSLCERT, ssl->certificate); - if(ssl->key) - SETCURLOPT(state, CURLOPT_SSLKEY, ssl->key); - if(ssl->keypasswd) + break; + case CURLOPT_SSLCERT: + if(state->auth->ssl.certificate) + SETCURLOPT(state, CURLOPT_SSLCERT, state->auth->ssl.certificate); + break; + case CURLOPT_SSLKEY: + if(state->auth->ssl.key) + SETCURLOPT(state, CURLOPT_SSLKEY, state->auth->ssl.key); + if(state->auth->ssl.keypasswd) /* libcurl prior to 7.16.4 used 'CURLOPT_SSLKEYPASSWD' */ - SETCURLOPT(state, CURLOPT_KEYPASSWD, ssl->keypasswd); - if(ssl->cainfo) - SETCURLOPT(state, CURLOPT_CAINFO, ssl->cainfo); - if(ssl->capath) - SETCURLOPT(state, CURLOPT_CAPATH, ssl->capath); - } - break; + SETCURLOPT(state, CURLOPT_SSLKEYPASSWD, state->auth->ssl.keypasswd); + break; + case CURLOPT_CAINFO: + if(state->auth->ssl.cainfo) + SETCURLOPT(state, CURLOPT_CAINFO, state->auth->ssl.cainfo); + break; + case CURLOPT_CAPATH: + if(state->auth->ssl.capath) + SETCURLOPT(state, CURLOPT_CAPATH, state->auth->ssl.capath); + break; + case CURLOPT_USE_SSL: + break; #ifdef HAVE_CURLOPT_BUFFERSIZE case CURLOPT_BUFFERSIZE: @@ -210,6 +222,12 @@ ocset_flags_perlink(OCstate* state) if(stat == OC_NOERR) stat = ocset_curlflag(state,CURLOPT_COOKIEJAR); if(stat == OC_NOERR) stat = ocset_curlflag(state,CURLOPT_USERPWD); if(stat == OC_NOERR) stat = ocset_curlflag(state,CURLOPT_PROXY); + if(stat == OC_NOERR) stat = ocset_curlflag(state,CURLOPT_SSL_VERIFYPEER); + if(stat == OC_NOERR) stat = ocset_curlflag(state,CURLOPT_SSL_VERIFYHOST); + if(stat == OC_NOERR) stat = ocset_curlflag(state,CURLOPT_SSLCERT); + if(stat == OC_NOERR) stat = ocset_curlflag(state,CURLOPT_SSLKEY); + if(stat == OC_NOERR) stat = ocset_curlflag(state,CURLOPT_CAINFO); + if(stat == OC_NOERR) stat = ocset_curlflag(state,CURLOPT_CAPATH); if(stat == OC_NOERR) stat = ocset_curlflag(state,CURLOPT_USE_SSL); if(stat == OC_NOERR) stat = ocset_curlflag(state, CURLOPT_FOLLOWLOCATION); if(stat == OC_NOERR) stat = ocset_curlflag(state, CURLOPT_MAXREDIRS); diff --git a/plugins/H5Zmisc.c b/plugins/H5Zmisc.c index 56e64734ff..8813af59d8 100644 --- a/plugins/H5Zmisc.c +++ b/plugins/H5Zmisc.c @@ -113,7 +113,7 @@ H5Z_filter_test(unsigned int flags, size_t cd_nelmts, break; case TC_ODDSIZE: /* Print out the chunk size */ - fprintf(stderr,"nbytes = %lld chunk size = %lld\n",(long long)nbytes,(long long)*buf_size); + fprintf(stderr,">>> nbytes = %lld chunk size = %lld\n",(long long)nbytes,(long long)*buf_size); fflush(stderr); break; default: break; @@ -122,13 +122,15 @@ H5Z_filter_test(unsigned int flags, size_t cd_nelmts, if (flags & H5Z_FLAG_REVERSE) { /* Decompress */ if(testcase == TC_EXPANDED) { +#ifdef DEBUG int i; float* b = (float*)*buf; -fprintf(stderr,"TC_EXPANDED: decompress: nbytes=%u buf_size=%u xdata[0..8]=|",(unsigned)nbytes,(unsigned)*buf_size); +fprintf(stderr,">>> TC_EXPANDED: decompress: nbytes=%u buf_size=%u xdata[0..8]=|",(unsigned)nbytes,(unsigned)*buf_size); for(i=0;i<8;i++) { fprintf(stderr," %u",(int)(b[1024+i])); } fprintf(stderr,"|\n"); +#endif /* Replace buffer */ newbuf = H5allocate_memory(*buf_size,0); if(newbuf == NULL) abort(); @@ -149,8 +151,8 @@ fprintf(stderr,"TC_EXPANDED: decompress: nbytes=%u buf_size=%u xdata[0..8]=|",(u if(testcase == TC_EXPANDED) { int i; float* b; -#if 0 -fprintf(stderr,"TC_EXPANDED: compress: nbytes=%u buf_size=%u size=%u\n",(unsigned)nbytes,(unsigned)*buf_size,(unsigned)size); +#ifdef DEBUG +fprintf(stderr,">>> TC_EXPANDED: compress: nbytes=%u buf_size=%u size=%u\n",(unsigned)nbytes,(unsigned)*buf_size,(unsigned)size); #endif /* Replace buffer with one that is bigger than the input size */ newbuf = H5allocate_memory(size,0); @@ -218,7 +220,7 @@ extract1(void* field, size_t size, const unsigned int* params) llp = (unsigned long long*)field; *llp = u.ll; break; - default: fprintf(stderr,"insert: unexpected size: %u\n",(unsigned)size); abort(); + default: fprintf(stderr,">>> insert: unexpected size: %u\n",(unsigned)size); abort(); } } @@ -247,7 +249,7 @@ paramcheck(size_t nparams, const unsigned int* params, struct All* extracted) memset(&all,0,sizeof(all)); if(nparams != NPARAMS) { - fprintf(stderr,"Incorrect number of parameters: expected=%ld sent=%ld\n",(unsigned long)NPARAMS,(unsigned long)nparams); + fprintf(stderr,">>> Incorrect number of parameters: expected=%ld sent=%ld\n",(unsigned long)NPARAMS,(unsigned long)nparams); goto fail; } @@ -270,7 +272,7 @@ paramcheck(size_t nparams, const unsigned int* params, struct All* extracted) #ifdef DEBUG { size_t i; - fprintf(stderr,"bigendian=%d nparams=%d params=\n",bigendian,nparams); + fprintf(stderr,">>> nparams=%lu params=\n",nparams); for(i=0;i>> mismatch: %s\n",which); fflush(stderr); } diff --git a/test_common.in b/test_common.in index 8be771e19e..5f0f066db8 100644 --- a/test_common.in +++ b/test_common.in @@ -129,6 +129,9 @@ top_builddir="$TOPBUILDDIR" # Currently not used, but left as a Visual Studio placeholder. # VS=Debug +# Set when using gdb +#DL=".libs/" + # srcdir may or may not be defined, but if not, then create it if test "x$srcdir" = x ; then # we need to figure out our directory @@ -169,11 +172,11 @@ fi # We need to locate certain executables (and other things), # capture absolute paths, and make visible -export NCDUMP="${top_builddir}/ncdump${VS}/ncdump${ext}" -export NCCOPY="${top_builddir}/ncdump${VS}/nccopy${ext}" -export NCGEN="${top_builddir}/ncgen${VS}/ncgen${ext}" -export NCGEN3="${top_builddir}/ncgen3${VS}/ncgen3${ext}" -export NCPATHCVT="${top_builddir}/ncdump${VS}/ncpathcvt${ext}" +export NCDUMP="${top_builddir}/ncdump${VS}/${DL}ncdump${ext}" +export NCCOPY="${top_builddir}/ncdump${VS}/${DL}nccopy${ext}" +export NCGEN="${top_builddir}/ncgen${VS}/${DL}ncgen${ext}" +export NCGEN3="${top_builddir}/ncgen3${VS}/${DL}ncgen3${ext}" +export NCPATHCVT="${top_builddir}/ncdump${VS}/${DL}ncpathcvt${ext}" # Temporary hacks (until we have a test_utils directory) # to locate certain specific test files