Provide a single option to disable all network access and testing.

Add the option "--disable-network-access" (automake)
or "-DENABLE_NETWORK_ACCESS=OFF" (cmake).
When disabled, this option transitively disables all
network access capabilities and testing.
If set, this option implies the following:
* --disable-dap
* --disable-byterange
* --disable-s3

This PR answers a request for a feature from Ed Hartnett.

## Misc. Other changes
* Take the opportunity to clean up some old, unused options;
e.g. --enable-multifilters.
* Fix bug in using S3 urls.

.github/workflows/run_tests_win_cygwin.yml

@@ -49,7 +49,7 @@ jobs:
           --disable-static --enable-dap --disable-dap-remote-tests
           --enable-plugins ${{ matrix.plugin_dir_option }}
           --disable-nczarr --disable-nczarr-filters
-          --disable-nczarr-s3 --disable-nczarr-s3-tests
+          --disable-s3 --with-s3-testing=no
 
 
       - name: Look at config.log if error

CMakeLists.txt

@@ -523,6 +523,15 @@ if(ENABLE_HDF5 OR ENABLE_HDF4 OR ENABLE_NCZARR)
   SET(ENABLE_NETCDF4 ON CACHE BOOL "Enable netCDF4 Alias" FORCE)
 endif()
 
+# enable|disable all forms of network access
+OPTION(ENABLE_NETWORK_ACCESS "Enable|disable all forms of network access" ON)
+MESSAGE(">>> ENABLE_NETWORK_ACCESS=${ENABLE_NETWORK_ACCESS}")
+if(NOT ENABLE_NETWORK_ACCESS)
+MESSAGE(WARNING "ENABLE_NETWORK_ACCESS=NO => ENABLE_DAP[4]=NO")
+SET(ENABLE_DAP OFF CACHE BOOL "ENABLE_NETWORK_ACCESS=NO => ENABLE_DAP=NO" FORCE)
+SET(ENABLE_DAP4 OFF CACHE BOOL "ENABLE_NETWORK_ACCESS=NO => ENABLE_DAP4=NO" FORCE)
+ENDIF()
+
 IF(ENABLE_HDF4)
   SET(USE_HDF4 ON)
   # Check for include files, libraries.
@@ -622,9 +631,6 @@ IF(ENABLE_NETCDF_4 AND NOT ENABLE_LOGGING AND ENABLE_SET_LOG_LEVEL_FUNC)
   SET(ENABLE_SET_LOG_LEVEL ON)
 ENDIF()
 
-# This has multiversion capability
-SET(ENABLE_MULTIFILTERS yes CACHE BOOL "")
-
 # Option to allow for strict null file padding.
 # See https://github.com/Unidata/netcdf-c/issues/657 for more information
 OPTION(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING "Enable strict null byte header padding." OFF)
@@ -633,12 +639,6 @@ IF(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING)
   SET(USE_STRICT_NULL_BYTE_HEADER_PADDING ON CACHE BOOL "")
 ENDIF(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING)
 
-# Option for building RPC
-#OPTION(ENABLE_RPC "Enable RPC Client and Server." OFF)
-#IF(ENABLE_RPC)
-#  SET(BUILD_RPC ON CACHE BOOL "")
-#ENDIF()
-
 # Note that szip management is tricky.
 # This is because we have three things to consider:
 # 1. is libsz available?
@@ -995,6 +995,12 @@ ELSE()
   SET(FOUND_CURL FALSE)
 ENDIF()
 
+# Start disabling if curl not found
+IF(NOT FOUND_CURL)
+  MESSAGE(WARNING "ENABLE_NETWORK_ACCESS requires libcurl; disabling")
+  SET(ENABLE_NETWORK_ACCESS OFF CACHE BOOL "ENABLE_NETWORK_ACCESS requires libcurl; disabling" FORCE)
+ENDIF()
+
 set (CMAKE_REQUIRED_INCLUDES ${CURL_INCLUDE_DIRS})
 # Check to see if we have libcurl 7.66 or later
 CHECK_C_SOURCE_COMPILES("
@@ -1078,6 +1084,11 @@ ENDIF()
 
 # Option to support byte-range reading of remote datasets
 OPTION(ENABLE_BYTERANGE "Enable byte-range access to remote datasets.." ON)
+if(NOT ENABLE_NETWORK_ACCESS)
+MESSAGE(WARNING "ENABLE_NETWORK_ACCESS=NO => ENABLE_BYTERANGE=NO")
+SET(ENABLE_BYTERANGE OFF CACHE BOOL "ENABLE_NETWORK_ACCESS=NO => ENABLE_BYTERANGE=NO" FORCE)
+ENDIF()
+
 
 # Check for the math library so it can be explicitly linked.
 IF(NOT WIN32)
@@ -1098,8 +1109,14 @@ ENDIF()
 OPTION(ENABLE_DAP_REMOTE_TESTS "Enable DAP remote tests." ON)
 OPTION(ENABLE_EXTERNAL_SERVER_TESTS "Enable external Server remote tests." OFF)
 OPTION(ENABLE_DAP_LONG_TESTS "Enable DAP long tests." OFF)
-SET(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers to use for remote test")
 
+if(NOT ENABLE_DAP)
+SET(ENABLE_DAP_REMOTE_TESTS OFF CACHE BOOL "" FORCE)
+SET(ENABLE_EXTERNAL_SERVER_TESTS OFF CACHE BOOL "" FORCE)
+SET(ENABLE_DAP_LONG_TESTS OFF CACHE BOOL "" FORCE)
+ENDIF()
+
+SET(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers to use for remote test")
 SET(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers to use for remote test")
 
 # See if we have zlib
@@ -1273,6 +1290,12 @@ OPTION(ENABLE_S3 "Enable S3 support." OFF)
 OPTION(ENABLE_S3_INTERNAL "Enable S3 Internal support." OFF)
 OPTION(ENABLE_NCZARR_S3 "Enable NCZarr S3 support; Deprecated in favor of ENABLE_S3" OFF)
 
+IF(NOT ENABLE_NETWORK_ACCESS)
+  SET(ENABLE_S3 OFF CACHE BOOL "" FORCE)
+  SET(ENABLE_S3_INTERNAL OFF CACHE BOOL "" FORCE)
+  SET(ENABLE_NCZARR_S3 OFF CACHE BOOL "" FORCE)
+ENDIF()
+
 # Control S3 Testing: Multi-valued option
 SET(WITH_S3_TESTING OFF CACHE STRING "Control S3 Testing: ON (i.e. all) OFF (i.e. none)  PUBLIC")
     SET_PROPERTY(CACHE WITH_S3_TESTING PROPERTY STRINGS ON OFF PUBLIC)  #
@@ -1290,6 +1313,15 @@ if (NOT ENABLE_S3 AND ENABLE_NCZARR_S3)
 ENDIF()
 UNSET(ENABLE_NCZARR_S3)
 
+IF(NOT ENABLE_NETWORK_ACCESS)
+  MESSAGE(WARNING "ENABLE_NETWORK_ACCESS=NO => disable all s3 functionality")
+  SET(ENABLE_S3 OFF CACHE BOOL "" FORCE)
+  SET(ENABLE_S3_INTERNAL OFF CACHE BOOL "" FORCE)
+  SET(ENABLE_NCZARR_S3 OFF CACHE BOOL "" FORCE)
+  SET(ENABLE_HDF5_ROS3 OFF CACHE BOOL "Use ROS3" FORCE)
+  SET(WITH_S3_TESTING OFF CACHE STRING "" FORCE)
+ENDIF()
+
 # Note we check for the library after checking for enable_s3
 # because for some reason this screws up if we unconditionally test for sdk
 # and it is not available. Fix someday
@@ -1332,29 +1364,6 @@ IF(WITH_S3_TESTING STREQUAL "PUBLIC" OR WITH_S3_TESTING)
 ENDIF()
 ENDIF()
 
-# Start disabling if curl not found
-IF(NOT FOUND_CURL)
-  IF(ENABLE_BYTERANGE)
-    MESSAGE(FATAL_ERROR "Byte-range support specified, CURL libraries are not found.")
-  ENDIF()
-
-  IF(ENABLE_DAP2 OR ENABLE_DAP4)
-    MESSAGE(WARNING "CURL libraries are not found; DAP support disabled")
-    SET(ENABLE_DAP2 OFF CACHE BOOL "Use DAP2" FORCE)
-    SET(ENABLE_DAP4 OFF CACHE BOOL "Use DAP4" FORCE)
-  ENDIF()
-
-  IF(ENABLE_HDF5_ROS3)
-    MESSAGE(WARNING "CURL libraries are not found; ROS3 support disabled.")
-    SET(ENABLE_HDF5_ROS3 OFF CACHE BOOL "Use ROS3" FORCE)
-  ENDIF()
-
-  IF(ENABLE_S3)
-    MESSAGE(FATAL_ERROR "S3 support specified, but CURL libraries are not found.")
-  ENDIF()
-ENDIF(NOT FOUND_CURL)
-
-
 OPTION(ENABLE_LIBXML2 "Link against libxml2 if it is available, use the packaged tinyxml2 parser otherwise." ON)
 SET(XMLPARSER "tinyxml2 (bundled)")
 
@@ -2569,7 +2578,6 @@ is_enabled(ENABLE_S3_INTERNAL HAS_S3_INTERNAL)
 is_enabled(HAS_HDF5_ROS3 HAS_HDF5_ROS3)
 is_enabled(ENABLE_NCZARR HAS_NCZARR)
 is_enabled(ENABLE_NCZARR_ZIP HAS_NCZARR_ZIP)
-is_enabled(ENABLE_MULTIFILTERS HAS_MULTIFILTERS)
 is_enabled(ENABLE_NCZARR_ZIP DO_NCZARR_ZIP_TESTS)
 is_enabled(ENABLE_QUANTIZE HAS_QUANTIZE)
 is_enabled(ENABLE_LOGGING HAS_LOGGING)
@@ -2579,6 +2587,7 @@ is_enabled(HAVE_SZ HAS_SZLIB_WRITE)
 is_enabled(HAVE_ZSTD HAS_ZSTD)
 is_enabled(HAVE_BLOSC HAS_BLOSC)
 is_enabled(HAVE_BZ2 HAS_BZ2)
+is_enabled(ENABLE_NETWORK_ACCESS DO_NETWORK_ACCESS)
 
 if(ENABLE_S3_INTERNAL)
 SET(WHICH_S3_SDK "internal")

configure.ac

@@ -178,6 +178,12 @@ AC_ARG_ENABLE([pnetcdf], [AS_HELP_STRING([--enable-pnetcdf],
 test "x$enable_pnetcdf" = xyes || enable_pnetcdf=no
 AC_MSG_RESULT($enable_pnetcdf)
 
+AC_MSG_CHECKING([whether any network access should be allowed])
+AC_ARG_ENABLE([network-access], [AS_HELP_STRING([--enable-network-access],
+              [enable|disable all forms of network access (default enabled)])])
+test "x$enable_network_access" = xno || enable_network_access=yes
+AC_MSG_RESULT($enable_network_access)
+
 # We need curl for remote operations
 AC_CHECK_LIB([curl],[curl_easy_setopt],[found_curl=yes],[found_curl=no])
 if test "x$found_curl" = "xyes" ; then
@@ -192,6 +198,11 @@ AC_ARG_ENABLE([dap],
 test "x$enable_dap" = xno || enable_dap=yes
 AC_MSG_RESULT($enable_dap)
 
+if test "x$enable_network_access" = xno ; then
+  AC_MSG_WARN([All network access is disabled => DAP support disabled.])
+  enable_dap=no
+fi
+
 if test "x$enable_dap" = xyes & test "x$found_curl" = xno ; then
   AC_MSG_WARN([curl required for dap access. DAP support disabled.])
   enable_dap=no
@@ -641,7 +652,7 @@ AC_ARG_ENABLE([dap-auth-tests],
 test "x$enable_dap_auth_tests" = xyes || enable_dap_auth_tests=no
 AC_MSG_RESULT($enable_dap_auth_tests)
 
-# dap must be enabled
+# dap must be enabled in order to use various other flags
 
 if test "x$enable_dap" = "xno" ; then
   AC_MSG_NOTICE([--disable-dap => --disable-dap-remote-tests --disable-auth-tests --disable-external-server-tests])
@@ -738,7 +749,6 @@ if test "x$have_zstd" = "xyes" ; then
    fi
 fi
 
-
 # See if we have libbz2
 AC_CHECK_LIB([bz2],[BZ2_bzCompress],[have_bz2=yes],[have_bz2=no])
 if test "x$have_bz2" = "xyes" ; then
@@ -799,6 +809,11 @@ AC_ARG_ENABLE([s3],
 test "x$enable_s3" = xyes || enable_s3=no
 AC_MSG_RESULT($enable_s3)
 
+if test "x$enable_network_access" = xno ; then
+  AC_MSG_WARN([--disable-network-access => --disable-s3])
+  enable_s3=no
+fi
+
 # --enable-nczarr-s3 is synonym for --enable-s3 (but...)
 AC_MSG_CHECKING([whether netcdf NCZarr S3 support should be enabled])
 AC_ARG_ENABLE([nczarr-s3],
@@ -807,7 +822,7 @@ AC_ARG_ENABLE([nczarr-s3],
 AC_MSG_RESULT([$enable_nczarr_s3 (Deprecated) Please use --enable-s3)])
 
 # Set enable_s3 instead of enable_nczarr_s3
-if test "x$enable_s3" = xno && test "x$enable_nczarr_s3" = xyes ; then
+if test "x$enable_s3" = xno && test "x$enable_nczarr_s3" = xyes && test "x$enable_network_access" = xyes; then
 enable_s3=yes # back compatibility
 fi
 unset enable_nczarr_s3
@@ -898,12 +913,6 @@ AC_ARG_WITH([chunk-cache-size-nczarr],
 AC_MSG_RESULT([$CHUNK_CACHE_SIZE_NCZARR])
 AC_DEFINE_UNQUOTED([CHUNK_CACHE_SIZE_NCZARR], [$CHUNK_CACHE_SIZE_NCZARR], [default nczarr chunk cache size.])
 
-# This has multiversion capability
-AC_MSG_CHECKING([whether multi-filter support is enabled])
-has_multifilters=yes
-AC_MSG_RESULT($has_multifilters)
-AC_DEFINE([HAS_MULTIFILTERS], [1], [if true, multi-filter support enabled])
-
 # Check whether we want to enable strict null byte header padding.
 # See https://github.com/Unidata/netcdf-c/issues/657 for more information.
 AC_MSG_CHECKING([whether to enable strict null-byte header padding when reading (default off)])
@@ -1310,6 +1319,11 @@ AC_ARG_ENABLE([byterange],
 test "x$enable_byterange" = xno || enable_byterange=yes
 AC_MSG_RESULT($enable_byterange)
 
+if test "x$enable_network_access" = xno ; then
+  AC_MSG_WARN([--disable-network-access => --disable-byterange])
+  enable_byterange=no
+fi
+
 # Need curl for byte ranges
 if test "x$found_curl" = xno && test "x$enable_byterange" = xyes ; then
   AC_MSG_ERROR([curl required for byte range support. Install curl or build without --enable-byterange.])
@@ -1433,7 +1447,6 @@ AC_CHECK_SIZEOF(size_t)
 $SLEEPCMD
 AC_CHECK_SIZEOF(unsigned long long)
 
-
 if test "$ac_cv_sizeof_size_t" -lt "8" ; then
    if test "x${enable_cdf5}" = xyes ; then
       dnl unable to support CDF5, but --enable-cdf5 is explicitly set
@@ -1860,7 +1873,6 @@ AM_CONDITIONAL(ENABLE_NCZARR, [test "x$enable_nczarr" = xyes])
 AM_CONDITIONAL(ENABLE_S3_TESTPUB, [test "x$with_s3_testing" != xno]) # all => public
 AM_CONDITIONAL(ENABLE_S3_TESTALL, [test "x$with_s3_testing" = xyes])
 AM_CONDITIONAL(ENABLE_NCZARR_ZIP, [test "x$enable_nczarr_zip" = xyes])
-AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes])
 AM_CONDITIONAL(HAVE_DEFLATE, [test "x$have_deflate" = xyes])
 AM_CONDITIONAL(HAVE_SZ, [test "x$have_sz" = xyes])
 AM_CONDITIONAL(HAVE_H5Z_SZIP, [test "x$enable_hdf5_szip" = xyes])
@@ -1971,7 +1983,6 @@ AC_SUBST(HAS_HDF5_ROS3,[$has_hdf5_ros3])
 AC_SUBST(HAS_NCZARR,[$enable_nczarr])
 AC_SUBST(DO_S3_TESTING,[$with_s3_testing])
 AC_SUBST(HAS_NCZARR_ZIP,[$enable_nczarr_zip])
-AC_SUBST(HAS_MULTIFILTERS,[$has_multifilters])
 AC_SUBST(DO_NCZARR_ZIP_TESTS,[$enable_nczarr_zip])
 AC_SUBST(HAS_QUANTIZE,[$enable_quantize])
 AC_SUBST(HAS_LOGGING,[$enable_logging])
@@ -1981,6 +1992,7 @@ AC_SUBST(HAS_SZLIB,[$have_sz])
 AC_SUBST(HAS_SZLIB_WRITE, [$have_sz])
 AC_SUBST(HAS_ZSTD,[$have_zstd])
 AC_SUBST(DO_LARGE_TESTS,[$enable_large_file_tests])
+AC_SUBST(DO_NETWORK_ACCESS,[$enable_network_access])
 
 if test "x$enable_s3_aws" = xyes ; then
 AC_SUBST(WHICH_S3_SDK,[aws-sdk-cpp])
@@ -2115,7 +2127,6 @@ AX_SET_META([NC_HAS_S3_AWS],[$enable_s3_aws],[yes])
 AX_SET_META([NC_HAS_S3_INTERNAL],[$enable_s3_internal],[yes])
 AX_SET_META([NC_HAS_HDF5_ROS3],[$has_hdf5_ros3],[yes])
 AX_SET_META([NC_HAS_NCZARR],[$enable_nczarr],[yes])
-AX_SET_META([NC_HAS_MULTIFILTERS],[$has_multifilters],[yes])
 AX_SET_META([NC_HAS_LOGGING],[$enable_logging],[yes])
 AX_SET_META([NC_HAS_QUANTIZE],[$enable_quantize],[yes])
 AX_SET_META([NC_HAS_SZIP],[$enable_hdf5_szip],[yes])

include/netcdf_meta.h.in

@@ -60,7 +60,6 @@
 #define NC_RELAX_COORD_BOUND 1 /*!< Always allow 0 counts in parallel I/O. */
 #define NC_DISPATCH_VERSION  @NC_DISPATCH_VERSION@ /*!< Dispatch table version. */
 #define NC_HAS_PAR_FILTERS   @NC_HAS_PAR_FILTERS@ /* Parallel I/O with filter support. */
-#define NC_HAS_MULTIFILTERS  @NC_HAS_MULTIFILTERS@ /*!< Nczarr support. */
 #define NC_HAS_LOGGING       @NC_HAS_LOGGING@ /*!< Logging support. */
 #define NC_HAS_QUANTIZE      @NC_HAS_QUANTIZE@ /*!< Quantization support. */
 #define NC_HAS_ZSTD          @NC_HAS_ZSTD@ /*!< Zstd support. */

libdispatch/ds3util.c

@@ -150,10 +150,10 @@ NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** ne
 
     /* Construct the revised path */
     ncbytesclear(buf);
-    ncbytescat(buf,"/");
-    if(bucket == NULL)
-        {stat = NC_EURL; goto done;}
-    ncbytescat(buf,bucket);
+    if(bucket != NULL) {
+        ncbytescat(buf,"/");
+        ncbytescat(buf,bucket);
+    }
     for(i=0;i<nclistlength(pathsegments);i++) {
 	ncbytescat(buf,"/");
 	ncbytescat(buf,nclistget(pathsegments,i));

libnetcdf.settings.in

@@ -51,11 +51,10 @@ JNA Support:		@HAS_JNA@
 ERANGE Fill Support:	@HAS_ERANGE_FILL@
 Relaxed Boundary Check:	@RELAX_COORD_BOUND@
 
-Multi-Filter Support:	@HAS_MULTIFILTERS@
 Quantization:		@HAS_QUANTIZE@
 Logging:     		@HAS_LOGGING@
 SZIP Write Support:     @HAS_SZLIB_WRITE@
 Standard Filters:       @STD_FILTERS@
 ZSTD Support:           @HAS_ZSTD@
 Parallel Filters:       @HAS_PAR_FILTERS@
-
+Allow Network Access:   @DO_NETWORK_ACCESS@

nc-config.cmake.in

@@ -116,12 +116,7 @@ else
     has_benchmarks="yes"
 fi
 
-has_multifilters="@HAS_MULTIFILTERS@"
-if [ -z "$has_multifilters" -o "$has_multifilters" = "OFF" -o "$has_multifilters" = "FALSE" ]; then
-    has_multifilters="no"
-else
-    has_multifilters="yes"
-fi
+has_multifilters="yes"
 
 has_zstd="@HAS_ZSTD"
 if [ -z "$has_zstd" -o "$has_zstd" = "OFF" -o "$has_zstd" = "FALSE" ]; then

nc-config.in

@@ -35,7 +35,7 @@ has_parallel4="@HAS_PARALLEL4@"
 has_nczarr="@HAS_NCZARR@"
 has_zstd="@HAS_ZSTD@"
 has_benchmarks="@HAS_BENCHMARKS@"
-has_multifilters="@HAS_MULTIFILTERS@"
+has_multifilters="yes"
 has_quantize="@HAS_QUANTIZE@"
 has_stdfilters="@STD_FILTERS@"
 version="@PACKAGE_NAME@ @PACKAGE_VERSION@"

ncdump/test_rcmerge.sh

@@ -105,7 +105,7 @@ mergecase3() {
     # Test cross file overrides
     resetrc
     rm -f tmp_rcmerge.txt tmp_rcmerge.txt
-    echo "ncrc=ncrc1" >> $HOME/.ncrc
+    echo "ncrc=ncrc1" >> $RCHOME/.ncrc
     echo "ncrcx=ncrcx" >> $RCHOME/.ncrc
     echo "ncrc=ncrc2" >> $RCHOME/.dodsrc
     echo "daprc=daprc" >> $RCHOME/.daprc
@@ -138,7 +138,4 @@ mergecase3
 # Test the .rc api
 rcapi1
 
-# Test the .rc api
-rcapi1
-
 resetrc