From fb40a72b45266d16fa1a0e8b4d04e0f107b45183 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 20 May 2023 17:11:25 -0600 Subject: [PATCH] Improve performance of the nc_reclaim_data and nc_copy_data functions. re: Issue https://github.com/Unidata/netcdf-c/issues/2685 re: PR https://github.com/Unidata/netcdf-c/pull/2179 As noted in PR https://github.com/Unidata/netcdf-c/pull/2179, the old code did not allow for reclaiming instances of types, nor for properly copying them. That PR provided new functions capable of reclaiming/copying instances of arbitrary types. However, as noted by Issue https://github.com/Unidata/netcdf-c/issues/2685, using these most general functions resulted in a significant performance degradation, even for common cases. This PR attempts to mitigate the cost of using the general reclaim/copy functions in two ways. First, the previous functions operating at the top level by using ncid and typeid arguments. These functions were augmented with equivalent versions that used the netcdf-c library internal data structures to allow direct access to needed information. These new functions are used internally to the library. The second mitigation involves optimizing the internal functions by providing early tests for common cases. This avoids unnecessary recursive function calls. The overall result is a significant improvement in speed by a factor of roughly twenty -- your mileage may vary. These optimized functions are still not as fast as the original (more limited) functions, but they are getting close. Additional optimizations are possible. But the cost is a significant "uglification" of the code that I deemed a step too far, at least for now. ## Misc. Changes 1. Added a test case to check the proper reclamation/copy of complex types. 2. Found and fixed some places where nc_reclaim/copy should have been used. 3. Replaced, in the netcdf-c library, (almost all) occurrences of nc_reclaim_copy with calls to NC_reclaim/copy. This plus the optimizations is the primary speed-up mechanism. 4. In DAP4, the metadata is held in a substrate in-memory file; this required some changes so that the reclaim/copy code accessed that substrate dispatcher rather than the DAP4 dispatcher. 5. Re-factored and isolated the code that computes if a type is (transitively) variable-sized or not. 6. Clean up the reclamation code in ncgen; adding the use of nc_reclaim exposed some memory problems. --- RELEASE_NOTES.md | 1 + configure.ac | 20 +- docs/cloud.md | 2 +- docs/internal.md | 129 ++++--- include/nc.h | 60 ++++ include/nc4internal.h | 7 +- include/ncoffsets.h | 10 + include/netcdf.h | 77 +++-- include/netcdf_aux.h | 7 + libdap2/ncd2dispatch.c | 20 ++ libdap4/d4debug.c | 5 +- libdap4/d4file.c | 28 +- libdap4/ncd4.h | 2 +- libdispatch/CMakeLists.txt | 2 +- libdispatch/Makefile.am | 2 +- libdispatch/daux.c | 28 ++ libdispatch/dinstance.c | 525 ++++++---------------------- libdispatch/dinstance_intern.c | 610 +++++++++++++++++++++++++++++++++ libdispatch/dvar.c | 13 +- libdispatch/dvlen.c | 84 ++--- libdispatch/ncs3sdk_aws.cpp | 7 +- libhdf5/hdf5attr.c | 16 +- libhdf5/hdf5internal.c | 2 +- libhdf5/hdf5open.c | 17 +- libhdf5/hdf5type.c | 24 +- libhdf5/hdf5var.c | 4 +- libnczarr/zattr.c | 24 +- libnczarr/zinternal.c | 2 +- libnczarr/zsync.c | 4 +- libnczarr/ztype.c | 4 + libnczarr/zutil.c | 10 +- libnczarr/zvar.c | 2 +- libnczarr/zwalk.c | 5 +- libnczarr/zxcache.c | 10 +- libsrc4/nc4attr.c | 2 +- libsrc4/nc4internal.c | 12 +- libsrc4/nc4type.c | 85 +++-- libsrc4/nc4var.c | 2 +- nc_test4/tst_vlenstr.c | 40 +-- ncgen/genbin.c | 127 ++++--- ncgen/generate.c | 3 +- ncgen/generate.h | 2 +- nczarr_test/zmapio.c | 6 +- plugins/Makefile.am | 5 +- unit_test/CMakeLists.txt | 21 +- unit_test/Makefile.am | 13 +- unit_test/reclaim_tests.cdl | 75 ++++ unit_test/run_reclaim_tests.sh | 16 + unit_test/tst_reclaim.c | 345 +++++++++++++++++++ 49 files changed, 1702 insertions(+), 815 deletions(-) create mode 100644 libdispatch/dinstance_intern.c create mode 100644 unit_test/reclaim_tests.cdl create mode 100755 unit_test/run_reclaim_tests.sh create mode 100644 unit_test/tst_reclaim.c diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 7b87710836..feecc5a6e4 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -8,6 +8,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD +* Improve performance and testing of the new nc_reclaim/copy functions. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). * Improve S3 documentation, testing, and support See [Github #2686](https://github.com/Unidata/netcdf-c/pull/2686). * Remove obsolete code. See [Github #2680](https://github.com/Unidata/netcdf-c/pull/2680). * [Bug Fix] Add a crude test to see if an NCZarr path looks like a valid NCZarr/Zarr file. See [Github #2658](https://github.com/Unidata/netcdf-c/pull/2658). diff --git a/configure.ac b/configure.ac index 63fdb9fe42..edf765783c 100644 --- a/configure.ac +++ b/configure.ac @@ -1488,10 +1488,14 @@ AC_CHECK_SIZEOF(ssize_t) $SLEEPCMD AC_CHECK_SIZEOF([void*]) -if test "x$enable_hdf5" = xyes || test "x$enable_dap" = xyes ; then - AC_SEARCH_LIBS([deflate], [zlibwapi zlibstat zlib zlib1 z], [], [ - AC_MSG_ERROR([Can't find or link to the z library. Turn off netCDF-4 and \ - DAP clients with --disable-hdf5 --disable-dap, or see config.log for errors.])]) +# Check for deflate library +AC_SEARCH_LIBS([deflate], [zlibwapi zlibstat zlib zlib1 z], [have_deflate=yes], [have_deflate=no]) +AC_MSG_CHECKING([whether deflate library is available]) +AC_MSG_RESULT([${have_deflate}]) + +if test "x$have_deflate" = xno ; then + AC_MSG_ERROR([Can't find or link to the z library. Turn off netCDF-4 and \ + DAP and NCZarr clients with --disable-hdf5 --disable-dap --disable-nczarr, or see config.log for errors.]) fi # We need the math library @@ -1857,6 +1861,7 @@ AM_CONDITIONAL(ENABLE_S3_TESTPUB, [test "x$with_s3_testing" != xno]) # all => pu AM_CONDITIONAL(ENABLE_S3_TESTALL, [test "x$with_s3_testing" = xyes]) AM_CONDITIONAL(ENABLE_NCZARR_ZIP, [test "x$enable_nczarr_zip" = xyes]) AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes]) +AM_CONDITIONAL(HAVE_DEFLATE, [test "x$have_deflate" = xyes]) AM_CONDITIONAL(HAVE_SZ, [test "x$have_sz" = xyes]) AM_CONDITIONAL(HAVE_H5Z_SZIP, [test "x$enable_hdf5_szip" = xyes]) AM_CONDITIONAL(HAVE_BLOSC, [test "x$have_blosc" = xyes]) @@ -1971,6 +1976,7 @@ AC_SUBST(DO_NCZARR_ZIP_TESTS,[$enable_nczarr_zip]) AC_SUBST(HAS_QUANTIZE,[$enable_quantize]) AC_SUBST(HAS_LOGGING,[$enable_logging]) AC_SUBST(DO_FILTER_TESTS,[$enable_filter_testing]) +AC_SUBST(HAS_DEFLATE,[$have_deflate]) AC_SUBST(HAS_SZLIB,[$have_sz]) AC_SUBST(HAS_SZLIB_WRITE, [$have_sz]) AC_SUBST(HAS_ZSTD,[$have_zstd]) @@ -1987,8 +1993,10 @@ AC_SUBST(WHICH_S3_SDK,[none]) fi # Always available -std_filters="deflate bz2" - +std_filters="bz2" +if test "x$have_deflate" = xyes ; then +std_filters="${std_filters} deflate" +fi if test "x$have_sz" = xyes ; then std_filters="${std_filters} szip" fi diff --git a/docs/cloud.md b/docs/cloud.md index fb459080df..69e8f50a29 100644 --- a/docs/cloud.md +++ b/docs/cloud.md @@ -258,7 +258,7 @@ This is an experimental SDK provided internally in the netcdf-c library. * It is written in C * It provides the minimum functionality necessary to read/write/search an Amazon S3 bucket. -* It was constructed by heavily modifying the HDF5 *H5FDros3* Virtual File Driver and combining it with crypto code wrappers provided by libcurl. The resulting file was then modified to fit into the netcdf coding style. +* It was constructed by heavily modifying the HDF5 *H5Fs3commons.c* file and combining it with crypto code wrappers provided by libcurl. The resulting file was then modified to fit into the netcdf coding style. * The resulting code is rather ugly, but appears to work under at least Linux and under Windows (using Visual C++). ### Dependencies diff --git a/docs/internal.md b/docs/internal.md index ccc509514c..fc6f06618e 100644 --- a/docs/internal.md +++ b/docs/internal.md @@ -4,10 +4,14 @@ Notes On the Internals of the NetCDF-C Library # Notes On the Internals of the NetCDF-C Library {#intern_head} -\tableofcontents - This document attempts to record important information about the internal architecture and operation of the netcdf-c library. +It covers the following issues. + +* [Including C++ Code in the netcdf-c Library](#intern_c++) +* [Managing instances of variable-length data types](#intern_vlens) +* [Inferring File Types](#intern_infer) +* [Adding a Standard Filter](#intern_filters) # 1. Including C++ Code in the netcdf-c Library {#intern_c++} @@ -41,46 +45,71 @@ AM_CXXFLAGS = -std=c++11 ```` It is possible that other values (e.g. *-std=c++14*) may also work. -# 2. Managing instances of complex data types +# 2. Managing instances of variable-length data types {#intern_vlens} For a long time, there have been known problems with the management of complex types containing VLENs. This also -involves the string type because it is stored as a VLEN of -chars. +involves the string type because it is stored as a VLEN of chars. -The term complex type refers to any type that directly or -recursively references a VLEN type. So an array of VLENS, a -compound with a VLEN field, and so on. +The term "variable-length" refers to any +type that directly or recursively references a VLEN type. So an +array of VLENS, a compound with a VLEN field, and so on. -In order to properly handle instances of these complex types, it +In order to properly handle instances of these variable-length types, it is necessary to have function that can recursively walk instances of such types to perform various actions on them. The term "deep" is also used to mean recursive. -Two deep walking operations are provided by the netcdf-c library -to aid in managing instances of complex structures. -- free'ing an instance of the complex type -- copying an instance of the complex type. +Two primary deep walking operations are provided by the netcdf-c library +to aid in managing instances of variable-length types. +- free'ing an instance of the type +- copying an instance of the type. + +Note that the term "vector" is used in the following text to +mean a contiguous (in memory) sequence of instances of some +type. Given an array with, say, dimensions 2 X 3 X 4, this will +be stored in memory as a vector of length 2\*3\*4=24 instances. + +## Special case reclamation functions + +Previously The netcdf-c library provided functions to reclaim an +instance of a subset of the possible variable-length types. It +provided no specialized support for copying instances of +variable-length types. + +These specialized functions are still available and can be used +when their pre-conditions are met. They have the advantage of +being faster than the general purpose functions described +later in this document. -Previously The netcdf-c library only did shallow free and shallow copy of -complex types. This meant that only the top level was properly -free'd or copied, but deep internal blocks in the instance were -not touched. This led to a host of memory leaks and failures -when the deep data was effectively shared between the netcdf-c library -internally and the user's data. +These functions, and their pre and post conditions, are as follows. +* *int nc_free_string(size_t len, char\* data[])* +Action: reclaim a vector of variable length strings. +Pre-condition(s): the data argument is a vector containing *len* pointers to variable length, nul-terminated strings. +Post-condition(s): only the strings in the vector are reclaimed -- the vector itself is not reclaimed. -Note that the term "vector" is used to mean a contiguous (in -memory) sequence of instances of some type. Given an array with, -say, dimensions 2 X 3 X 4, this will be stored in memory as a -vector of length 2\*3\*4=24 instances. +* int nc_free_vlens(size_t len, nc_vlen_t vlens[]) +Action: reclaim a vector of VLEN instances. +Pre-condition(s): the data argument is a vector containing *len* pointers to variable length, nul-terminated strings. +Post-condition(s): + * only the data pointed to by the VLEN instances (i.e. *nc_vlen_t.p* in the vector are reclaimed -- the vector itself is not reclaimed. + * the base type of the VLEN must be a fixed-size type -- this means atomic types in the range NC_CHAR thru NC_UINT64, and compound types where the basetype of each field is itself fixed-size. -The use cases are primarily these. +* *int nc_free_vlen(nc_vlen_t \*vl)* +Action: this is equivalent to calling *nc_free_vlens(1,&vl)*. -## nc\_get\_vars +If the pre and post conditions are not met, then using these +functions can lead to a host of memory leaks and failures +because the deep data variable-length data is in effect +shared between the netcdf-c library internally and the user's data. + +## Typical use cases + +### *nc\_get\_vars* Suppose one is reading a vector of instances using nc\_get\_vars (or nc\_get\_vara or nc\_get\_var, etc.). These functions will return the vector in the top-level memory provided. All -interior blocks (form nested VLEN or strings) will have been +interior blocks (from nested VLEN or strings) will have been dynamically allocated. Note that computing the size of the vector may be tricky because the strides must be taken into account. @@ -90,14 +119,7 @@ memory leak occurs. So, the recursive reclaim function is used to walk the returned instance vector and do a deep reclaim of the data. -Currently functions are defined in netcdf.h that are supposed to -handle this: nc\_free\_vlen(), nc\_free\_vlens(), and -nc\_free\_string(). Unfortunately, these functions only do a -shallow free, so deeply nested instances are not properly -handled by them. They are marked in the description as -deprecated in favor of the newer recursive function. - -## nc\_put\_vars +### *nc\_put\_vars* Suppose one is writing a vector of instances using nc\_put\_vars (or nc\_put\_vara or nc\_put\_var, etc.). These functions will @@ -113,7 +135,10 @@ So again, the recursive reclaim function can be used to walk the returned instance vector and do a deep reclaim of the data. -## nc\_put\_att +WARNING: If the data passed into these functions contains statically +allocated data, then using any of the reclaim functions will fail. + +### *nc\_put\_att* Suppose one is writing a vector of instances as the data of an attribute using, say, nc\_put\_att. @@ -122,15 +147,11 @@ so that changes/reclamation of the input data will not affect the attribute. Note that this copying behavior is different from writing to a variable, where the data is written immediately. -Again, the code inside the netcdf library used to use only shallow copying -rather than deep copy. As a result, one saw effects such as described -in Github Issue https://github.com/Unidata/netcdf-c/issues/2143. - -Also, after defining the attribute, it may be necessary for the user +After defining the attribute, it may be necessary for the user to free the data that was provided as input to nc\_put\_att() as in the nc\_put\_xxx functions (previously described). -## nc\_get\_att +### *nc\_get\_att* Suppose one is reading a vector of instances as the data of an attribute using, say, nc\_get\_att. @@ -138,10 +159,6 @@ Internally, the existing attribute data must be copied and returned to the caller, and the caller is responsible for reclaiming the returned data. -Again, the code inside the netcdf library used to only do shallow copying -rather than deep copy. So this could lead to memory leaks and errors -because the deep data was shared between the library and the user. - ## New Instance Walking API Proper recursive functions were added to the netcdf-c library to @@ -156,14 +173,14 @@ int nc_reclaim_data_all(int ncid, nc_type xtypeid, void* memory, size_t count); int nc_copy_data(int ncid, nc_type xtypeid, const void* memory, size_t count, void* copy); int nc_copy_data_all(int ncid, nc_type xtypeid, const void* memory, size_t count, void** copyp); ```` -There are two variants. The first two, nc\_reclaim\_data() and +There are two variants. The two functions, nc\_reclaim\_data() and nc\_copy\_data(), assume the top-level vector is managed by the caller. For reclaim, this is so the user can use, for example, a statically allocated vector. For copy, it assumes the user provides the space into which the copy is stored. -The second two, nc\_reclaim\_data\_all() and -nc\_copy\_data\_all(), allows the functions to manage the +The other two, nc\_reclaim\_data\_all() and +nc\_copy\_data\_all(), allow the called functions to manage the top-level. So for nc\_reclaim\_data\_all, the top level is assumed to be dynamically allocated and will be free'd by nc\_reclaim\_data\_all(). The nc\_copy\_data\_all() function @@ -171,12 +188,10 @@ will allocate the top level and return a pointer to it to the user. The user can later pass that pointer to nc\_reclaim\_data\_all() to reclaim the instance(s). -# Internal Changes +## Internal Changes The netcdf-c library internals are changed to use the proper reclaim and copy functions. This also allows some simplification of the code since the stdata and vldata fields of NC\_ATT\_INFO are no longer needed. -Currently this is commented out using the SEPDATA \#define macro. -When the bugs are found and fixed, all this code will be removed. ## Optimizations @@ -202,7 +217,7 @@ The classification of types can be made at the time the type is defined or is read in from some existing file. The reclaim and copy functions use this information to speed up the handling of fixed size types. -# Warnings +## Warnings 1. The new API functions require that the type information be accessible. This means that you cannot use these functions @@ -228,7 +243,7 @@ use this information to speed up the handling of fixed size types. } ```` -# 3. Inferring File Types +# 3. Inferring File Types {#intern_infer} As described in the companion document -- docs/dispatch.md -- when nc\_create() or nc\_open() is called, it must figure out what @@ -478,7 +493,7 @@ So for example, if DAP2 is the model, then all netcdf-4 mode flags and some netcdf-3 flags are removed from the set of mode flags because DAP2 provides only a standard netcdf-classic format. -# 4. Adding a Standard Filter +# 4. Adding a Standard Filter {#intern_filters} The standard filter system extends the netcdf-c library API to support a fixed set of "standard" filters. This is similar to the @@ -534,7 +549,7 @@ and CMake (CMakeLists.txt) Configure.ac must have a block that similar to this that locates the implementing library. ```` -# See if we have libzstd +\# See if we have libzstd AC_CHECK_LIB([zstd],[ZSTD_compress],[have_zstd=yes],[have_zstd=no]) if test "x$have_zstd" = "xyes" ; then AC_SEARCH_LIBS([ZSTD_compress],[zstd zstd.dll cygzstd.dll], [], []) @@ -559,7 +574,7 @@ endif ```` ```` -# Need our version of szip if libsz available and we are not using HDF5 +\# Need our version of szip if libsz available and we are not using HDF5 if HAVE_SZ noinst_LTLIBRARIES += libh5szip.la libh5szip_la_SOURCES = H5Zszip.c H5Zszip.h @@ -637,4 +652,4 @@ done: *Author*: Dennis Heimbigner
*Email*: dmh at ucar dot edu
*Initial Version*: 12/22/2021
-*Last Revised*: 01/25/2022 +*Last Revised*: 5/16/2023 diff --git a/include/nc.h b/include/nc.h index 6dc2e44322..0879476849 100644 --- a/include/nc.h +++ b/include/nc.h @@ -77,4 +77,64 @@ extern int iterate_NCList(int i,NC**); /* Walk from 0 ...; ERANGE return => stop extern void free_NC(NC*); extern int new_NC(const struct NC_Dispatch*, const char*, int, NC**); +/* Defined in dinstance_intern.c */ + +/**************************************************/ +/** +Following are the internal implementation signatures upon which +are built the API functions: nc_reclaim_data, nc_reclaim_data_all, +nc_copy_data, and nc_copy_data_all. +These functions access internal data structures instead of using e.g. ncid. +This is to improve performance. +*/ + +/* +Reclaim a vector of instances of arbitrary type. +This recursively walks the top-level instances to reclaim any +nested data such as vlen or strings or such. + +Assumes it is passed a pointer to count instances of xtype. +Reclaims any nested data. + +WARNING: nc_reclaim_data does not reclaim the top-level +memory because we do not know how it was allocated. However +nc_reclaim_data_all does reclaim top-level memory assuming that it +was allocated using malloc(). + +WARNING: all data blocks below the top-level (e.g. string +instances) will be reclaimed, so do not call if there is any +static data in the instance. + +Should work for any netcdf format. +*/ + +EXTERNL int NC_reclaim_data(NC* nc, nc_type xtypeid, void* memory, size_t count); +EXTERNL int NC_reclaim_data_all(NC* nc, nc_type xtypeid, void* memory, size_t count); + +/** +Copy vector of arbitrary type instances. This recursively walks +the top-level instances to copy any nested data such as vlen or +strings or such. + +Assumes it is passed a pointer to count instances of xtype. +WARNING: nc_copy_data does not copy the top-level memory, but +assumes a block of proper size was passed in. However +nc_copy_data_all does allocate top-level memory copy. + +Should work for any netcdf format. +*/ + +EXTERNL int NC_copy_data(NC* nc, nc_type xtypeid, const void* memory, size_t count, void* copy); +EXTERNL int NC_copy_data_all(NC* nc, nc_type xtypeid, const void* memory, size_t count, void** copyp); + +/* Macros to map NC_FORMAT_XX to metadata structure (e.g. NC_FILE_INFO_T) */ +/* Fast test for what file structure is used */ +#define NC3INFOFLAGS ((1<dispatch->model)) & NC3INFOFLAGS) +#define USEFILEINFO(nc) ((1<<(nc->dispatch->model)) & FILEINFOFLAGS) +#define USED2INFO(nc) ((1<<(nc->dispatch->model)) & (1<dispatch->model)) & (1< */ - int varsized; /**< dispatch is the DAP2 dispatcher, then do a level of indirection */ + if(USED2INFO(nc)) { + NCDAPCOMMON* d2 = (NCDAPCOMMON*)nc->dispatchdata; + /* Find pointer to NC struct for this file. */ + (void)NC_check_id(d2->substrate.nc3id,&subnc); + } else subnc = nc; + return subnc; +} + diff --git a/libdap4/d4debug.c b/libdap4/d4debug.c index b9bf0a0eba..d2d40b44a6 100644 --- a/libdap4/d4debug.c +++ b/libdap4/d4debug.c @@ -10,6 +10,8 @@ #include "ncdispatch.h" #include "netcdf_aux.h" +extern NC* NCD4_get_substrate(NC* nc); + #ifdef D4CATCH /* Place breakpoint here to catch errors close to where they occur*/ int @@ -112,7 +114,6 @@ NCD4_debugcopy(NCD4INFO* info) int varid = var->meta.id; d4size_t varsize; size_t dimprod = NCD4_dimproduct(var); - int ncid = info->substrate.nc4id; varsize = type->meta.memsize * dimprod; memory = d4alloc(varsize); @@ -141,7 +142,7 @@ NCD4_debugcopy(NCD4INFO* info) if((ret=nc_put_vara(grpid,varid,NC_coord_zero,edges,memory))) goto done; } - if((ret=ncaux_reclaim_data(ncid,type->meta.id,memory,dimprod))) + if((ret=NC_reclaim_data(NCD4_get_substrate(ncp),type->meta.id,memory,dimprod))) goto done; nullfree(memory); memory = NULL; } diff --git a/libdap4/d4file.c b/libdap4/d4file.c index 998ac15cf3..e8982db1d0 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -275,11 +275,9 @@ freeInfo(NCD4INFO* d4info) when aborted, it should be deleted. But that is not working for some reason, so we delete it ourselves. */ -#if 0 if(d4info->substrate.filename != NULL) { unlink(d4info->substrate.filename); } -#endif } nullfree(d4info->substrate.filename); /* always reclaim */ NCD4_reclaimMeta(d4info->substrate.metadata); @@ -580,15 +578,21 @@ makesubstrate(NCD4INFO* d4info) return THROW(ret); } -int -NCD4_get_substrate(int ncid) +/* This function breaks the abstraction, but is necessary for + code that accesses the underlying netcdf-4 metadata objects. + Used in: NC_reclaim_data[_all] + NC_copy_data[_all] +*/ + +NC* +NCD4_get_substrate(NC* nc) { - NC* nc = NULL; - NCD4INFO* d4 = NULL; - int subncid = 0; - /* Find pointer to NC struct for this file. */ - (void)NC_check_id(ncid,&nc); - d4 = (NCD4INFO*)nc->dispatchdata; - subncid = d4->substrate.nc4id; - return subncid; + NC* subnc = NULL; + /* Iff nc->dispatch is the DAP4 dispatcher, then do a level of indirection */ + if(USED4INFO(nc)) { + NCD4INFO* d4 = (NCD4INFO*)nc->dispatchdata; + /* Find pointer to NC struct for this file. */ + (void)NC_check_id(d4->substrate.nc4id,&subnc); + } else subnc = nc; + return subnc; } diff --git a/libdap4/ncd4.h b/libdap4/ncd4.h index e9f9c20a96..058e40ff7c 100644 --- a/libdap4/ncd4.h +++ b/libdap4/ncd4.h @@ -253,7 +253,7 @@ EXTERNL d4size_t NCD4_getcounter(NCD4offset* p); */ #define HYRAXHACK -EXTERNL int NCD4_get_substrate(int ncid); +EXTERNL NC* NCD4_get_substrate_nc(NC* nc); #endif /*NCD4_H*/ diff --git a/libdispatch/CMakeLists.txt b/libdispatch/CMakeLists.txt index f518a1adbc..730e877c92 100644 --- a/libdispatch/CMakeLists.txt +++ b/libdispatch/CMakeLists.txt @@ -5,7 +5,7 @@ # See netcdf-c/COPYRIGHT file for more info. SET(libdispatch_SOURCES dcopy.c dfile.c ddim.c datt.c dattinq.c dattput.c dattget.c derror.c dvar.c dvarget.c dvarput.c dvarinq.c ddispatch.c nclog.c dstring.c dutf8.c dinternal.c doffsets.c ncuri.c nclist.c ncbytes.c nchashmap.c nctime.c nc.c nclistmgr.c utf8proc.h utf8proc.c dpathmgr.c dutil.c drc.c dauth.c dreadonly.c dnotnc4.c dnotnc3.c dinfermodel.c -daux.c dinstance.c +daux.c dinstance.c dinstance_intern.c dcrc32.c dcrc32.h dcrc64.c ncexhash.c ncxcache.c ncjson.c ds3util.c dparallel.c dmissing.c) # Netcdf-4 only functions. Must be defined even if not used diff --git a/libdispatch/Makefile.am b/libdispatch/Makefile.am index a5bac12c6b..b49d4f6152 100644 --- a/libdispatch/Makefile.am +++ b/libdispatch/Makefile.am @@ -21,7 +21,7 @@ dinternal.c ddispatch.c dutf8.c nclog.c dstring.c ncuri.c nclist.c \ ncbytes.c nchashmap.c nctime.c nc.c nclistmgr.c dauth.c doffsets.c \ dpathmgr.c dutil.c dreadonly.c dnotnc4.c dnotnc3.c dinfermodel.c \ daux.c dinstance.c dcrc32.c dcrc32.h dcrc64.c ncexhash.c ncxcache.c \ -ncjson.c ds3util.c dparallel.c dmissing.c +ncjson.c ds3util.c dparallel.c dmissing.c dinstance_intern.c # Add the utf8 codebase libdispatch_la_SOURCES += utf8proc.c utf8proc.h diff --git a/libdispatch/daux.c b/libdispatch/daux.c index 1618933f1b..2011c93d42 100644 --- a/libdispatch/daux.c +++ b/libdispatch/daux.c @@ -25,6 +25,7 @@ See COPYRIGHT for license information. #include "config.h" #include "netcdf.h" #include "netcdf_aux.h" +#include "nc4internal.h" #include "ncoffsets.h" #include "nclog.h" #include "ncrc.h" @@ -915,6 +916,14 @@ ncaux_reclaim_data_all(int ncid, int xtype, void* memory, size_t count) return nc_reclaim_data_all(ncid, xtype, memory, count); } +EXTERNL int NC_inq_any_type(int ncid, nc_type typeid, char *name, size_t *size, nc_type *basetypep, size_t *nfieldsp, int *classp); + +EXTERNL int +ncaux_inq_any_type(int ncid, nc_type typeid, char *name, size_t *sizep, nc_type *basetypep, size_t *nfieldsp, int *classp) +{ + return NC_inq_any_type(ncid, typeid, name, sizep, basetypep, nfieldsp, classp); +} + /** @param ncid - only needed for a compound type @param xtype - type for which alignment is requested @@ -925,3 +934,22 @@ ncaux_type_alignment(int xtype, int ncid, size_t* alignp) /* Defer to the internal version */ return NC_type_alignment(ncid, xtype, alignp); } + +/** +Dump the output tree of data from a call +to e.g. nc_get_vara or the input to e.g. nc_put_vara. +This function is just a wrapper around nc_dump__data. + +@param ncid file ncid +@param xtype type id +@param memory to print +@param count number of instances of the type in memory +@return error code +*/ + +EXTERNL int +ncaux_dump_data(int ncid, int xtype, void* memory, size_t count, char** bufp) +{ +EXTERNL int nc_dump_data(int ncid, nc_type xtype, void* memory, size_t count, char** bufp); + return nc_dump_data(ncid, xtype, memory, count, bufp); +} diff --git a/libdispatch/dinstance.c b/libdispatch/dinstance.c index 64756929bd..d2a66079b7 100644 --- a/libdispatch/dinstance.c +++ b/libdispatch/dinstance.c @@ -16,6 +16,7 @@ Currently two operations are defined: #include #include #include "netcdf.h" +#include "netcdf_aux.h" #include "nc4internal.h" #include "nc4dispatch.h" #include "ncoffsets.h" @@ -24,42 +25,46 @@ Currently two operations are defined: #undef REPORT #undef DEBUG +/* DAP2 and DAP4 currently defer most of their API to a substrate NC + that hold the true metadata. So there is a level of indirection + necessary in order to get to the right NC* instance. +*/ + +#if defined(ENABLE_DAP4) || defined(ENABLE_DAP2) +EXTERNL NC* NCD4_get_substrate(NC* nc); +EXTERNL NC* NCD2_get_substrate(NC* nc); +static NC* +DAPSUBSTRATE(NC* nc) +{ + if(USED2INFO(nc) != 0) + return NCD2_get_substrate(nc); + else if(USED4INFO(nc) != 0) + return NCD4_get_substrate(nc); + return nc; +} +#else +#define DAPSUBSTRATE(nc) (nc) +#endif /* It is helpful to have a structure that contains memory and an offset */ typedef struct Position{char* memory; ptrdiff_t offset;} Position; -static int type_alignment_initialized = 0; - /* Forward */ #ifdef USE_NETCDF4 -static int reclaim_datar(int ncid, nc_type xtype, Position*); -static int reclaim_compound(int ncid, nc_type xtype, size_t size, size_t nfields, Position* offset); -static int reclaim_vlen(int ncid, nc_type xtype, nc_type basetype, Position* offset); -static int reclaim_enum(int ncid, nc_type xtype, nc_type basetype, Position* offset); -static int reclaim_opaque(int ncid, nc_type xtype, size_t size, Position* offset); - -static int copy_datar(int ncid, nc_type xtype, Position* src, Position* dst); -static int copy_compound(int ncid, nc_type xtype, size_t size, size_t nfields, Position* src, Position* dst); -static int copy_vlen(int ncid, nc_type xtype, nc_type basetype, Position* src, Position* dst); -static int copy_enum(int ncid, nc_type xtype, nc_type basetype, Position* src, Position* dst); -static int copy_opaque(int ncid, nc_type xtype, size_t size, Position* src,Position* dst); - static int dump_datar(int ncid, nc_type xtype, Position*, NCbytes* buf); static int dump_compound(int ncid, nc_type xtype, size_t size, size_t nfields, Position* offset, NCbytes* buf); static int dump_vlen(int ncid, nc_type xtype, nc_type basetype, Position* offset, NCbytes* buf); static int dump_enum(int ncid, nc_type xtype, nc_type basetype, Position* offset, NCbytes* buf); static int dump_opaque(int ncid, nc_type xtype, size_t size, Position* offset, NCbytes* buf); - -static ptrdiff_t read_align(ptrdiff_t offset, size_t alignment); #endif -static int NC_inq_any_type(int ncid, nc_type typeid, char *name, size_t *size, nc_type *basetypep, size_t *nfieldsp, int *classp); - /** - -Reclaim a vector of instances of a type. This improves upon -e.g. nc_free_vlen. This recursively walks the top-level -instances to reclaim any nested data such as vlen or strings or such. +\ingroup user_types +Reclaim an array of instances of an arbitrary type. +This function should be used when the other simpler functions +such as *nc_free_vlens* or *nc_free_string* cannot be used. +This recursively walks the top-level instances to reclaim +any nested data such as vlen or strings or such. Assumes it is passed a pointer to count instances of xtype. Reclaims any nested data. @@ -88,10 +93,8 @@ int nc_reclaim_data(int ncid, nc_type xtype, void* memory, size_t count) { int stat = NC_NOERR; - size_t i; - Position offset; - int isf; - + NC* nc = NULL; + if(ncid < 0 || xtype <= 0) {stat = NC_EINVAL; goto done;} if(memory == NULL && count > 0) @@ -102,35 +105,41 @@ nc_reclaim_data(int ncid, nc_type xtype, void* memory, size_t count) fprintf(stderr,">>> reclaim: memory=%p count=%lu ncid=%d xtype=%d\n",memory,(unsigned long)count,ncid,xtype); #endif - /* Optimizations */ - /* 1. Vector of fixed size types */ - if((stat = NC4_inq_type_fixed_size(ncid,xtype,&isf))) goto done; - if(isf) goto done; /* no need to reclaim anything */ + if((stat = NC_check_id(ncid,&nc))) goto done; + nc = DAPSUBSTRATE(nc); + /* Call internal version */ + stat = NC_reclaim_data(nc,xtype,memory,count); + +#if 0 #ifdef USE_NETCDF4 - /* 2.Vector of strings */ - if(xtype == NC_STRING) { - char** ss = (char**)memory; - for(i=0;ioffset += xsize; - goto done; - } - - switch (xtype) { - case NC_STRING: { - char** sp = (char**)(offset->memory + offset->offset); - /* Need to reclaim string */ - if(*sp != NULL) free(*sp); - offset->offset += xsize; - } break; - default: - /* reclaim a user type */ - switch (klass) { - case NC_OPAQUE: stat = reclaim_opaque(ncid,xtype,xsize,offset); break; - case NC_ENUM: stat = reclaim_enum(ncid,xtype,basetype,offset); break; - case NC_COMPOUND: stat = reclaim_compound(ncid,xtype,xsize,nfields,offset); break; - case NC_VLEN: - stat = reclaim_vlen(ncid,xtype,basetype,offset); - break; - default: - stat = NC_EINVAL; - break; - } - break; - } -done: - return stat; -} - -static int -reclaim_vlen(int ncid, nc_type xtype, nc_type basetype, Position* offset) -{ - int stat = NC_NOERR; - size_t i; - nc_vlen_t* vl = (nc_vlen_t*)(offset->memory+offset->offset); - - if(vl->len > 0 && vl->p == NULL) - {stat = NC_EINVAL; goto done;} - - /* Free up each entry in the vlen list */ - if(vl->len > 0) { - Position voffset; - size_t alignment = 0; - if((stat = NC_type_alignment(ncid,basetype,&alignment))) goto done;; - voffset.memory = vl->p; - voffset.offset = 0; - for(i=0;ilen;i++) { - voffset.offset = read_align(voffset.offset,alignment); - if((stat = reclaim_datar(ncid,basetype,&voffset))) goto done; - } - free(vl->p); - } - offset->offset += sizeof(nc_vlen_t); - -done: - return stat; -} - -static int -reclaim_enum(int ncid, nc_type xtype, nc_type basetype, Position* offset) -{ - int stat = NC_NOERR; -abort(); - - /* basically same as an instance of the enum's integer basetype */ - stat = reclaim_datar(ncid,basetype,offset); - return stat; -} - -static int -reclaim_opaque(int ncid, nc_type xtype, size_t size, Position* offset) -{ -abort(); - /* basically a fixed size sequence of bytes */ - offset->offset += size; - return NC_NOERR; -} - -static int -reclaim_compound(int ncid, nc_type xtype, size_t size, size_t nfields, Position* offset) -{ - int stat = NC_NOERR; - size_t fid, i, arraycount; - ptrdiff_t saveoffset; - int ndims; - int dimsizes[NC_MAX_VAR_DIMS]; - - saveoffset = offset->offset; - - /* Get info about each field in turn and reclaim it */ - for(fid=0;fidoffset = saveoffset + fieldalignment; - /* compute the total number of elements in the field array */ - arraycount = 1; - for(i=0;ioffset = saveoffset; - offset->offset += size; - -done: - return stat; -} -#endif - /**************************************************/ /** -Copy a vector of instances of a type. This recursively walks -the top-level instances to copy any nested data such as vlen or -strings or such. +\ingroup user_types +Copy an array of instances of an arbitrary type. This recursively walks +the top-level instances to copy any nested data such as vlen or strings or such. Assumes it is passed a pointer to count instances of xtype and a space into which to copy the instance. Copys any nested data. @@ -307,12 +182,8 @@ int nc_copy_data(int ncid, nc_type xtype, const void* memory, size_t count, void* copy) { int stat = NC_NOERR; - size_t i; - Position src; - Position dst; - size_t xsize; - int isf; - + NC* nc = NULL; + if(ncid < 0 || xtype <= 0) {stat = NC_EINVAL; goto done;} if(memory == NULL && count > 0) @@ -326,34 +197,32 @@ nc_copy_data(int ncid, nc_type xtype, const void* memory, size_t count, void* co fprintf(stderr,">>> copy : copy =%p memory=%p count=%lu ncid=%d xtype=%d\n",copy,memory,(unsigned long)count,ncid,xtype); #endif - /* Get type size */ - if((stat = NC_inq_any_type(ncid,xtype,NULL,&xsize,NULL,NULL,NULL))) goto done; - - /* Optimizations */ - /* 1. Vector of fixed sized objects */ - if((stat = NC4_inq_type_fixed_size(ncid,xtype,&isf))) goto done; - if(isf) { - memcpy(copy,memory,xsize*count); - goto done; - } + if((stat = NC_check_id(ncid,&nc))) goto done; + nc = DAPSUBSTRATE(nc); -#ifdef USE_NETCDF4 - src.memory = (char*)memory; /* use char* so we can do pointer arithmetic */ - src.offset = 0; - dst.memory = (char*)copy; /* use char* so we can do pointer arithmetic */ - dst.offset = 0; - for(i=0;i 0) { @@ -379,191 +248,10 @@ nc_copy_data_all(int ncid, nc_type xtype, const void* memory, size_t count, void return stat; } -#ifdef USE_NETCDF4 -/* Recursive type walker: copy a single instance */ -static int -copy_datar(int ncid, nc_type xtype, Position* src, Position* dst) -{ - int stat = NC_NOERR; - size_t xsize; - nc_type basetype; - size_t nfields; - int xclass,isf; - - if((stat = NC_inq_any_type(ncid,xtype,NULL,&xsize,&basetype,&nfields,&xclass))) goto done; - - /* Optimizations */ - /* 1. Vector of fixed size types */ - if((stat = NC4_inq_type_fixed_size(ncid,xtype,&isf))) goto done; - if(isf) { - memcpy(dst->memory+dst->offset,src->memory+src->offset,xsize*1); - src->offset += xsize; - dst->offset += xsize; - goto done; - } - - switch (xtype) { - case NC_STRING: { - char** sp = (char**)(src->memory + src->offset); - char* copy = NULL; - /* Need to copy string */ - if(*sp != NULL) { - if((copy = strdup(*sp))==NULL) {stat = NC_ENOMEM; goto done;} - } - memcpy(dst->memory+dst->offset,(void*)©,sizeof(char*)); - src->offset += xsize; - dst->offset += xsize; - } break; - default: - /* copy a user type */ - switch (xclass) { - case NC_OPAQUE: stat = copy_opaque(ncid,xtype,xsize,src,dst); break; - case NC_ENUM: stat = copy_enum(ncid,xtype,basetype,src,dst); break; - case NC_COMPOUND: stat = copy_compound(ncid,xtype,xsize,nfields,src,dst); break; - case NC_VLEN: stat = copy_vlen(ncid,xtype,basetype,src,dst); break; - default: stat = NC_EINVAL; break; - } - break; - } -done: - return stat; -} - -static int -copy_vlen(int ncid, nc_type xtype, nc_type basetype, Position* src, Position* dst) -{ - int stat = NC_NOERR; - size_t i, basetypesize; - nc_vlen_t* vl = (nc_vlen_t*)(src->memory+src->offset); - nc_vlen_t copy = {0,NULL}; - - if(vl->len > 0 && vl->p == NULL) - {stat = NC_EINVAL; goto done;} - - /* Get basetype info */ - if((stat = NC_inq_any_type(ncid,basetype,NULL,&basetypesize,NULL,NULL,NULL))) goto done; - - /* Make space in the copy vlen */ - if(vl->len > 0) { - copy.len = vl->len; - if((copy.p = calloc(copy.len,basetypesize))==NULL) {stat = NC_ENOMEM; goto done;} - } - /* Copy each entry in the vlen list */ - if(vl->len > 0) { - Position vsrc, vdst; - size_t alignment = 0; - if((stat = NC_type_alignment(ncid,basetype,&alignment))) goto done;; - vsrc.memory = vl->p; - vsrc.offset = 0; - vdst.memory = copy.p; - vdst.offset = 0; - for(i=0;ilen;i++) { - vsrc.offset= read_align(vsrc.offset,alignment); - vdst.offset= read_align(vdst.offset,alignment); - if((stat = copy_datar(ncid,basetype,&vsrc,&vdst))) goto done; - } - } - /* Move into place */ - memcpy(dst->memory+dst->offset,©,sizeof(nc_vlen_t)); - src->offset += sizeof(nc_vlen_t); - dst->offset += sizeof(nc_vlen_t); - -done: - if(stat) { - nullfree(copy.p); - } - return stat; -} - -static int -copy_enum(int ncid, nc_type xtype, nc_type basetype, Position* src, Position* dst) -{ - int stat = NC_NOERR; -abort(); - /* basically same as an instance of the enum's integer basetype */ - stat = copy_datar(ncid,basetype,src,dst); - return stat; -} - -static int -copy_opaque(int ncid, nc_type xtype, size_t size, Position* src, Position* dst) -{ -abort(); - /* basically a fixed size sequence of bytes */ - memcpy(dst->memory+dst->offset,src->memory+src->offset,size); - src->offset += size; - dst->offset += size; - return NC_NOERR; -} - -static int -copy_compound(int ncid, nc_type xtype, size_t size, size_t nfields, Position* src, Position* dst) -{ - int stat = NC_NOERR; - size_t fid, i, arraycount; - ptrdiff_t savesrcoffset, savedstoffset; - int ndims; - int dimsizes[NC_MAX_VAR_DIMS]; - - savesrcoffset = src->offset; - savedstoffset = dst->offset; - - /* Get info about each field in turn and copy it */ - for(fid=0;fidoffset,(int)(savesrcoffset+fieldoffset)); -#endif - src->offset = savesrcoffset+fieldoffset; - dst->offset = savedstoffset+fieldoffset; -#ifdef DEBUG -fprintf(stderr,"field %s(%d) = %d\n",name,(int)fieldoffset,(int)src->offset); -#endif - /* compute the total number of elements in the field array */ - arraycount = 1; - for(i=0;ioffset,src->memory); -#endif - } -#ifdef DEBUG -fprintf(stderr,"\n"); -#endif - /* Return to beginning of the compound and move |compound| */ - src->offset = savesrcoffset; - dst->offset = savedstoffset; - src->offset += size; - dst->offset += size; - -done: - return stat; -} -#endif - /**************************************************/ /* Alignment functions */ #ifdef USE_NETCDF4 -static ptrdiff_t -read_align(ptrdiff_t offset, size_t alignment) -{ - size_t loc_align = (alignment == 0 ? 1 : alignment); - size_t delta = (offset % loc_align); - if(delta == 0) return offset; - return offset + (alignment - delta); -} - /** @param ncid - only needed for a compound type @@ -574,36 +262,15 @@ int NC_type_alignment(int ncid, nc_type xtype, size_t* alignp) { int stat = NC_NOERR; - size_t align = 0; - int klass; + NC* nc = NULL; - if(!type_alignment_initialized) { - NC_compute_alignments(); - type_alignment_initialized = 1; - } - if(xtype <= NC_MAX_ATOMIC_TYPE) - {stat = NC_class_alignment(xtype,&align); goto done;} - else {/* Presumably a user type */ - if((stat = NC_inq_any_type(ncid,xtype,NULL,NULL,NULL,NULL,&klass))) goto done; - switch(klass) { - case NC_VLEN: stat = NC_class_alignment(klass,&align); break; - case NC_OPAQUE: stat = NC_class_alignment(klass,&align); break; - case NC_COMPOUND: {/* get alignment of the first field of the compound */ - nc_type fieldtype; - /* Get all relevant info about the first field */ - if((stat = nc_inq_compound_field(ncid,xtype,0,NULL,NULL,&fieldtype,NULL,NULL))) goto done; - stat = NC_type_alignment(ncid,fieldtype,&align); /* may recurse repeatedly */ - } break; - default: break; - } - } - if(alignp) *alignp = align; + if((stat = NC_check_id(ncid,&nc))) goto done; + nc = DAPSUBSTRATE(nc); + if(USENC3INFO(nc)) goto done; + /* Call internal version */ + stat = NC_type_alignment_internal((NC_FILE_INFO_T*)nc->dispatchdata,xtype,NULL,alignp); done: -#if 0 -Why was this here? - if(stat == NC_NOERR && align == 0) stat = NC_EINVAL; -#endif return stat; } #endif @@ -619,7 +286,7 @@ Why was this here? */ int -nc_dump_data(int ncid, nc_type xtype, void* memory, size_t count, char** bufp) +nc_dump_data(int ncid, nc_type xtype, const void* memory, size_t count, char** bufp) { int stat = NC_NOERR; size_t i; @@ -651,7 +318,7 @@ nc_dump_data(int ncid, nc_type xtype, void* memory, size_t count, char** bufp) } int -nc_print_data(int ncid, nc_type xtype, void* memory, size_t count) +nc_print_data(int ncid, nc_type xtype, const void* memory, size_t count) { char* s = NULL; int stat = NC_NOERR; @@ -673,7 +340,7 @@ dump_datar(int ncid, nc_type xtype, Position* offset, NCbytes* buf) char s[128]; /* Get relevant type info */ - if((stat = NC_inq_any_type(ncid,xtype,NULL,&xsize,&basetype,&nfields,&klass))) goto done; + if((stat = ncaux_inq_any_type(ncid,xtype,NULL,&xsize,&basetype,&nfields,&klass))) goto done; switch (xtype) { case NC_CHAR: @@ -773,7 +440,7 @@ dump_vlen(int ncid, nc_type xtype, nc_type basetype, Position* offset, NCbytes* voffset.offset = 0; for(i=0;ilen;i++) { if(i > 0) ncbytescat(buf," "); - voffset.offset = read_align(voffset.offset,alignment); + voffset.offset = NC_read_align(voffset.offset,alignment); if((stat = dump_datar(ncid,basetype,&voffset,buf))) goto done; } } @@ -835,13 +502,15 @@ dump_compound(int ncid, nc_type xtype, size_t size, size_t nfields, Position* of if((stat = nc_inq_compound_field(ncid,xtype,fid,name,&fieldalignment,&fieldtype,&ndims,dimsizes))) goto done; if(fid > 0) ncbytescat(buf,";"); ncbytescat(buf,name); + ncbytescat(buf,"("); if(ndims > 0) { int j; for(j=0;joffset = saveoffset + fieldalignment; diff --git a/libdispatch/dinstance_intern.c b/libdispatch/dinstance_intern.c new file mode 100644 index 0000000000..c64398b434 --- /dev/null +++ b/libdispatch/dinstance_intern.c @@ -0,0 +1,610 @@ +/* +Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata +See COPYRIGHT for license information. +*/ + +/** \internal +This file contains various instance operations that operate +on a deep level rather than the shallow level of e.g. nc_free_vlen_t. +Currently two operations are defined: +1. reclaim a vector of instances +2. copy a vector of instances +*/ + +#include "config.h" +#include +#include +#include +#include "netcdf.h" +#include "nc4internal.h" +#include "nc4dispatch.h" +#include "ncoffsets.h" +#include "ncbytes.h" + +#undef REPORT +#undef DEBUG + +/* It is helpful to have a structure that identifies a pointer into the overall memory */ +typedef struct Position{char* memory;} Position; + +static int type_alignment_initialized = 0; + +/* Forward */ +#ifdef USE_NETCDF4 +static int reclaim_datar(NC_FILE_INFO_T* file, NC_TYPE_INFO_T* utype, Position instance); +static int copy_datar(NC_FILE_INFO_T* file, NC_TYPE_INFO_T* utype, Position src, Position dst); +#endif + +int NC_print_data(NC_FILE_INFO_T* file, nc_type xtype, const void* memory, size_t count); + +/** +Reclaim a vector of instances of a type. This recursively walks the +top-level instances to reclaim any nested data such as vlen or strings +or such. + +Assumes it is passed a pointer to count instances of xtype. +Reclaims any nested data. + +These are the internal equivalents of nc_reclaim_data[_all] and +nc_copy_data[_all] and as such operate using internal data structures. + +WARNING: DOES NOT RECLAIM THE TOP-LEVEL MEMORY. +The reason is that we do not know how it was allocated (e.g. static vs +dynamic); only the caller can know that. But note that it assumes all +memory blocks other than the top were dynamically allocated, so they +will be free'd. + +Should work for any netcdf type. + +Note that this has been optimized significantly, largely +by unwinding various reclaim cliche procedures. + +@param nc NC* structure +@param xtype type id +@param memory ptr to top-level memory to reclaim +@param count number of instances of the type in memory block +@return error code +*/ + +int +NC_reclaim_data(NC* nc, nc_type xtype, void* memory, size_t count) +{ + int stat = NC_NOERR; + size_t i; + Position instance; + NC_FILE_INFO_T* file = NULL; + NC_TYPE_INFO_T* utype = NULL; + + assert(nc != NULL); + assert((memory == NULL && count == 0) || (memory != NULL || count > 0)); + + /* Process atomic types */ + + /* Optimize: Vector of fixed size atomic types (always the case for netcdf-3)*/ + if(xtype < NC_STRING) goto done; + +#ifdef USE_NETCDF4 + /* Optimize: Vector of strings */ + if(xtype == NC_STRING) { + char** ss = (char**)memory; + for(i=0;idispatchdata; + if((stat = nc4_find_type(file,xtype,&utype))) goto done; + + /* Optimize: vector of fixed sized compound type instances */ + if(!utype->varsized) goto done; /* no need to reclaim anything */ + + /* Remaining cases: vector of VLEN and vector of (transitive) variable sized compound types. + These all require potential recursion. + */ + + /* Build a memory walker object */ + instance.memory = (char*)memory; /* use char* so we can do pointer arithmetic */ + /* Walk each vector instance */ + /* Note that we avoid reclaiming the top level memory */ + for(i=0;isize; /* move to next entry */ + } +#else + stat = NC_EBADTYPE; +#endif + +done: + return stat; +} + +#ifdef USE_NETCDF4 +/* Recursive type walker: reclaim a single instance of a variable-sized user-defined type; + specifically a vlen or a variable-sized compound type instance +*/ +static int +reclaim_datar(NC_FILE_INFO_T* file, NC_TYPE_INFO_T* utype, Position instance) +{ + int i,stat = NC_NOERR; + nc_type basetypeid; + NC_TYPE_INFO_T* basetype = NULL; + size_t nfields; + nc_vlen_t* vlen; + size_t fid, arraycount; + int ndims; + int dimsizes[NC_MAX_VAR_DIMS]; + size_t alignment = 0; + Position vinstance; /* walk the vlen instance memory */ + + assert(utype->varsized); /* All fixed size cases are optimized out */ + + /* Leaving VLEN or Compound */ + + if(utype->nc_type_class == NC_VLEN) { + basetypeid = utype->u.v.base_nc_typeid; /* Get basetype */ + vlen = (nc_vlen_t*)instance.memory;/* memory as vector of nc_vlen_t instances */ + /* Optimize: basetype is atomic fixed size */ + if(basetypeid < NC_STRING) { + goto out; + } + /* Optimize: basetype is string */ + if(basetypeid == NC_STRING) { + if(vlen->len > 0 && vlen->p != NULL) { + char** slist = (char**)vlen->p; /* vlen instance is a vector of string pointers */ + for(i=0;ilen;i++) {if(slist[i] != NULL) free(slist[i]);} + } + goto out; + } + /* Optimize: vlen basetype is a fixed-size user-type */ + if((stat = nc4_find_type(file,basetypeid,&basetype))) goto done; + if(!basetype->varsized) { + goto out; + } + /* Remaining case: basetype is itself variable size => recurse */ + if((stat = NC_type_alignment_internal(file,basetypeid,basetype,&alignment))) goto done;; + vinstance.memory = (char*)vlen->p; /* use char* so we can do pointer arithmetic */ + vinstance.memory = (void*)NC_read_align((uintptr_t)vinstance.memory,alignment); + for(i=0;ilen;i++) { + if((stat=reclaim_datar(file,basetype,vinstance))) goto done; /* reclaim one basetype instance */ + vinstance.memory += basetype->size; /* move to next base instance */ + } +out: + if(vlen->len > 0 && vlen->p != NULL) {free(vlen->p);} + goto done; + } else if(utype->nc_type_class == NC_COMPOUND) { + Position finstance; /* mark the fields's instance */ + nfields = nclistlength(utype->u.c.field); + /* Get info about each field in turn and reclaim it */ + for(fid=0;fidu.c.field,fid); + ndims = field->ndims; + arraycount = 1; + for(i=0;idim_size[i]; arraycount *= dimsizes[i];} + if(field->ndims == 0) {ndims=1; dimsizes[0]=1;} /* fake the scalar case */ + + /* "Move" to start of this field's instance */ + finstance.memory = instance.memory + field->offset; /* includes proper alignment */ + + /* optimize: fixed length atomic type */ + if(field->nc_typeid < NC_STRING) continue; + + /* optimize: string field type */ + if(field->nc_typeid == NC_STRING) { + char** strvec = (char**)finstance.memory; + for(i=0;inc_typeid,&basetype))) goto done; + if(!basetype->varsized) continue; + + /* Field is itself variable length (possibly transitively) */ + for(i=0;isize; + } + } + goto done; + } else {stat = NC_EBADTYPE; goto done;} + +done: + return stat; +} +#endif + +/**************************************************/ + +/** +Copy a vector of instances of a type. This recursively walks +the top-level instances to copy any nested data such as vlen or +strings or such. + +Assumes it is passed a pointer to count instances of xtype and a +space into which to copy the instance. Copys any nested data +by calling malloc(). + +WARNING: DOES NOT ALLOCATE THE TOP-LEVEL MEMORY (see the +nc_copy_data_all function). Note that all memory blocks other +than the top are dynamically allocated. + +Should work for any netcdf type. + +@param file NC_FILE_INFO_T* structure +@param xtype type id +@param memory ptr to top-level memory to reclaim +@param count number of instances of the type in memory block +@param copy top-level space into which to copy the instance +@return error code +*/ + +int +NC_copy_data(NC* nc, nc_type xtype, const void* memory, size_t count, void* copy) +{ + int stat = NC_NOERR; + size_t i; + Position src; + Position dst; + NC_FILE_INFO_T* file = NULL; + NC_TYPE_INFO_T* utype = NULL; + size_t typesize = 0; + + if(memory == NULL || count == 0) + goto done; /* ok, do nothing */ + + assert(nc != NULL); + assert(memory != NULL || count > 0); + assert(copy != NULL || count == 0); + + /* Process atomic types */ + + /* Optimize: Vector of fixed size atomic types */ + if(xtype < NC_STRING) { + typesize = NC_atomictypelen(xtype); + memcpy(copy,memory,count*typesize); + goto done; + } + +#ifdef USE_NETCDF4 + /* Optimize: Vector of strings */ + if(xtype == NC_STRING) { + char** svec = (char**)memory; + char** dvec = (char**)copy; + size_t len; + for(i=0;idispatchdata; + + /* Process User types */ + if((stat = nc4_find_type(file,xtype,&utype))) goto done; + + /* Optimize: vector of fixed sized compound type instances */ + if(!utype->varsized) { + memcpy(copy,memory,count*utype->size); + goto done; + } + + /* Remaining cases: vector of VLEN and vector of variable sized compound types. + These all require potential recursion. + */ + + src.memory = (char*)memory; /* use char* so we can do pointer arithmetic */ + dst.memory = (char*)copy; /* use char* so we can do pointer arithmetic */ + /* Walk each vector instance */ + for(i=0;isize; + dst.memory += utype->size; + } +#else + stat = NC_EBADTYPE; +#endif + +done: + return stat; +} + +#ifdef USE_NETCDF4 +/* Recursive type walker: reclaim an instance of variable-sized user-defined types; + specifically a vlen or a variable-sized compound type instance +*/ +static int +copy_datar(NC_FILE_INFO_T* file, NC_TYPE_INFO_T* utype, Position src, Position dst) +{ + int i, stat = NC_NOERR; + nc_type basetypeid; + NC_TYPE_INFO_T* basetype = NULL; + size_t nfields; + nc_vlen_t* srcvlens = NULL; + nc_vlen_t* dstvlens = NULL; + size_t fid, arraycount; + int ndims; + int dimsizes[NC_MAX_VAR_DIMS]; + Position vsrc, vdst; + size_t alignment = 0; + + assert(utype->varsized); /* All fixed size cases are optimized out */ + + /* Leaving VLEN or Compound */ + + if(utype->nc_type_class == NC_VLEN) { + size_t basetypesize = 0; + size_t copycount = 0; + + basetypeid = utype->u.v.base_nc_typeid; /* Get basetype */ + srcvlens = (nc_vlen_t*)src.memory; + dstvlens = (nc_vlen_t*)dst.memory; + dstvlens->len = srcvlens->len; /* always */ + + if(srcvlens->len == 0) { + dstvlens->p = NULL; + goto done; + } + + if(basetypeid <= NC_MAX_ATOMIC_TYPE) + basetypesize = NC_atomictypelen(basetypeid); + copycount = srcvlens->len*basetypesize; + + /* Optimize: basetype is atomic fixed size */ + if(basetypeid < NC_STRING) { + if((dstvlens->p = (void*)malloc(copycount))==NULL) {stat = NC_ENOMEM; goto done;} + memcpy(dstvlens->p,srcvlens->p,copycount); + goto done; + } + + /* Optimize: basetype is string */ + if(basetypeid == NC_STRING) { + char** srcstrvec = (char**)srcvlens->p; + char** dststrvec = NULL; + if((dststrvec = (void*)malloc(copycount))==NULL) {stat = NC_ENOMEM; goto done;} + dstvlens->p = (void*)dststrvec; + for(i=0;ilen;i++) { + if((dststrvec[i] = strdup(srcstrvec[i]))==NULL) {stat = NC_ENOMEM; goto done;} + } + goto done; + } + + /* User-defined type */ + /* Recompute base type size */ + if((stat = nc4_find_type(file,basetypeid,&basetype))) goto done; + basetypesize = basetype->size; + copycount = srcvlens->len*basetypesize; + + /* Optimize: basetype is user-type fixed size */ + if(!basetype->varsized) { + if((dstvlens->p = (void*)malloc(copycount))==NULL) {stat = NC_ENOMEM; goto done;} + memcpy(dstvlens->p,srcvlens->p,copycount); + goto done; + } + + /* Remaining case: basetype is itself variable size => recurse */ + if((stat = NC_type_alignment_internal(file,basetypeid,basetype,&alignment))) goto done;; + vsrc.memory = (char*)srcvlens->p; /* use char* so we can do pointer arithmetic */ + if((vdst.memory = (char*)malloc(copycount))==NULL) {stat = NC_ENOMEM; goto done;} + dstvlens->p = vdst.memory; /* don't lose it */ + vsrc.memory = (void*)NC_read_align((uintptr_t)vsrc.memory,alignment); + vdst.memory = (void*)NC_read_align((uintptr_t)vdst.memory,alignment); + for(i=0;ilen;i++) { + if((stat=copy_datar(file,basetype,vsrc,vdst))) goto done; + vsrc.memory += basetype->size; + vdst.memory += basetype->size; + } + goto done; + } else if(utype->nc_type_class == NC_COMPOUND) { + Position fsrc; /* mark the src fields's instance */ + Position fdst; /* mark the dst fields's instance */ + nfields = nclistlength(utype->u.c.field); + /* Get info about each field in turn and copy it */ + for(fid=0;fidu.c.field,fid); + ndims = field->ndims; + arraycount = 1; + for(i=0;idim_size[i]; arraycount *= dimsizes[i];} + if(field->ndims == 0) {ndims=1; dimsizes[0]=1;} /* fake the scalar case */ + + /* "move" to this field */ + fsrc.memory = src.memory + field->offset; + fdst.memory = dst.memory + field->offset; + + /* optimize: fixed length atomic type */ + if(field->nc_typeid < NC_STRING) { + size_t typesize = NC_atomictypelen(field->nc_typeid); + memcpy(fdst.memory,fsrc.memory,arraycount*typesize); + continue; /* move to next field */ + } + + /* optimize: string field type */ + if(field->nc_typeid == NC_STRING) { + char** srcstrvec = (char**)src.memory; + char** dststrvec = (char**)dst.memory; + for(i=0;inc_typeid,&basetype))) goto done; + if(!basetype->varsized) { + memcpy(fdst.memory,fsrc.memory,arraycount*basetype->size); + continue; /* move to next field */ + } + + /* Remaining case; field type is variable type */ + for(i=0;isize; + fdst.memory += basetype->size; + } + } + goto done; + + } else {stat = NC_EBADTYPE; goto done;} + +done: + return stat; +} +#endif + +/**************************************************/ +/* Alignment functions */ + +#ifdef USE_NETCDF4 +uintptr_t +NC_read_align(uintptr_t addr, size_t alignment) +{ + size_t loc_align = (alignment == 0 ? 1 : alignment); + size_t delta = (addr % loc_align); + if(delta == 0) return addr; + return (addr + (alignment - delta)); +} + + +/** + * Compute proper data alignment for a type + * @param file - only needed for a compound type + * @param xtype - type for which alignment is requested + * @param utype - if known + * @return 0 if not found + */ +int +NC_type_alignment_internal(NC_FILE_INFO_T* file, nc_type xtype, NC_TYPE_INFO_T* utype, size_t* alignp) +{ + int stat = NC_NOERR; + size_t align = 0; + int klass; + + if(!type_alignment_initialized) { + NC_compute_alignments(); + type_alignment_initialized = 1; + } + if(xtype <= NC_MAX_ATOMIC_TYPE) + {stat = NC_class_alignment(xtype,&align); goto done;} + else {/* Presumably a user type */ + if(utype == NULL) { + if((stat = nc4_find_type(file,xtype,&utype))) goto done; + } + klass = utype->nc_type_class; + switch(klass) { + case NC_VLEN: stat = NC_class_alignment(klass,&align); break; + case NC_OPAQUE: stat = NC_class_alignment(klass,&align); break; + case NC_COMPOUND: {/* get alignment of the first field of the compound */ + NC_FIELD_INFO_T* field = NULL; + NC_TYPE_INFO_T* basetype = NULL; + if(nclistlength(utype->u.c.field) == 0) {stat = NC_EINVAL; goto done;} + field = (NC_FIELD_INFO_T*)nclistget(utype->u.c.field,0); + if(field->nc_typeid <= NC_MAX_ATOMIC_TYPE) { + if((stat = nc4_find_type(file,field->nc_typeid,&basetype))) goto done; + } + stat = NC_type_alignment_internal(file,field->nc_typeid,basetype,&align); /* may recurse repeatedly */ + } break; + default: break; + } + } + if(alignp) *alignp = align; + +done: +#if 0 +Why was this here? + if(stat == NC_NOERR && align == 0) stat = NC_EINVAL; +#endif + return stat; +} +#endif + +/**************************************************/ + +/* Internal versions of the XX_all functions */ + +/* Alternate entry point: includes recovering the top-level memory */ +int +NC_reclaim_data_all(NC* nc, nc_type xtypeid, void* memory, size_t count) +{ + int stat = NC_NOERR; + + assert(nc != NULL); + + stat = NC_reclaim_data(nc,xtypeid,memory,count); + if(stat == NC_NOERR && memory != NULL) + free(memory); + return stat; +} + +/* Alternate entry point: includes recovering the top-level memory */ +int +NC_copy_data_all(NC* nc, nc_type xtype, const void* memory, size_t count, void** copyp) +{ + int stat = NC_NOERR; + size_t xsize = 0; + void* copy = NULL; + NC_FILE_INFO_T* file = NULL; + NC_TYPE_INFO_T* utype = NULL; + + assert(nc != NULL); + + if(xtype <= NC_STRING && count > 0) { + xsize = NC_atomictypelen(xtype); + if((copy = calloc(xsize,count))==NULL) {stat = NC_ENOMEM; goto done;} + if(xtype < NC_STRING) /* fixed-size atomic type */ + memcpy(copy,memory,xsize*count); + else { /* string type */ + size_t i; + char** strvec = (char**)memory; + char** scopyvec = (char**)copy; + for(i=0;idispatchdata; + if((stat = nc4_find_type(file,xtype,&utype))) goto done; + xsize = utype->size; + /* allocate the top-level */ + if(count > 0) { + if((copy = calloc(xsize,count))==NULL) {stat = NC_ENOMEM; goto done;} + } + if((stat = NC_copy_data(nc,xtype,memory,count,copy))) + (void)NC_reclaim_data_all(nc,xtype,copy,count); + } +#endif + if(copyp) {*copyp = copy; copy = NULL;} +done: + return stat; +} + +/* Alternate entry point: includes recovering the top-level memory */ +int +NC_print_data(NC_FILE_INFO_T* file, nc_type xtype, const void* memory, size_t count) +{ + EXTERNL int nc_print_data(int ncid, nc_type xtype, const void* memory, size_t count); + return nc_print_data(file->controller->ext_ncid,xtype,memory,count); +} + diff --git a/libdispatch/dvar.c b/libdispatch/dvar.c index f42f2b6aaf..03f0eb7aba 100644 --- a/libdispatch/dvar.c +++ b/libdispatch/dvar.c @@ -1291,19 +1291,18 @@ NC_check_nulls(int ncid, int varid, const size_t *start, size_t **count, /** @name Free String Resources - Use this functions to free resources associated with ::NC_STRING - data. + Use these functions to free resources associated with ::NC_STRING data. */ /*! @{ */ /** Free string space allocated by the library. - When you read string type the library will allocate the storage - space for the data. This storage space must be freed, so pass the - pointer back to this function, when you're done with the data, and - it will free the string memory. + When you read an array string typed data the library will allocate the storage + space for the data. The allocated strings must be freed, so pass the + pointer to the array plus a count of the number of elements in the array to this function, + when you're done with the data, and it will free the allocated string memory. - WARNING: This does not free the data vector itself, only + WARNING: This does not free the top-level array itself, only the strings to which it points. @param len The number of character arrays in the array. diff --git a/libdispatch/dvlen.c b/libdispatch/dvlen.c index 41fbf58889..f8d7cdd9ee 100644 --- a/libdispatch/dvlen.c +++ b/libdispatch/dvlen.c @@ -11,81 +11,59 @@ Functions to create and learn about VLEN types. */ /*! \{ */ /* All these functions are part of this named group... */ -/** -\ingroup user_types -Free memory in a VLEN object. - -When you read VLEN type the library will actually allocate the storage -space for the data. This storage space must be freed, so pass the -pointer back to this function, when you're done with the data, and it -will free the vlen memory. - -The function nc_free_vlens() is more useful than this function, -because it can free an array of VLEN objects. - -WARNING: this code is incorrect because it will only -work if the basetype of the vlen is -- atomic (excluding string basetype) -- + enum -- + opaque - -The reason is that to operate properly, it needs to recurse when -the basetype is a complex object such as another vlen or compound. - -This function is deprecated in favor of the function "nc_reclaim_data". -See include/netcdf.h. - -\param vl pointer to the vlen object. - -\returns ::NC_NOERR No error. -*/ -int -nc_free_vlen(nc_vlen_t *vl) -{ - free(vl->p); - return NC_NOERR; -} - /** \ingroup user_types Free an array of vlens given the number of elements and an array. -When you read VLEN type the library will actually allocate the storage -space for the data. This storage space must be freed, so pass the -pointer back to this function, when you're done with the data, and it -will free the vlen memory. +When you read an array of VLEN typed instances, the library will allocate +the storage space for the data in each VLEN in the array (but not the array itself). +That VLEN data must be freed eventually, so pass the pointer to the array plus +the number of elements in the array to this function when you're done with +the data, and it will free the all the VLEN instances. +The caller is still responsible for free'ing the array itself, +if it was dynamically allocated. -WARNING: this code is incorrect because it will only -work if the basetype of the vlen is -- atomic -- + enum -- + opaque -- excluding string basetype, +WARNING: this function only works if the basetype of the vlen type +is fixed size. This means it is an atomic type except NC_STRING, +or an NC_ENUM, or and NC_OPAQUE, or an NC_COMPOUND where all +the fields of the compound type are themselves fixed size. -The reason is that to operate properly, it needs to recurse when -the basetype is a complex object such as another vlen or compound. +If you have a more complex VLEN base-type, then it is better to call +the "nc_reclaim_data" function. -This function is deprecated in favor of the function "nc_reclaim_data". -See include/netcdf.h. - -\param len number of elements in the array. +\param nelems number of elements in the array. \param vlens pointer to the vlen object. \returns ::NC_NOERR No error. */ int -nc_free_vlens(size_t len, nc_vlen_t vlens[]) +nc_free_vlens(size_t nelems, nc_vlen_t vlens[]) { int ret; size_t i; - for(i = 0; i < len; i++) + for(i = 0; i < nelems; i++) if ((ret = nc_free_vlen(&vlens[i]))) return ret; return NC_NOERR; } +/** +\ingroup user_types +Free memory in a single VLEN object. +This function is equivalent to calling *nc_free_vlens* with nelems == 1. + +\param vl pointer to the vlen object. +\returns ::NC_NOERR No error. +*/ +int +nc_free_vlen(nc_vlen_t *vl) +{ + free(vl->p); + return NC_NOERR; +} + /** \ingroup user_types Use this function to define a variable length array type. diff --git a/libdispatch/ncs3sdk_aws.cpp b/libdispatch/ncs3sdk_aws.cpp index 18ca603905..5e3c3a3e9a 100644 --- a/libdispatch/ncs3sdk_aws.cpp +++ b/libdispatch/ncs3sdk_aws.cpp @@ -24,11 +24,13 @@ #define NCTRACEMORE(level,fmt,...) nctracemore((level),fmt,##__VA_ARGS__) #define NCUNTRACE(e) ncuntrace(__func__,THROW(e),NULL) #define NCUNTRACEX(e,fmt,...) ncuntrace(__func__,THROW(e),fmt,##__VA_ARGS__) +#define NCUNTRACENOOP(e) NCUNTRACE(e) #else #define NCTRACE(level,fmt,...) #define NCTRACEMORE(level,fmt,...) #define NCUNTRACE(e) (e) #define NCUNTRACEX(e,fmt,...) (e) +#define NCUNTRACENOOP(e) #endif #ifdef __CYGWIN__ @@ -150,7 +152,6 @@ makeerrmsg(const Aws::Client::AWSError err, const char* key=" static Aws::Client::ClientConfiguration s3sdkcreateconfig(NCS3INFO* info) { - int stat = NC_NOERR; NCTRACE(11,"info=%s", dumps3info(info)); Aws::Client::ClientConfiguration config; @@ -164,7 +165,7 @@ s3sdkcreateconfig(NCS3INFO* info) if(info->host) config.endpointOverride = info->host; config.enableEndpointDiscovery = true; config.followRedirects = Aws::Client::FollowRedirectsPolicy::ALWAYS; - stat = NCUNTRACE(stat); + NCUNTRACENOOP(NC_NOERR); return config; } @@ -190,7 +191,7 @@ NC_s3sdkcreateclient(NCS3INFO* info) false); } // delete config; - stat = NCUNTRACE(stat); + NCUNTRACENOOP(NC_NOERR); return (void*)s3client; } diff --git a/libhdf5/hdf5attr.c b/libhdf5/hdf5attr.c index 8b15ef3101..72bae528d2 100644 --- a/libhdf5/hdf5attr.c +++ b/libhdf5/hdf5attr.c @@ -316,7 +316,7 @@ NC4_HDF5_del_att(int ncid, int varid, const char *name) /* Reclaim the content of the attribute */ if(att->data) - if((retval = nc_reclaim_data_all(ncid,att->nc_typeid,att->data,att->len))) return retval; + if((retval = NC_reclaim_data_all(h5->controller,att->nc_typeid,att->data,att->len))) return retval; att->data = NULL; att->len = 0; @@ -618,7 +618,7 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, } else { /* no conversion */ /* Still need a copy of the input data */ copy = NULL; - if((retval = nc_copy_data_all(h5->controller->ext_ncid, mem_type, data, 1, ©))) + if((retval = NC_copy_data_all(h5->controller, mem_type, data, 1, ©))) BAIL(retval); } var->fill_value = copy; @@ -652,7 +652,7 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, NC_NOQUANTIZE, 0))) BAIL(retval); } else if(mem_type == file_type) { /* General case: no conversion */ - if((retval = nc_copy_data(h5->controller->ext_ncid,file_type,data,len,copy))) + if((retval = NC_copy_data(h5->controller,file_type,data,len,copy))) BAIL(retval); } else BAIL(NC_EURL); @@ -670,30 +670,30 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, /* Reclaim saved data */ if(attsave.data != NULL) { assert(attsave.len > 0); - (void)nc_reclaim_data_all(h5->controller->ext_ncid,attsave.type,attsave.data,attsave.len); + (void)NC_reclaim_data_all(h5->controller,attsave.type,attsave.data,attsave.len); attsave.len = 0; attsave.data = NULL; } if(fillsave.data != NULL) { assert(fillsave.len > 0); - (void)nc_reclaim_data_all(h5->controller->ext_ncid,fillsave.type,fillsave.data,fillsave.len); + (void)NC_reclaim_data_all(h5->controller,fillsave.type,fillsave.data,fillsave.len); fillsave.len = 0; fillsave.data = NULL; } exit: if(copy) - (void)nc_reclaim_data_all(h5->controller->ext_ncid,file_type,copy,len); + (void)NC_reclaim_data_all(h5->controller,file_type,copy,len); if(retval) { /* Rollback */ if(attsave.data != NULL) { assert(attsave.len > 0); if(att->data) - (void)nc_reclaim_data_all(h5->controller->ext_ncid,attsave.type,att->data,att->len); + (void)NC_reclaim_data_all(h5->controller,attsave.type,att->data,att->len); att->len = attsave.len; att->data = attsave.data; } if(fillsave.data != NULL) { assert(fillsave.len > 0); if(att->data) - (void)nc_reclaim_data_all(h5->controller->ext_ncid,fillsave.type,var->fill_value,1); + (void)NC_reclaim_data_all(h5->controller,fillsave.type,var->fill_value,1); var->fill_value = fillsave.data; } } diff --git a/libhdf5/hdf5internal.c b/libhdf5/hdf5internal.c index 40f80a9681..0f21ddf65d 100644 --- a/libhdf5/hdf5internal.c +++ b/libhdf5/hdf5internal.c @@ -614,7 +614,7 @@ close_vars(NC_GRP_INFO_T *grp) if (var->type_info) { int stat = NC_NOERR; - if((stat = nc_reclaim_data(grp->nc4_info->controller->ext_ncid,var->type_info->hdr.id,var->fill_value,1))) + if((stat = NC_reclaim_data(grp->nc4_info->controller,var->type_info->hdr.id,var->fill_value,1))) return stat; nullfree(var->fill_value); } diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 4bdddbca74..caeec0c1e5 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -183,6 +183,7 @@ get_type_info2(NC_FILE_INFO_T *h5, hid_t datasetid, NC_TYPE_INFO_T **type_info) /* Set a class for the type */ t = NUM_TYPES - 1; (*type_info)->nc_type_class = NC_STRING; + NC4_set_varsize(*type_info); } else { @@ -2014,6 +2015,7 @@ read_type(NC_GRP_INFO_T *grp, hid_t hdf_typeid, char *type_name) { case H5T_STRING: type->nc_type_class = NC_STRING; + if((retval = NC4_set_varsize(type))) return retval; break; case H5T_COMPOUND: @@ -2026,6 +2028,7 @@ read_type(NC_GRP_INFO_T *grp, hid_t hdf_typeid, char *type_name) #endif type->nc_type_class = NC_COMPOUND; + if((retval = NC4_set_varsize(type))) return retval; if ((nmembers = H5Tget_nmembers(hdf_typeid)) < 0) return NC_EHDFERR; @@ -2045,7 +2048,6 @@ read_type(NC_GRP_INFO_T *grp, hid_t hdf_typeid, char *type_name) * compound type. */ if ((member_hdf_typeid = H5Tget_member_type(native_typeid, m)) < 0) return NC_EHDFERR; - if ((member_native_typeid = H5Tget_native_type(member_hdf_typeid, H5T_DIR_DEFAULT)) < 0) return NC_EHDFERR; @@ -2107,11 +2109,8 @@ read_type(NC_GRP_INFO_T *grp, hid_t hdf_typeid, char *type_name) return retval; } - { /* See if this changes from fixed size to variable size */ - int fixedsize; - if((retval = NC4_inq_type_fixed_size(grp->nc4_info->controller->ext_ncid,member_xtype,&fixedsize))) return retval; - if(!fixedsize) type->u.c.varsized = 1; - } + /* See if this changes from fixed size to variable size */ + if((retval=NC4_recheck_varsize(type,member_xtype))) return retval; hdf5free(member_name); } @@ -2156,11 +2155,13 @@ read_type(NC_GRP_INFO_T *grp, hid_t hdf_typeid, char *type_name) /* Remember the base type for this vlen. */ type->u.v.base_nc_typeid = base_nc_type; } + if((retval = NC4_set_varsize(type))) return retval; } break; case H5T_OPAQUE: type->nc_type_class = NC_OPAQUE; + if((retval = NC4_set_varsize(type))) return retval; break; case H5T_ENUM: @@ -2175,7 +2176,8 @@ read_type(NC_GRP_INFO_T *grp, hid_t hdf_typeid, char *type_name) #endif type->nc_type_class = NC_ENUM; - + if((retval = NC4_set_varsize(type))) return retval; + /* Find the base type of this enum (i.e. what is this a * enum of?) */ if (!(base_hdf_typeid = H5Tget_super(hdf_typeid))) @@ -2236,6 +2238,7 @@ read_type(NC_GRP_INFO_T *grp, hid_t hdf_typeid, char *type_name) LOG((0, "unknown class")); return NC_EBADCLASS; } + return retval; } diff --git a/libhdf5/hdf5type.c b/libhdf5/hdf5type.c index 6fc84efd1d..7d971dfcd5 100644 --- a/libhdf5/hdf5type.c +++ b/libhdf5/hdf5type.c @@ -176,13 +176,22 @@ add_user_type(int ncid, size_t size, const char *name, nc_type base_typeid, /* Remember info about this type. */ type->nc_type_class = type_class; - if (type_class == NC_VLEN) - type->u.v.base_nc_typeid = base_typeid; - else if (type_class == NC_ENUM) { + switch(type_class) { + case NC_ENUM: type->u.e.base_nc_typeid = base_typeid; type->u.e.enum_member = nclistnew(); - } else if (type_class == NC_COMPOUND) + break; + case NC_OPAQUE: + break; + case NC_VLEN: + type->u.v.base_nc_typeid = base_typeid; + break; + case NC_COMPOUND: type->u.c.field = nclistnew(); + break; + default: break; + } + if((retval=NC4_set_varsize(type))) return retval; /* Return the typeid to the user. */ if (typeidp) @@ -263,7 +272,6 @@ NC4_insert_array_compound(int ncid, int typeid1, const char *name, NC_TYPE_INFO_T *type; char norm_name[NC_MAX_NAME + 1]; int retval; - int fixedsize = 0; LOG((2, "nc_insert_array_compound: ncid 0x%x, typeid %d name %s " "offset %d field_typeid %d ndims %d", ncid, typeid1, @@ -296,11 +304,7 @@ NC4_insert_array_compound(int ncid, int typeid1, const char *name, return retval; /* See if this changes from fixed size to variable size */ - if((retval = NC4_inq_type_fixed_size(ncid,field_typeid,&fixedsize))) - return retval; - if(!fixedsize) - type->u.c.varsized = 1; - + if((retval=NC4_recheck_varsize(type,field_typeid))) return retval; return NC_NOERR; } diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index f8ba81db20..0a1273250b 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -684,7 +684,7 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, /* If there's a _FillValue attribute, delete it. */ retval = NC4_HDF5_del_att(ncid, varid, _FillValue); if (retval && retval != NC_ENOTATT) return retval; - if((retval = nc_reclaim_data_all(ncid,var->type_info->hdr.id,var->fill_value,1))) return retval; + if((retval = NC_reclaim_data_all(h5->controller,var->type_info->hdr.id,var->fill_value,1))) return retval; var->fill_value = NULL; } @@ -2186,7 +2186,7 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, { /* Copy one instance of the fill_value */ - if((retval = nc_copy_data(ncid,var->type_info->hdr.id,fillvalue,1,filldata))) + if((retval = NC_copy_data(h5->controller,var->type_info->hdr.id,fillvalue,1,filldata))) BAIL(retval); } filldata = (char *)filldata + file_type_size; diff --git a/libnczarr/zattr.c b/libnczarr/zattr.c index c3f890d744..5b7cfb2b27 100644 --- a/libnczarr/zattr.c +++ b/libnczarr/zattr.c @@ -326,7 +326,7 @@ NCZ_del_att(int ncid, int varid, const char *name) /* Reclaim the content of the attribute */ if(att->data) { - if((retval = nc_reclaim_data_all(ncid,att->nc_typeid,att->data,att->len))) return retval; + if((retval = NC_reclaim_data_all(h5->controller,att->nc_typeid,att->data,att->len))) return retval; } att->data = NULL; att->len = 0; @@ -628,7 +628,7 @@ ncz_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, } else { /* no conversion */ /* Still need a copy of the input data */ copy = NULL; - if((retval = nc_copy_data_all(h5->controller->ext_ncid, mem_type, data, 1, ©))) + if((retval = NC_copy_data_all(h5->controller, mem_type, data, 1, ©))) BAIL(retval); } var->fill_value = copy; @@ -665,7 +665,7 @@ ncz_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, NC_NOQUANTIZE, 0))) BAIL(retval); } else if(mem_type == file_type) { /* General case: no conversion */ - if((retval = nc_copy_data(h5->controller->ext_ncid,file_type,data,len,copy))) + if((retval = NC_copy_data(h5->controller,file_type,data,len,copy))) BAIL(retval); } else BAIL(NC_EURL); @@ -700,30 +700,30 @@ ncz_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, /* Reclaim saved data */ if(attsave.data != NULL) { assert(attsave.len > 0); - (void)nc_reclaim_data_all(h5->controller->ext_ncid,attsave.type,attsave.data,attsave.len); + (void)NC_reclaim_data_all(h5->controller,attsave.type,attsave.data,attsave.len); attsave.len = 0; attsave.data = NULL; } if(fillsave.data != NULL) { assert(fillsave.len > 0); - (void)nc_reclaim_data_all(h5->controller->ext_ncid,fillsave.type,fillsave.data,fillsave.len); + (void)NC_reclaim_data_all(h5->controller,fillsave.type,fillsave.data,fillsave.len); fillsave.len = 0; fillsave.data = NULL; } exit: if(copy) - (void)nc_reclaim_data_all(h5->controller->ext_ncid,file_type,copy,len); + (void)NC_reclaim_data_all(h5->controller,file_type,copy,len); if(retval) { /* Rollback */ if(attsave.data != NULL) { assert(attsave.len > 0); if(att->data) - (void)nc_reclaim_data_all(h5->controller->ext_ncid,attsave.type,att->data,att->len); + (void)NC_reclaim_data_all(h5->controller,attsave.type,att->data,att->len); att->len = attsave.len; att->data = attsave.data; } if(fillsave.data != NULL) { assert(fillsave.len > 0); if(att->data) - (void)nc_reclaim_data_all(h5->controller->ext_ncid,fillsave.type,var->fill_value,1); + (void)NC_reclaim_data_all(h5->controller,fillsave.type,var->fill_value,1); var->fill_value = fillsave.data; } } @@ -1014,20 +1014,14 @@ ncz_makeattr(NC_OBJ* container, NCindex* attlist, const char* name, nc_type type NCZ_ATT_INFO_T* zatt = NULL; void* clone = NULL; size_t typesize, clonesize; - int ncid; - NC* nc = NULL; NC_GRP_INFO_T* grp = (container->sort == NCGRP ? (NC_GRP_INFO_T*)container : ((NC_VAR_INFO_T*)container)->container); - nc = grp->nc4_info->controller; - ncid = nc->ext_ncid | grp->hdr.id; - /* Duplicate the values */ if ((stat = nc4_get_typelen_mem(grp->nc4_info, typeid, &typesize))) goto done; clonesize = len*typesize; if((clone = malloc(clonesize))==NULL) {stat = NC_ENOMEM; goto done;} - if((stat = nc_copy_data(ncid, typeid, values, len, clone))) goto done; - + if((stat = NC_copy_data(grp->nc4_info->controller, typeid, values, len, clone))) goto done; if((stat=nc4_att_list_add(attlist,name,&att))) goto done; if((zatt = calloc(1,sizeof(NCZ_ATT_INFO_T))) == NULL) diff --git a/libnczarr/zinternal.c b/libnczarr/zinternal.c index 802cc07cfd..9e4bfcb625 100644 --- a/libnczarr/zinternal.c +++ b/libnczarr/zinternal.c @@ -330,7 +330,7 @@ close_vars(NC_GRP_INFO_T *grp) if (var->type_info) { int stat = NC_NOERR; - if((stat = nc_reclaim_data(grp->nc4_info->controller->ext_ncid,var->type_info->hdr.id,var->fill_value,1))) + if((stat = NC_reclaim_data(grp->nc4_info,var->type_info->hdr.id,var->fill_value,1))) return stat; nullfree(var->fill_value); } diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c index 8617769c36..075d91dbf5 100644 --- a/libnczarr/zsync.c +++ b/libnczarr/zsync.c @@ -1341,7 +1341,7 @@ ncz_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container) if((stat = ncz_makeattr(container,attlist,aname,typeid,len,data,&att))) goto done; /* No longer need this copy of the data */ - if((stat = nc_reclaim_data_all(file->controller->ext_ncid,att->nc_typeid,data,len))) goto done; + if((stat = NC_reclaim_data_all(file->controller,att->nc_typeid,data,len))) goto done; data = NULL; if(isfillvalue) fillvalueatt = att; @@ -1365,7 +1365,7 @@ ncz_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container) done: if(data != NULL) - stat = nc_reclaim_data(file->controller->ext_ncid,att->nc_typeid,data,len); + stat = NC_reclaim_data(file->controller,att->nc_typeid,data,len); NCJreclaim(jattrs); nclistfreeall(atypes); nullfree(fullpath); diff --git a/libnczarr/ztype.c b/libnczarr/ztype.c index 0f3d6e3014..7c8ecec8da 100644 --- a/libnczarr/ztype.c +++ b/libnczarr/ztype.c @@ -228,6 +228,8 @@ add_user_type(int ncid, size_t size, const char *name, nc_type base_typeid, } else if (type_class == NC_COMPOUND) type->u.c.field = nclistnew(); + (void)NC4_set_varsize(type); + /* Return the typeid to the user. */ if (typeidp) *typeidp = type->hdr.id; @@ -338,6 +340,8 @@ NCZ_insert_array_compound(int ncid, int typeid1, const char *name, ndims, dim_sizesp))) return retval; + NC4_recheck_varsize(type,field_typeid); + return NC_NOERR; } diff --git a/libnczarr/zutil.c b/libnczarr/zutil.c index d8d10006c3..403e079e19 100644 --- a/libnczarr/zutil.c +++ b/libnczarr/zutil.c @@ -948,9 +948,8 @@ NCZ_reclaim_fill_value(NC_VAR_INFO_T* var) { int stat = NC_NOERR; if(var->fill_value) { - int ncid = var->container->nc4_info->controller->ext_ncid; int tid = var->type_info->hdr.id; - stat = nc_reclaim_data_all(ncid,tid,var->fill_value,1); + stat = NC_reclaim_data_all(var->container->nc4_info->controller,tid,var->fill_value,1); var->fill_value = NULL; } /* Reclaim any existing fill_chunk */ @@ -962,16 +961,15 @@ int NCZ_copy_fill_value(NC_VAR_INFO_T* var, void** dstp) { int stat = NC_NOERR; - int ncid = var->container->nc4_info->controller->ext_ncid; int tid = var->type_info->hdr.id; void* dst = NULL; if(var->fill_value) { - if((stat = nc_copy_data_all(ncid,tid,var->fill_value,1,&dst))) goto done; + if((stat = NC_copy_data_all(var->container->nc4_info->controller,tid,var->fill_value,1,&dst))) goto done; } if(dstp) {*dstp = dst; dst = NULL;} done: - if(dst) (void)nc_reclaim_data_all(ncid,tid,dst,1); + if(dst) (void)NC_reclaim_data_all(var->container->nc4_info->controller,tid,dst,1); return stat; } @@ -1057,7 +1055,7 @@ NCZ_copy_data(NC_FILE_INFO_T* file, NC_TYPE_INFO_T* xtype, const void* memory, s scopy[i] = NULL; } } - return nc_copy_data(file->controller->ext_ncid,xtype->hdr.id,memory,count,copy); + return NC_copy_data(file->controller,xtype->hdr.id,memory,count,copy); } #if 0 diff --git a/libnczarr/zvar.c b/libnczarr/zvar.c index 2a98646e36..0dd6098ebc 100644 --- a/libnczarr/zvar.c +++ b/libnczarr/zvar.c @@ -2127,7 +2127,7 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, for (i = 0; i < fill_len; i++) { /* Copy one instance of the fill_value */ - if((retval = nc_copy_data(ncid,var->type_info->hdr.id,var->fill_value,1,filldata))) + if((retval = NC_copy_data(h5->controller,var->type_info->hdr.id,var->fill_value,1,filldata))) BAIL(retval); filldata = (char *)filldata + file_type_size; } diff --git a/libnczarr/zwalk.c b/libnczarr/zwalk.c index 5c6672e845..bc423706d0 100644 --- a/libnczarr/zwalk.c +++ b/libnczarr/zwalk.c @@ -755,19 +755,20 @@ EXTERNL int NCZ_read_chunk(int ncid, int varid, size64_t* zindices, void* chunkdata) { int stat = NC_NOERR; + NC_FILE_INFO_T* h5 = NULL; NC_VAR_INFO_T* var = NULL; NCZ_VAR_INFO_T* zvar = NULL; struct NCZChunkCache* cache = NULL; void* cachedata = NULL; - if ((stat = nc4_find_grp_h5_var(ncid, varid, NULL, NULL, &var))) + if ((stat = nc4_find_grp_h5_var(ncid, varid, &h5, NULL, &var))) return THROW(stat); zvar = (NCZ_VAR_INFO_T*)var->format_var_info; cache = zvar->cache; if((stat = NCZ_read_cache_chunk(cache,zindices,&cachedata))) goto done; if(chunkdata) { - if((stat = nc_copy_data(ncid,var->type_info->hdr.id,cachedata,cache->chunkcount,chunkdata))) goto done; + if((stat = NC_copy_data(h5->controller,var->type_info->hdr.id,cachedata,cache->chunkcount,chunkdata))) goto done; } done: diff --git a/libnczarr/zxcache.c b/libnczarr/zxcache.c index 374890be68..eea9ab6a13 100644 --- a/libnczarr/zxcache.c +++ b/libnczarr/zxcache.c @@ -207,8 +207,7 @@ free_cache_entry(NCZChunkCache* cache, NCZCacheEntry* entry) if(entry) { int tid = cache->var->type_info->hdr.id; if(tid == NC_STRING && !entry->isfixedstring) { - int ncid = cache->var->container->nc4_info->controller->ext_ncid; - nc_reclaim_data(ncid,tid,entry->data,cache->chunkcount); + NC_reclaim_data(cache->var->container->nc4_info->controller,tid,entry->data,cache->chunkcount); } nullfree(entry->data); nullfree(entry->key.varkey); @@ -508,10 +507,9 @@ NCZ_reclaim_fill_chunk(NCZChunkCache* zcache) int stat = NC_NOERR; if(zcache && zcache->fillchunk) { NC_VAR_INFO_T* var = zcache->var; - int ncid = var->container->nc4_info->controller->ext_ncid; int tid = var->type_info->hdr.id; size_t chunkcount = zcache->chunkcount; - stat = nc_reclaim_data_all(ncid,tid,zcache->fillchunk,chunkcount); + stat = NC_reclaim_data_all(var->container->nc4_info->controller,tid,zcache->fillchunk,chunkcount); zcache->fillchunk = NULL; } return stat; @@ -610,7 +608,6 @@ put_chunk(NCZChunkCache* cache, NCZCacheEntry* entry) char* path = NULL; nc_type tid = NC_NAT; void* strchunk = NULL; - int ncid = 0; ZTRACE(5,"cache.var=%s entry.key=%s",cache->var->hdr.name,entry->key); LOG((3, "%s: var: %p", __func__, cache->var)); @@ -620,7 +617,6 @@ put_chunk(NCZChunkCache* cache, NCZCacheEntry* entry) map = zfile->map; /* Collect some info */ - ncid = file->controller->ext_ncid; tid = cache->var->type_info->hdr.id; if(tid == NC_STRING && !entry->isfixedstring) { @@ -631,7 +627,7 @@ put_chunk(NCZChunkCache* cache, NCZCacheEntry* entry) /* copy char* to char[] format */ if((stat = NCZ_char2fixed((const char**)entry->data,strchunk,cache->chunkcount,maxstrlen))) goto done; /* Reclaim the old chunk */ - if((stat = nc_reclaim_data_all(ncid,tid,entry->data,cache->chunkcount))) goto done; + if((stat = NC_reclaim_data_all(file->controller,tid,entry->data,cache->chunkcount))) goto done; entry->data = NULL; entry->data = strchunk; strchunk = NULL; entry->size = cache->chunkcount * maxstrlen; diff --git a/libsrc4/nc4attr.c b/libsrc4/nc4attr.c index 5b52c3c4a9..fed4976142 100644 --- a/libsrc4/nc4attr.c +++ b/libsrc4/nc4attr.c @@ -140,7 +140,7 @@ nc4_get_att_ptrs(NC_FILE_INFO_T *h5, NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, if (data) { { - if((retval = nc_copy_data(h5->controller->ext_ncid,mem_type,bufr,att->len,data))) + if((retval = NC_copy_data(h5->controller,mem_type,bufr,att->len,data))) BAIL(retval); } } diff --git a/libsrc4/nc4internal.c b/libsrc4/nc4internal.c index d64f0c12a0..99ed65d031 100644 --- a/libsrc4/nc4internal.c +++ b/libsrc4/nc4internal.c @@ -1109,6 +1109,9 @@ nc4_type_list_add(NC_GRP_INFO_T *grp, size_t size, const char *name, ncindexadd(grp->type, (NC_OBJ *)new_type); obj_track(grp->nc4_info,(NC_OBJ*)new_type); + /* back link */ + new_type->container = grp; + /* Return a pointer to the new type. */ *type = new_type; @@ -1337,7 +1340,7 @@ nc4_att_free(NC_ATT_INFO_T *att) assert(parent->sort == NCGRP); h5 = ((NC_GRP_INFO_T*)parent)->nc4_info; /* Reclaim the attribute data */ - if((stat = nc_reclaim_data(h5->controller->ext_ncid,att->nc_typeid,att->data,att->len))) goto done; + if((stat = NC_reclaim_data(h5->controller,att->nc_typeid,att->data,att->len))) goto done; free(att->data); /* reclaim top level */ att->data = NULL; } @@ -1386,9 +1389,8 @@ var_free(NC_VAR_INFO_T *var) /* Delete any fill value allocation. */ if (var->fill_value) { - int ncid = var->container->nc4_info->controller->ext_ncid; int tid = var->type_info->hdr.id; - if((retval = nc_reclaim_data_all(ncid, tid, var->fill_value, 1))) return retval; + if((retval = NC_reclaim_data_all(var->container->nc4_info->controller, tid, var->fill_value, 1))) return retval; var->fill_value = NULL; } @@ -1563,7 +1565,7 @@ nc4_rec_grp_del_att_data(NC_GRP_INFO_T *grp) /* Free attribute data in this group */ for (i = 0; i < ncindexsize(grp->att); i++) { NC_ATT_INFO_T * att = (NC_ATT_INFO_T*)ncindexith(grp->att, i); - if((retval = nc_reclaim_data_all(grp->nc4_info->controller->ext_ncid,att->nc_typeid,att->data,att->len))) + if((retval = NC_reclaim_data_all(grp->nc4_info->controller,att->nc_typeid,att->data,att->len))) return retval; att->data = NULL; att->len = 0; @@ -1576,7 +1578,7 @@ nc4_rec_grp_del_att_data(NC_GRP_INFO_T *grp) NC_VAR_INFO_T* v = (NC_VAR_INFO_T *)ncindexith(grp->vars, i); for(j=0;jatt);j++) { NC_ATT_INFO_T* att = (NC_ATT_INFO_T*)ncindexith(v->att, j); - if((retval = nc_reclaim_data_all(grp->nc4_info->controller->ext_ncid,att->nc_typeid,att->data,att->len))) + if((retval = NC_reclaim_data_all(grp->nc4_info->controller,att->nc_typeid,att->data,att->len))) return retval; att->data = NULL; att->len = 0; diff --git a/libsrc4/nc4type.c b/libsrc4/nc4type.c index 62b254741c..f0b827b0e2 100644 --- a/libsrc4/nc4type.c +++ b/libsrc4/nc4type.c @@ -14,8 +14,10 @@ #include "nc4internal.h" #include "nc4dispatch.h" +#if 0 #ifdef ENABLE_DAP4 -EXTERNL int NCD4_get_substrate(int ncid); +EXTERNL NC* NCD4_get_substrate_nc(int ncid); +#endif #endif /* The sizes of types may vary from platform to platform, but within @@ -729,38 +731,71 @@ NC4_inq_type_fixed_size(int ncid, nc_type xtype, int* fixedsizep) { int stat = NC_NOERR; int f = 0; - int xclass; + NC_FILE_INFO_T* h5 = NULL; + NC_TYPE_INFO_T* typ = NULL; if(xtype < NC_STRING) {f = 1; goto done;} if(xtype == NC_STRING) {f = 0; goto done;} #ifdef USE_NETCDF4 /* Must be user type */ - if((stat = nc_inq_user_type(ncid,xtype,NULL,NULL,NULL,NULL,&xclass))) goto done; - switch (xclass) { - case NC_ENUM: case NC_OPAQUE: f = 1; break; - case NC_VLEN: f = 0; break; - case NC_COMPOUND: { - NC_FILE_INFO_T* h5 = NULL; - NC_TYPE_INFO_T* typ = NULL; -#ifdef ENABLE_DAP4 - NC* nc = NULL; - int xformat; - if ((stat = NC_check_id(ncid, &nc))) goto done; - xformat = nc->dispatch->model; - if(xformat == NC_FORMATX_DAP4) { - ncid = NCD4_get_substrate(ncid); - } /* Fall thru */ -#endif - if ((stat = nc4_find_grp_h5(ncid, NULL, &h5))) - goto done; - if((stat = nc4_find_type(h5,xtype,&typ))) goto done; - f = !typ->u.c.varsized; - } break; - default: stat = NC_EBADTYPE; goto done; - } + if ((stat = nc4_find_grp_h5(ncid, NULL, &h5))) + goto done; + if((stat = nc4_find_type(h5,xtype,&typ))) goto done; + f = !typ->varsized; #endif done: if(fixedsizep) *fixedsizep = f; return stat; } + + +/** +For types with one or more subtypes (e.g. basetype or +fieldtype), determine the varsizedness of the type based on the +basetype. The idea is to inform the code of the fact that +parenttype has addedtype "inserted" into it. +@param parenttype +@param subtype +*/ + +int +NC4_recheck_varsize(NC_TYPE_INFO_T* parenttype, nc_type subtype) +{ + int stat = NC_NOERR; + NC_FILE_INFO_T* file = NULL; + NC_TYPE_INFO_T* utype = NULL; + if(subtype < NC_STRING) goto done; /* will not change the "variable-sizedness" of parenttype */ + if(subtype == NC_STRING) {parenttype->varsized = 1; goto done;} + /* Get the inferred user-type */ + file = parenttype->container->nc4_info; + if((stat = nc4_find_type(file,subtype,&utype))) goto done; + switch (utype->nc_type_class) { + case NC_OPAQUE: case NC_ENUM: break; /* no change */ + case NC_VLEN: parenttype->varsized = 1; break; + case NC_COMPOUND: if(utype->varsized) parenttype->varsized = 1; break; + } + +done: + return stat; +} + +/** +When creating a type, mark it as variable-sized if known for sure. +@param typ +*/ + +int +NC4_set_varsize(NC_TYPE_INFO_T* typ) +{ + int stat = NC_NOERR; + if(typ->hdr.id < NC_STRING) goto done; /* will not change the "variable-sizedness" of typ */ + if(typ->hdr.id == NC_STRING) {typ->varsized = 1; goto done;} + switch (typ->nc_type_class) { + case NC_OPAQUE: case NC_ENUM: break; /* no change */ + case NC_VLEN: typ->varsized = 1; break; + case NC_COMPOUND: typ->varsized = 0; break; /* until proven otherwise */ + } +done: + return stat; +} diff --git a/libsrc4/nc4var.c b/libsrc4/nc4var.c index f93a3f4660..61f43ec132 100644 --- a/libsrc4/nc4var.c +++ b/libsrc4/nc4var.c @@ -265,7 +265,7 @@ NC4_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep, if (var->fill_value) { int xtype = var->type_info->hdr.id; - if((retval = nc_copy_data(ncid,xtype,var->fill_value,1,fill_valuep))) return retval; + if((retval = NC_copy_data(h5->controller,xtype,var->fill_value,1,fill_valuep))) return retval; } else { diff --git a/nc_test4/tst_vlenstr.c b/nc_test4/tst_vlenstr.c index ca5c715856..8e76211af0 100644 --- a/nc_test4/tst_vlenstr.c +++ b/nc_test4/tst_vlenstr.c @@ -87,6 +87,17 @@ writeVariable(int dimlength, int ncid, nc_type vlen_typeID) free(data); } +static void +createFruitsData(int elemno, int vlensize, int* stringIndexp, nc_vlen_t* data) +{ + data[elemno].len = vlensize; + data[elemno].p = NULL; + if(vlensize > 0) { + data[elemno].p = charPointers+(*stringIndexp); + (*stringIndexp) += vlensize; + } +} + void writeAttribute(int len, int ncid, nc_type vlen_typeID) { @@ -98,29 +109,12 @@ writeAttribute(int len, int ncid, nc_type vlen_typeID) /* create six variable length arrays of strings */ int stringIndex = 0; - data[0].len = first_size; - data[0].p = charPointers+stringIndex; - stringIndex += first_size; - - data[1].len = second_size; - data[1].p = charPointers+stringIndex; - stringIndex += second_size; - - data[2].len = third_size; - data[2].p = charPointers+stringIndex; - stringIndex += third_size; - - data[3].len = fourth_size; - data[3].p = charPointers+stringIndex; - stringIndex += fourth_size; - - data[4].len = fifth_size; - data[4].p = charPointers+stringIndex; - stringIndex += fifth_size; - - data[5].len = sixth_size; - data[5].p = charPointers+stringIndex; - stringIndex += sixth_size; + createFruitsData(0,first_size,&stringIndex,data); + createFruitsData(1,second_size,&stringIndex,data); + createFruitsData(2,third_size,&stringIndex,data); + createFruitsData(3,fourth_size,&stringIndex,data); + createFruitsData(4,fifth_size,&stringIndex,data); + createFruitsData(5,sixth_size,&stringIndex,data); #ifdef DEBUG if(buf) {free(buf); buf = NULL;} diff --git a/ncgen/genbin.c b/ncgen/genbin.c index 9e2070b626..1691a06542 100644 --- a/ncgen/genbin.c +++ b/ncgen/genbin.c @@ -14,11 +14,12 @@ #undef TRACE /* Forward*/ -static int genbin_defineattr(Symbol* asym); -static int genbin_definevardata(Symbol* vsym); -static int genbin_write(Generator*,Symbol*,Bytebuffer*,int,size_t*,size_t*); +static int genbin_defineattr(int ncid, Symbol* asym); +static int genbin_definevardata(int ncid, Symbol* vsym); +static int genbin_write(Generator*,Symbol*,Bytebuffer*,int,size_t*,size_t*); static int genbin_writevar(Generator*,Symbol*,Bytebuffer*,int,size_t*,size_t*); static int genbin_writeattr(Generator*,Symbol*,Bytebuffer*,int,size_t*,size_t*); + #ifdef USE_NETCDF4 static int genbin_deftype(Symbol* tsym); static int genbin_definespecialattributes(Symbol* var); @@ -67,7 +68,7 @@ genbin_netcdf(void) rootgroup->nc_id = ncid; if (nofill_flag) { - stat = nc_set_fill(rootgroup->nc_id, NC_NOFILL, 0); + stat = nc_set_fill(ncid, NC_NOFILL, 0); CHECK_ERR(stat); } @@ -146,7 +147,7 @@ genbin_netcdf(void) if(ngatts > 0) { for(iatt = 0; iatt < ngatts; iatt++) { Symbol* gasym = (Symbol*)listget(gattdefs,iatt); - genbin_defineattr(gasym); + genbin_defineattr(ncid,gasym); } } @@ -154,17 +155,17 @@ genbin_netcdf(void) if(natts > 0) { for(iatt = 0; iatt < natts; iatt++) { Symbol* asym = (Symbol*)listget(attdefs,iatt); - genbin_defineattr(asym); + genbin_defineattr(ncid,asym); } } if (nofill_flag) { - stat = nc_set_fill(rootgroup->nc_id, NC_NOFILL, 0); + stat = nc_set_fill(ncid, NC_NOFILL, 0); CHECK_ERR(stat); } /* leave define mode */ - stat = nc_enddef(rootgroup->nc_id); + stat = nc_enddef(ncid); CHECK_ERR(stat); if(!header_only) { @@ -173,7 +174,7 @@ genbin_netcdf(void) for(ivar = 0; ivar < nvars; ivar++) { Symbol* vsym = (Symbol*)listget(vardefs,ivar); if(vsym->data != NULL) { - genbin_definevardata(vsym); + genbin_definevardata(ncid,vsym); } } } @@ -316,7 +317,7 @@ genbin_deftype(Symbol* tsym) for(i=0;isubnodes);i++) { Symbol* econst = (Symbol*)listget(tsym->subnodes,i); ASSERT(econst->subclass == NC_ECONST); - generator_reset(bin_generator,NULL); + generator_reset(bin_generator,rootgroup); bbClear(datum); generate_basetype(econst->typ.basetype,econst->typ.econst,datum,NULL,bin_generator); stat = nc_insert_enum(tsym->container->nc_id, @@ -378,13 +379,13 @@ genbin_deftype(Symbol* tsym) #endif /*USE_NETCDF4*/ static int -genbin_defineattr(Symbol* asym) +genbin_defineattr(int ncid, Symbol* asym) { int stat = NC_NOERR; Bytebuffer* databuf = bbNew(); - generator_reset(bin_generator,NULL); + generator_reset(bin_generator,rootgroup); generate_attrdata(asym,bin_generator,(Writer)genbin_write,databuf); - stat = nc_reclaim_data(asym->container->nc_id,asym->typ.basetype->nc_id,bbContents(databuf),datalistlen(asym->data)); + stat = nc_reclaim_data(ncid,asym->typ.basetype->nc_id,bbContents(databuf),datalistlen(asym->data)); bbFree(databuf); return stat; } @@ -392,15 +393,15 @@ genbin_defineattr(Symbol* asym) /* Following is patterned after the walk functions in semantics.c */ static int -genbin_definevardata(Symbol* vsym) +genbin_definevardata(int ncid, Symbol* vsym) { int stat = NC_NOERR; Bytebuffer* databuf = NULL; if(vsym->data == NULL) goto done; databuf = bbNew(); - generator_reset(bin_generator,NULL); + generator_reset(bin_generator,rootgroup); generate_vardata(vsym,bin_generator,(Writer)genbin_write,databuf); - stat = nc_reclaim_data_all(vsym->container->nc_id,vsym->typ.basetype->nc_id,bbExtract(databuf),datalistlen(vsym->data)); + stat = nc_reclaim_data(ncid,vsym->typ.basetype->nc_id,bbContents(databuf),datalistlen(vsym->data)); done: bbFree(databuf); return stat; @@ -416,6 +417,7 @@ genbin_write(Generator* generator, Symbol* sym, Bytebuffer* memory, return genbin_writevar(generator,sym,memory,rank,start,count); else PANIC("illegal symbol for genbin_write"); + return NC_EINVAL; } @@ -453,12 +455,6 @@ genbin_writevar(Generator* generator, Symbol* vsym, Bytebuffer* memory, stat = nc_put_vara(vsym->container->nc_id, vsym->nc_id, start, count, data); } CHECK_ERR(stat); -#if 0 - /* Reclaim the data */ - stat = nc_reclaim_data(vsym->container->nc_id, vsym->typ.basetype->nc_id, data, nelems); - CHECK_ERR(stat); - bbClear(memory); /* reclaim top-level memory */ -#endif return stat; } @@ -489,9 +485,9 @@ genbin_writeattr(Generator* generator, Symbol* asym, Bytebuffer* databuf, } break; case NC_CHAR: { char* data = (char*)bbContents(databuf); - size_t slen = bbLength(databuf); - /* Revise length if slen == 0 */ - if(slen == 0) { + size_t slen = bbLength(databuf); + /* Revise length if slen == 0 */ + if(slen == 0) { bbAppend(databuf,'\0'); /* bbAppend frees the memory pointed to by char* data, so re-assign. See Coverity issue: 1265731.*/ @@ -506,55 +502,55 @@ genbin_writeattr(Generator* generator, Symbol* asym, Bytebuffer* databuf, stat = nc_put_att_short(grpid,varid,asym->name,typid,len,data); CHECK_ERR(stat); } break; - case NC_INT: { - int* data = (int*)bbContents(databuf); - stat = nc_put_att_int(grpid,varid,asym->name,typid,len,data); - CHECK_ERR(stat); - } break; - case NC_FLOAT: { - float* data = (float*)bbContents(databuf); - stat = nc_put_att_float(grpid,varid,asym->name,typid,len,data); - CHECK_ERR(stat); + case NC_INT: { + int* data = (int*)bbContents(databuf); + stat = nc_put_att_int(grpid,varid,asym->name,typid,len,data); + CHECK_ERR(stat); } break; - case NC_DOUBLE: { - double* data = (double*)bbContents(databuf); - stat = nc_put_att_double(grpid,varid,asym->name,typid,len,data); - CHECK_ERR(stat); + case NC_FLOAT: { + float* data = (float*)bbContents(databuf); + stat = nc_put_att_float(grpid,varid,asym->name,typid,len,data); + CHECK_ERR(stat); } break; - case NC_STRING: { - const char** data; - data = (const char**)bbContents(databuf); - stat = nc_put_att_string(grpid,varid,asym->name, + case NC_DOUBLE: { + double* data = (double*)bbContents(databuf); + stat = nc_put_att_double(grpid,varid,asym->name,typid,len,data); + CHECK_ERR(stat); + } break; + case NC_STRING: { + const char** data; + data = (const char**)bbContents(databuf); + stat = nc_put_att_string(grpid,varid,asym->name, bbLength(databuf)/sizeof(char*), data); - } break; - case NC_UBYTE: { - unsigned char* data = (unsigned char*)bbContents(databuf); - stat = nc_put_att_uchar(grpid,varid,asym->name,typid,len,data); - CHECK_ERR(stat); + } break; + case NC_UBYTE: { + unsigned char* data = (unsigned char*)bbContents(databuf); + stat = nc_put_att_uchar(grpid,varid,asym->name,typid,len,data); + CHECK_ERR(stat); } break; - case NC_USHORT: { - unsigned short* data = (unsigned short*)bbContents(databuf); - stat = nc_put_att_ushort(grpid,varid,asym->name,typid,len,data); - CHECK_ERR(stat); + case NC_USHORT: { + unsigned short* data = (unsigned short*)bbContents(databuf); + stat = nc_put_att_ushort(grpid,varid,asym->name,typid,len,data); + CHECK_ERR(stat); } break; - case NC_UINT: { - unsigned int* data = (unsigned int*)bbContents(databuf); - stat = nc_put_att_uint(grpid,varid,asym->name,typid,len,data); - CHECK_ERR(stat); + case NC_UINT: { + unsigned int* data = (unsigned int*)bbContents(databuf); + stat = nc_put_att_uint(grpid,varid,asym->name,typid,len,data); + CHECK_ERR(stat); } break; - case NC_INT64: { - long long* data = (long long*)bbContents(databuf); - stat = nc_put_att_longlong(grpid,varid,asym->name,typid,len,data); - CHECK_ERR2(stat,asym->lineno); + case NC_INT64: { + long long* data = (long long*)bbContents(databuf); + stat = nc_put_att_longlong(grpid,varid,asym->name,typid,len,data); + CHECK_ERR2(stat,asym->lineno); } break; - case NC_UINT64: { - unsigned long long* data = (unsigned long long*)bbContents(databuf); - stat = nc_put_att_ulonglong(grpid,varid,asym->name,typid,len,data); - CHECK_ERR(stat); + case NC_UINT64: { + unsigned long long* data = (unsigned long long*)bbContents(databuf); + stat = nc_put_att_ulonglong(grpid,varid,asym->name,typid,len,data); + CHECK_ERR(stat); } break; - default: PANIC1("genbin_defineattr: unexpected basetype: %d",basetype->typ.typecode); - } + default: PANIC1("genbin_defineattr: unexpected basetype: %d",basetype->typ.typecode); + } } else { /* use the generic put_attribute for user defined types*/ const char* data; data = (const char*)bbContents(databuf); @@ -570,6 +566,7 @@ genbin_writeattr(Generator* generator, Symbol* asym, Bytebuffer* databuf, } #endif } + CHECK_ERR(stat); return stat; } diff --git a/ncgen/generate.c b/ncgen/generate.c index 66a3077332..c08f488be4 100644 --- a/ncgen/generate.c +++ b/ncgen/generate.c @@ -180,7 +180,7 @@ generate_basetype(Symbol* tsym, NCConstant* con, Bytebuffer* codebuf, Datalist* } else { generator->listbegin(generator,tsym,NULL,LISTVLEN,data->length,codebuf,&uid); for(count=0;countlength;count++) { - NCConstant* con; + NCConstant* con; generator->list(generator,tsym,NULL,LISTVLEN,uid,count,vlenbuf); con = datalistith(data,count); generate_basetype(tsym->typ.basetype,con,vlenbuf,NULL,generator); @@ -399,6 +399,7 @@ generate_arrayR(struct Args* args, int dimindex, size_t* index, Datalist* data) args->generator->listend(args->generator,args->vsym,NULL,LISTDATA,uid,counter,args->code); memcpy(count,onesvector,sizeof(size_t)*dimindex); count[dimindex] = stop; + /* Write the data; also reclaims written data */ args->writer(args->generator,args->vsym,args->code,args->rank,index,count); bbClear(args->code); } else { diff --git a/ncgen/generate.h b/ncgen/generate.h index c09b5397d0..0aa60a91ed 100644 --- a/ncgen/generate.h +++ b/ncgen/generate.h @@ -55,7 +55,7 @@ extern int generator_reset(Generator*,void*); typedef int (*Writer)(Generator*,struct Symbol*,Bytebuffer*,int,const size_t*,const size_t*); extern void generate_attrdata(struct Symbol*, Generator*, Writer writer, Bytebuffer*); -extern void generate_vardata(struct Symbol*, Generator*, Writer writer,Bytebuffer*); +extern void generate_vardata(struct Symbol*, Generator*, Writer writer, Bytebuffer*); extern void generate_basetype(struct Symbol*,NCConstant*,Bytebuffer*,Datalist*,Generator*); #endif /*DATA_H*/ diff --git a/nczarr_test/zmapio.c b/nczarr_test/zmapio.c index 679ab2af27..0007b7f3d1 100644 --- a/nczarr_test/zmapio.c +++ b/nczarr_test/zmapio.c @@ -277,7 +277,6 @@ rootpathfor(const char* path) NCURI* uri = NULL; char* rootpath = NULL; NClist* segments = nclistnew(); - char* p = NULL; ncuriparse(path,&uri); if(uri == NULL) goto done; @@ -287,7 +286,8 @@ rootpathfor(const char* path) rootpath = strdup("/"); /*constant*/ break; #ifdef ENABLE_S3 - case NCZM_S3: + case NCZM_S3: { + char* p = NULL; /* Split the path part */ if((stat = nczm_split(uri->path,segments))) goto done; /* remove the bucket name */ @@ -295,7 +295,7 @@ rootpathfor(const char* path) nullfree(p); p = NULL; /* Put it back together */ if((stat = nczm_join(segments,&rootpath))) goto done; - break; + } break; #endif default: stat = NC_EINVAL; diff --git a/plugins/Makefile.am b/plugins/Makefile.am index f7d86fd078..5ad2a51435 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -65,10 +65,13 @@ EXTRA_DIST += H5checksum.c if ENABLE_FILTER_TESTING if ENABLE_NCZARR_FILTERS -plugins_to_install += lib__nch5fletcher32.la lib__nch5shuffle.la lib__nch5deflate.la +plugins_to_install += lib__nch5fletcher32.la lib__nch5shuffle.la lib__nch5shuffle_la_SOURCES = H5Zshuffle.c lib__nch5fletcher32_la_SOURCES = H5Zfletcher32.c H5checksum.c +if HAVE_DEFLATE +plugins_to_install += lib__nch5deflate.la lib__nch5deflate_la_SOURCES = H5Zdeflate.c +endif # Need our version of szip if libsz available and we are not using HDF5 if HAVE_SZ diff --git a/unit_test/CMakeLists.txt b/unit_test/CMakeLists.txt index f186a86269..e9cd48e082 100644 --- a/unit_test/CMakeLists.txt +++ b/unit_test/CMakeLists.txt @@ -19,15 +19,14 @@ IF(USE_X_GETOPT) SET(XGETOPTSRC "${CMAKE_CURRENT_SOURCE_DIR}/../libdispatch/XGetopt.c") ENDIF() -IF(NOT MSVC) - IF(ENABLE_NETCDF_4) - SET(UNIT_TESTS ${UNIT_TESTS} tst_nclist tst_nc4internal) - ENDIF(ENABLE_NETCDF_4) -ENDIF(NOT MSVC) - -FOREACH(CTEST ${UNIT_TESTS}) - add_bin_test(unit_test ${CTEST}) -ENDFOREACH() +IF(ENABLE_HDF5) + IF(NOT MSVC) + add_bin_test(unit_test tst_nclist) + add_bin_test(unit_test tst_nc4internal) + ENDIF(NOT MSVC) + build_bin_test(tst_reclaim ${XGETOPTSRC}) + add_sh_test(unit_test run_reclaim_tests) +ENDIF(ENABLE_HDF5) # Path convert test(s) add_bin_test(unit_test test_pathcvt) @@ -35,8 +34,8 @@ add_bin_test(unit_test test_pathcvt) IF(BUILD_UTILITIES) IF(ENABLE_S3 AND WITH_S3_TESTING) # SDK Test -BUILD_BIN_TEST(test_s3sdk ${XGETOPTSRC}) -ADD_SH_TEST(unit_test run_s3sdk) +build_bin_test(test_s3sdk ${XGETOPTSRC}) +add_sh_test(unit_test run_s3sdk) ENDIF() ENDIF() diff --git a/unit_test/Makefile.am b/unit_test/Makefile.am index a0f730a409..9948bab576 100644 --- a/unit_test/Makefile.am +++ b/unit_test/Makefile.am @@ -32,10 +32,11 @@ tst_xcache_SOURCES = tst_xcache.c timer_utils.c timer_utils.h TESTS += tst_nclist test_ncuri test_pathcvt tst_exhash tst_xcache -if USE_NETCDF4 -check_PROGRAMS += tst_nc4internal +if USE_HDF5 +check_PROGRAMS += tst_nc4internal tst_reclaim TESTS += tst_nc4internal -endif # USE_NETCDF4 +TESTS += run_reclaim_tests.sh +endif # USE_HDF5 if ENABLE_S3 if ENABLE_S3_TESTALL @@ -44,8 +45,10 @@ TESTS += run_s3sdk.sh endif endif -EXTRA_DIST = CMakeLists.txt run_s3sdk.sh -EXTRA_DIST += nctest_netcdf4_classic.nc +EXTRA_DIST = CMakeLists.txt run_s3sdk.sh run_reclaim.sh +EXTRA_DIST += nctest_netcdf4_classic.nc reclaim_tests.cdl + +CLEANFILES = reclaim_tests*.txt reclaim_tests.nc # If valgrind is present, add valgrind targets. @VALGRIND_CHECK_RULES@ diff --git a/unit_test/reclaim_tests.cdl b/unit_test/reclaim_tests.cdl new file mode 100644 index 0000000000..acf716793d --- /dev/null +++ b/unit_test/reclaim_tests.cdl @@ -0,0 +1,75 @@ +netcdf reclaim_tests { +types: + int(*) vint_t; + string(*) vstr_t; + vint_t(*) vvint_t; + vstr_t(*) vvstr_t; + + compound cmpd_atomic_t { /* compound with all atomic fields */ + int f1; + float f2(2,2); + }; + compound cmpd_str_t { /* compound with string typed field(s) */ + string f1(3); + }; + compound cmpd_vlen_t { /* compound with vlen typed field(s) */ + vint_t f1(2); + }; + compound cmpd_cmpd_t { /* compound with nested variable length compound field(s) */ + vstr_t f1(2) ; + cmpd_vlen_t f2(2); + }; + + cmpd_atomic_t(*) vcmpd_atomic_t; + cmpd_str_t(*) vcmpd_str_t; + cmpd_vlen_t(*) vcmpd_vlen_t; + cmpd_cmpd_t(*) vcmpd_cmpd_t; + +dimensions: + d1 = 1; + d2 = 2; + d3 = 3; + d4 = 4; + +variables: + /* atomic types */ + int intvar(d4); /* fixed-size atomic type */ + string strvar(d4); /* string type */ + + /* vlen types */ + vint_t vintvar(d4); + vstr_t vstrvar(d4); + vvint_t vvintvar(d4); + vvstr_t vvstrvar(d4); + + /* compound types */ + cmpd_atomic_t catomvar(d2); + cmpd_str_t cstrvar(d2); + cmpd_vlen_t cvlenvar(d2); + cmpd_cmpd_t ccmpdvar(d2); + + /* vlen of compound types */ + vcmpd_atomic_t vcatomvar; + vcmpd_str_t vcstrvar(d2); + vcmpd_vlen_t vcvlenvar(d3); + vcmpd_cmpd_t vccmpdvar; + +data: + intvar = 17, 13, 11, 7 ; + strvar = "a", "ab", "abc", "abcd" ; + vintvar = {1}, {1,2}, {1,2,3}, {1,2,3,4} ; + vstrvar = {"a","ab","abc","abcd"}, {"abcd","abc","ab"}, {"a","ab"}, {"abcd"} ; + vvintvar = {{1}, {1,2}}, {{1,2,3}}, {{1,2,3,4}}, {{17,13},{11,7},{5,3},{2,1}} ; + vvstrvar = {{"1"}, {"1","2"}}, {{"1","2","3"}}, {{"1","2","3","4"}}, {{"17","13"},{"11","7"},{5,"3"},{"2","1"}} ; + catomvar = {17, {1.1, 2.2, 3.3, 4.4}}, {27, {4.1, 3.2, 3.3, 1.4}} ; + cstrvar = {{"a","abc","abcd"}}, {{"x","yz", "ijkl"}} ; + cvlenvar = {{{17,13},{11}}}, {{{3,5},{7,11}}} ; + ccmpdvar = {{{"xxx"},{"yyy"}},{{{{17,13},{12,9}}},{{{1,3},{2,11}}}}}, {{{"uv"},{"w"}},{{{{117,113},{112,119}}},{{{111,113},{112,1111}}}}} ; + vcatomvar = {{17, {1.1, 2.2, 3.3, 4.4}}, {27, {4.1, 3.2, 3.3, 1.4}}} ; + vcstrvar = {{{"c11a","c12a","c13a"}},{{"c11b","c12b","c13b"}},{{"c11c","c12c","c13c"}},{{"c11d","c12d","c13d"}}},{{{"c21a","c22a","c23a"}},{{"c21b","c22b","c23b"}},{{"c21c","c22c","c23c"}},{{"c21d","c22d","c23d"}}}; + vcvlenvar = {{{{117,113},{111}}}, {{{13,15},{17,111}}},{{{217,213},{211}}}, {{{23,25},{27,211}}}}, {{{{2117,2113},{2211}}}, {{{2223,2215},{2227,22111}}},{{{22217,22213},{22211}}}, {{{2223,2225},{2227,22211}}}}, {{{{33117,33113},{33111}}}, {{{3313,3315},{3317,33111}}},{{{33217,33213},{33211}}}, {{{3323,3325},{3327,33211}}}}; + vccmpdvar = {{{{"xxx"},{"yyy"}},{{{{17,13},{12,9}}},{{{1,3},{2,11}}}}}, {{{"uv"},{"w"}},{{{{117,113},{112,119}}},{{{111,113},{112,1111}}}}}}; + +} //reclaim_tests + + diff --git a/unit_test/run_reclaim_tests.sh b/unit_test/run_reclaim_tests.sh new file mode 100755 index 0000000000..d17777cf73 --- /dev/null +++ b/unit_test/run_reclaim_tests.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +set -e + +rm -f reclaim_tests.nc reclaim_tests*.txt +${NCGEN} -4 ${srcdir}/reclaim_tests.cdl +${execdir}/tst_reclaim > reclaim_tests.txt +sed -e '/^(o)/p' -ed reclaim_tests.txt | sed -e 's/^(o) //' > reclaim_tests_o.txt +sed -e '/^(c)/p' -ed reclaim_tests.txt | sed -e 's/^(c) //' > reclaim_tests_c.txt +diff reclaim_tests_o.txt reclaim_tests_c.txt + + + diff --git a/unit_test/tst_reclaim.c b/unit_test/tst_reclaim.c new file mode 100644 index 0000000000..869f3c71f3 --- /dev/null +++ b/unit_test/tst_reclaim.c @@ -0,0 +1,345 @@ +/********************************************************************* + * Copyright 2018, UCAR/Unidata + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + *********************************************************************/ + +/** +Test the various type walker functions: +nc_reclaim_data => NC_reclaim_data +nc_reclaim_data_all => NC_reclaim_data_all => NC_reclaim_data +nc_copy_data => NC_copy_data +nc_copy_data_all => NC_copy_data_all => NC_copy_data +nc_dump_data => NC_dump_data + +See the file "reclaim_tests.cdl" to see the input file semantics. +*/ + + +#include "config.h" +#include +#include +#include +#include +#include "netcdf.h" +#include "netcdf_aux.h" + +#define NCCATCH +#include "nclog.h" + +#ifdef HAVE_GETOPT_H +#include +#endif + +#if defined(_WIN32) && !defined(__MINGW32__) +#include "XGetopt.h" +#endif + +#define DEBUG + +#define FILE "reclaim_tests.nc" + +#define MAXOBJECTS 1024 + +struct Options { + int debug; +} dumpoptions; + +struct Type { + char name[NC_MAX_NAME]; + int tid; + size_t size; +}; + +struct Dim { + char name[NC_MAX_NAME]; + int did; + size_t size; +}; + +struct Var { + char name[NC_MAX_NAME]; + int vid; + int tid; + size_t dimprod; + void* data; +}; + +struct Metadata { + int ncid; + /* type ids */ + struct Type vint_t; + struct Type vstr_t; + struct Type vvint_t; + struct Type vvstr_t; + struct Type cmpd_atomic_t; + struct Type cmpd_str_t; + struct Type cmpd_vlen_t; + struct Type cmpd_cmpd_t; + struct Type vcmpd_atomic_t; + struct Type vcmpd_str_t; + struct Type vcmpd_vlen_t; + struct Type vcmpd_cmpd_t; + /* dim ids */ + struct Dim d1; + struct Dim d2; + struct Dim d4; + /* var ids */ + struct Var intvar; + struct Var strvar; + struct Var vintvar; + struct Var vstrvar; + struct Var vvintvar; + struct Var vvstrvar; + struct Var catomvar; + struct Var cstrvar; + struct Var cvlenvar; + struct Var ccmpdvar; + struct Var vcatomvar; + struct Var vcstrvar; + struct Var vcvlenvar; + struct Var vccmpdvar; +} metadata; + +/* atomic Types */ +static struct Type atomics[NC_MAX_ATOMIC_TYPE+1]; + +/* Track the metadata objects */ +static struct Type* typemap[MAXOBJECTS]; +static struct Dim* dimmap[MAXOBJECTS]; +static struct Var* varmap[MAXOBJECTS]; + + +#define CHECK(code) do {stat = check(code,__func__,__LINE__); if(stat) {goto done;}} while(0) + +static int +check(int code, const char* fcn, int line) +{ + if(code == NC_NOERR) return code; + fprintf(stderr,"***fail: (%d) %s @ %s:%d\n",code,nc_strerror(code),fcn,line); +#ifdef debug + abort(); +#endif + exit(1); +} + +static int +initialize(void) +{ + int i,stat = NC_NOERR; + memset(atomics,0,sizeof(atomics)); + memset(typemap,0,sizeof(typemap)); + memset(dimmap,0,sizeof(dimmap)); + memset(varmap,0,sizeof(varmap)); + for(i=1;i<=NC_MAX_ATOMIC_TYPE;i++) { + struct Type* t = &atomics[i]; + t->tid = i; + if((stat=ncaux_inq_any_type(0,i,t->name,&t->size,NULL,NULL,NULL))) return NCTHROW(stat); + typemap[i] = t; + } + return NCTHROW(stat); +} + +static void +cleanup(void) +{ + int i; + for(i=0;idata != NULL) free(varmap[i]->data); + } +} + + +static int +setuptype(int ncid, const char* name, struct Type* t) +{ + int stat = NC_NOERR; + strcpy(t->name,name); + if((stat=nc_inq_typeid(ncid,name,&t->tid))) return NCTHROW(stat); + if((stat=ncaux_inq_any_type(ncid,t->tid,NULL,&t->size,NULL,NULL,NULL))) return NCTHROW(stat); + typemap[t->tid] = t; + return NCTHROW(stat); +} + +static int +setupdim(int ncid, const char* name, struct Dim* dim) +{ + int stat = NC_NOERR; + strcpy(dim->name,name); + if((stat = nc_inq_dimid(ncid,name,&dim->did))) return NCTHROW(stat); + if((nc_inq_dimlen(ncid,dim->did,&dim->size))) return NCTHROW(stat); + dimmap[dim->did] = dim; + return NCTHROW(stat); +} + +static int +setupvar(int ncid, const char* name, struct Var* var) +{ + int stat = NC_NOERR; + int i,ndims; + int dimids[NC_MAX_VAR_DIMS]; + size_t product,dimsizes[NC_MAX_VAR_DIMS]; + + strcpy(var->name,name); + if((stat=nc_inq_varid(ncid,name,&var->vid))) return NCTHROW(stat); + if((stat=nc_inq_vartype(ncid,var->vid,&var->tid))) return NCTHROW(stat); + if((stat=nc_inq_varndims(ncid,var->vid,&ndims))) return NCTHROW(stat); + if((stat=nc_inq_vardimid(ncid,var->vid,dimids))) return NCTHROW(stat); + for(product=1,i=0;idimprod = product; + varmap[var->vid] = var; + return NCTHROW(stat); +} + +static void +setup(struct Metadata* md) +{ + int stat = NC_NOERR; + + CHECK(nc_open(FILE,NC_NETCDF4,&md->ncid)); + + CHECK(setupdim(md->ncid,"d1",&md->d1)); + CHECK(setupdim(md->ncid,"d2",&md->d2)); + CHECK(setupdim(md->ncid,"d4",&md->d4)); + + CHECK(setuptype(md->ncid,"vint_t",&md->vint_t)); + CHECK(setuptype(md->ncid,"vstr_t",&md->vstr_t)); + CHECK(setuptype(md->ncid,"vvint_t",&md->vvint_t)); + CHECK(setuptype(md->ncid,"vvstr_t",&md->vvstr_t)); + CHECK(setuptype(md->ncid,"cmpd_atomic_t",&md->cmpd_atomic_t)); + CHECK(setuptype(md->ncid,"cmpd_str_t",&md->cmpd_str_t)); + CHECK(setuptype(md->ncid,"cmpd_vlen_t",&md->cmpd_vlen_t)); + CHECK(setuptype(md->ncid,"cmpd_cmpd_t",&md->cmpd_cmpd_t)); + CHECK(setuptype(md->ncid,"vcmpd_atomic_t",&md->vcmpd_atomic_t)); + CHECK(setuptype(md->ncid,"vcmpd_str_t",&md->vcmpd_str_t)); + CHECK(setuptype(md->ncid,"vcmpd_vlen_t",&md->vcmpd_vlen_t)); + CHECK(setuptype(md->ncid,"vcmpd_cmpd_t",&md->vcmpd_cmpd_t)); + + CHECK(setupvar(md->ncid,"intvar",&md->intvar)); + CHECK(setupvar(md->ncid,"strvar",&md->strvar)); + CHECK(setupvar(md->ncid,"vintvar",&md->vintvar)); + CHECK(setupvar(md->ncid,"vstrvar",&md->vstrvar)); + CHECK(setupvar(md->ncid,"vvintvar",&md->vvintvar)); + CHECK(setupvar(md->ncid,"vvstrvar",&md->vvstrvar)); + CHECK(setupvar(md->ncid,"catomvar",&md->catomvar)); + CHECK(setupvar(md->ncid,"cstrvar",&md->cstrvar)); + CHECK(setupvar(md->ncid,"cvlenvar",&md->cvlenvar)); + CHECK(setupvar(md->ncid,"ccmpdvar",&md->ccmpdvar)); + CHECK(setupvar(md->ncid,"vcatomvar",&md->vcatomvar)); + CHECK(setupvar(md->ncid,"vcstrvar",&md->vcstrvar)); + CHECK(setupvar(md->ncid,"vcvlenvar",&md->vcvlenvar)); + CHECK(setupvar(md->ncid,"vccmpdvar",&md->vccmpdvar)); + +done: + assert(stat == NC_NOERR); +} + +static int +readvar(int ncid, struct Var* v) +{ + int stat = NC_NOERR; + size_t size; + struct Type* basetype = typemap[v->tid]; + size = (basetype->size * v->dimprod); + v->data = calloc(1,size); + if((stat=nc_get_var(ncid,v->vid,v->data))) + return NCTHROW(stat); + return NCTHROW(stat); +} + +static int +dumpvar(int ncid, struct Var* v, void* data, char** bufp) +{ + int stat = NC_NOERR; + if((stat=ncaux_dump_data(ncid,v->tid,data,v->dimprod,bufp))) return NCTHROW(stat); + return NCTHROW(stat); +} + +static int +testvar(int ncid, struct Var* v) +{ + int stat = NC_NOERR; + char* buforig = NULL; + char* bufcopy = NULL; + void* copy = NULL; + + + if((stat=readvar(ncid,v))) return NCTHROW(stat); + if((stat=dumpvar(ncid,v,v->data,&buforig))) return NCTHROW(stat); + printf("(o) %s: %s\n",v->name,buforig); + // Test copying + if((stat=nc_copy_data_all(ncid,v->tid,v->data,v->dimprod,©))) return NCTHROW(stat); + // Print copy + if((stat=dumpvar(ncid,v,copy,&bufcopy))) return NCTHROW(stat); + printf("(c) %s: %s\n",v->name,bufcopy); + /* Compare */ + if(strcmp(buforig,bufcopy) != 0) + fprintf(stderr,"*** orig != copy\n"); + if(buforig) free(buforig); + if(bufcopy) free(bufcopy); + + // reclaim original + if((stat=nc_reclaim_data_all(ncid,v->tid,v->data,v->dimprod))) return NCTHROW(stat); + // reclaim copy + if((stat=nc_reclaim_data_all(ncid,v->tid,copy,v->dimprod))) return NCTHROW(stat); + v->data = NULL; + return NCTHROW(stat); +} + +static void +test(struct Metadata* md) +{ + int stat = NC_NOERR; + CHECK(testvar(md->ncid,&md->intvar)); + CHECK(testvar(md->ncid,&md->strvar)); + CHECK(testvar(md->ncid,&md->vintvar)); + CHECK(testvar(md->ncid,&md->vstrvar)); + CHECK(testvar(md->ncid,&md->vvintvar)); + CHECK(testvar(md->ncid,&md->vvstrvar)); + CHECK(testvar(md->ncid,&md->catomvar)); + CHECK(testvar(md->ncid,&md->cstrvar)); + CHECK(testvar(md->ncid,&md->cvlenvar)); + CHECK(testvar(md->ncid,&md->ccmpdvar)); + CHECK(testvar(md->ncid,&md->vcatomvar)); + CHECK(testvar(md->ncid,&md->vcstrvar)); + CHECK(testvar(md->ncid,&md->vcvlenvar)); + CHECK(testvar(md->ncid,&md->vccmpdvar)); +done: + return; +} + +int +main(int argc, char** argv) +{ + int c,stat = NC_NOERR; + + CHECK(nc_initialize()); + CHECK(initialize()); + + /* Init options */ + memset((void*)&dumpoptions,0,sizeof(dumpoptions)); + + while ((c = getopt(argc, argv, "dhk:tu:")) != EOF) { + switch(c) { + case 'd': + dumpoptions.debug = 1; + break; + case '?': + fprintf(stderr,"unknown option\n"); + stat = NC_EINVAL; + goto done; + } + } + + setup(&metadata); + test(&metadata); + +done: + cleanup(); + if(metadata.ncid) nc_close(metadata.ncid); + (void)nc_finalize(); + exit(stat?1:0); +}