From 51d22184871da850af4e84d10227d62ed61ec14b Mon Sep 17 00:00:00 2001 From: "Mark (he/his) C. Miller" Date: Tue, 29 Aug 2023 17:14:47 -0700 Subject: [PATCH] Documentation fixes and fixes from testing 1.1.1 (#125) * minor fixes for gnu make builds * updates to documentation * update version numbers in docs and in headers --- config.make | 16 ++- docs/cd_vals.rst | 4 + docs/cmake.rst | 95 --------------- docs/conf.py | 3 +- docs/h5repack.rst | 98 +++++++-------- docs/index.rst | 1 - docs/installation.rst | 276 ++++++++++++++++++++++++++++-------------- docs/interfaces.rst | 143 ++++++++-------------- src/H5Zzfp_version.h | 2 +- 9 files changed, 294 insertions(+), 344 deletions(-) delete mode 100644 docs/cmake.rst diff --git a/config.make b/config.make index 5d57b6b..d125002 100644 --- a/config.make +++ b/config.make @@ -30,12 +30,12 @@ ifeq ($(PWD_BASE),src) H5Z_ZFP_BASE := . else ifeq ($(PWD_BASE),test) H5Z_ZFP_BASE := ../src -else ifeq ($(PWD_BASE),H5Z-ZFP) +else ifeq ($(findstring H5Z-ZFP, $(PWD_BASE)), H5Z-ZFP) H5Z_ZFP_BASE := ./src endif H5Z_ZFP_PLUGIN := $(H5Z_ZFP_BASE)/plugin -H5Z_ZFP_VERSINFO := $(shell grep '^\#define H5Z_FILTER_ZFP_VERSION_[MP]' $(H5Z_ZFP_BASE)/H5Zzfp_plugin.h | cut -d' ' -f3 | tr '\n' '.' | cut -d'.' -f-3 2>/dev/null) +H5Z_ZFP_VERSINFO := $(shell grep '^\#define H5Z_FILTER_ZFP_VERSION_[MP]' $(H5Z_ZFP_BASE)/H5Zzfp_version.h | cut -d' ' -f3 | tr '\n' '.' | cut -d'.' -f-3 2>/dev/null) ZFP_HAS_REVERSIBLE := ifneq ($(ZFP_HOME),) ZFP_HAS_REVERSIBLE := $(shell grep zfp_stream_set_reversible $(ZFP_HOME)/include/zfp.h 2>/dev/null) @@ -67,6 +67,7 @@ HAS_ICC := $(shell basename $$(which icc 2>/dev/null) 2>/dev/null) HAS_PGCC := $(shell basename $$(which pgcc 2>/dev/null) 2>/dev/null) HAS_XLCR := $(shell basename $$(which xlc_r 2>/dev/null) 2>/dev/null) HAS_BGXLCR := $(shell basename $$(which bgxlc_r 2>/dev/null) 2>/dev/null) +HAS_CC := $(shell basename $$(which cc 2>/dev/null) 2>/dev/null) # Common Fortran compilers HAS_GFORTRAN := $(shell basename $$(which gfortran 2>/dev/null) 2>/dev/null) @@ -81,6 +82,8 @@ ifeq ($(CC),) CC = $(HAS_CLANG) else ifneq ($(strip $(HAS_GCC)),) CC = $(HAS_GCC) + else ifneq ($(strip $(HAS_CC)),) + CC = $(HAS_CC) endif else ifneq ($(findstring ppc, $(PROCESSOR),),) ifneq ($(strip $(HAS_BGXLCR)),) @@ -93,6 +96,8 @@ ifeq ($(CC),) else ifneq ($(strip $(HAS_GCC)),) CC = $(HAS_GCC) + else ifneq ($(strip $(HAS_CLANG)),) + CC = $(HAS_CLANG) else ifneq ($(strip $(HAS_ICC)),) CC = $(HAS_ICC) else ifneq ($(strip $(HAS_PGCC)),) @@ -117,7 +122,7 @@ ifneq ($(findstring gcc, $(CC)),) else ifneq ($(findstring clang, $(CC)),) SOEXT ?= dylib SHFLAG ?= -dynamiclib - PREPATH = -L + PREPATH = -Wl,-rpath, else ifneq ($(findstring icc, $(CC)),) CFLAGS += -fpic SOEXT ?= so @@ -128,6 +133,11 @@ else ifneq ($(findstring pgcc, $(CC)),) SOEXT ?= so SHFLAG ?= -shared PREPATH = -Wl,-rpath, +else ifneq ($(findstring cc, $(CC)),) + CFLAGS += -fPIC + SOEXT ?= so + SHFLAG ?= -shared + PREPATH = -Wl,-rpath, else ifneq ($(findstring xlc_r, $(CC)),) CFLAGS += -qpic SOEXT ?= so diff --git a/docs/cd_vals.rst b/docs/cd_vals.rst index 64083e4..d96f0ae 100644 --- a/docs/cd_vals.rst +++ b/docs/cd_vals.rst @@ -2,6 +2,10 @@ H5Z-ZFP and the HDF5 filter's cd_values ======================================= +.. note:: + The details described here are likely relevant only to *developers* of the H5Z-ZFP_ filter. + If you just want to *use* the filter, you can ignore this material. + The HDF5_ library uses an array of values, named ``cd_values`` in formal arguments documenting various API functions, for managing *auxiliary data* for a filter. Instances of this ``cd_values`` array are used in two subtly different ways within HDF5. diff --git a/docs/cmake.rst b/docs/cmake.rst deleted file mode 100644 index 471e650..0000000 --- a/docs/cmake.rst +++ /dev/null @@ -1,95 +0,0 @@ -.. _cmake: - -===== -CMake -===== - -It is possible to build the H5Z-ZFP filter using the CMake build system. If you decide to -do so, please build ZFP also with its CMake build system. This is necessary to get the -correct dependencies from ZFP. For example, it is possible to build ZFP with OpenMP support. -The resulting CMake config files of ZFP build will make sure that this OpenMP dependency is -correctly propagated to the build of H5Z-ZFP filter. However, for HDF5 it is not -necessary to build it with its CMake build system but it is strongly recommended. - ------------------ -Compiling H5Z-ZFP ------------------ - -Similar as for the Makefile installation, the CMake build system is designed such it -compiles both the standalone HDF5_ *plugin* and a separate *library* an application -can explicitly link. See :ref:`plugin-vs-library` - -Once you have installed both HDF5 and ZFP, you can compile H5Z-ZFP_ using a command=line... - -:: - - export HDF5_DIR= - export ZFP_DIR= - CC= FC= cmake -DCMAKE_INSTALL_PREFIX= - -where ```` is a directory containing ZFP_ ``inc[lude]`` and ``lib`` directories and -```` is a directory containing HDF5_ ``include`` and ``lib`` directories. Furthermore, -``src-dir`` is the directory where the source is located and ``path-to-install`` is the directory in -which the resulting *plugin* and *library* will be installed. Once cmake has finished successfully, -you can build and install the filter using the command... - -:: - - make install - -This cmake and make combination builds both the C and Fortran interface. In the case you want to specify -the ```` and ``>`` via command-line to CMake, the command looks like this... - -:: - - CC= FC= cmake -DCMAKE_INSTALL_PREFIX= - -DCMAKE_PREFIX_PATH=";" - -Please, notice the double quotes in the CMAKE_PREFIX_PATH expression. These are needed to make sure that semicolon -is interpreted as a semicolon instead of a new command. - -It is possible to build the filter without the Fortran interface. This is done as follows... - -:: - - export HDF5_DIR= - export ZFP_DIR= - CC= cmake -DCMAKE_INSTALL_PREFIX= -DFORTRAN_INTERFACE:BOOL=OFF - -followed by the same make command... - -:: - - make install - -------------------------------------------- -Including H5Z-ZFP filter in a CMake project -------------------------------------------- - -Suppose you have built the H5Z-ZFP filter using the CMake build system and installed it in ````. -To include it in another CMake project is done using the following steps. First edit the ``CMakeLists.txt`` -by adding the following two lines... - -:: - - cmake_policy(SET CMP0028 NEW) # Double colon in target name means ALIAS or IMPORTED target. - ... - set(H5Z_ZFP_USE_STATIC_LIBS OFF) - find_package(H5Z_ZFP 1.0.1 CONFIG) - ... - target_link_libraries( h5z_zfp::h5z_zfp) - ... - -where ```` in the target within the CMake project. This could be, for example, a executable or library. -Furthermore, check if the cmake version is equal or greater than 3.9. Next, you need to make sure that the filter -can be found by CMake, followed CMake itself and make... - -:: - - export H5Z_ZFP_DIR= - CC= cmake -DCMAKE_INSTALL_PREFIX= - make install - -The cmake command itself could be different depending on the CMake project you have created. If you want -to make use of the H5Z_ZFP *library* instead of the plugin, change cmake variable H5Z_ZFP_USE_STATIC_LIBS -to ON and build the project. diff --git a/docs/conf.py b/docs/conf.py index 98903ed..6021417 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,6 +51,7 @@ .. _HDF5: https://docs.hdfgroup.org/hdf5/develop/ .. _H5Z-ZFP: https://github.com/LLNL/H5Z-ZFP .. _Spack: https://spack.io +.. _CMake: https://cmake.org/cmake/help/latest/ """ # The version info for the project you're documenting, acts as replacement for @@ -60,7 +61,7 @@ # The short X.Y version. version = '1.1' # The full version, including alpha/beta/rc tags. -release = '1.1.0' +release = '1.1.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/h5repack.rst b/docs/h5repack.rst index 0b5cb4e..b760de0 100644 --- a/docs/h5repack.rst +++ b/docs/h5repack.rst @@ -1,76 +1,69 @@ ================================== Using H5Z-ZFP Plugin with H5Repack ================================== -A convenient way to use and play with the ZFP_ filter is a *plugin* with -the HDF5_ `h5repack `__ (**note**: the preceding link to ``h5repack`` may not render correctly from Safari) -utility using the ``-f`` filter argument to apply ZFP to existing data in a file. +A convenient way to use and play with the ZFP_ filter is as a *plugin* with the HDF5_ `h5repack `__ utility using the ``-f`` command-line argument to apply ZFP to existing data in a file. ----------------- Patching h5repack ----------------- -Some versions of HDF5_'s ``h5repack`` utility contain a bug that prevents -proper parsing of the ``-f`` argument's option. In order to use ``h5repack`` -with ``-f`` argument as described here, you need to apply the patch from -`h5repack_parse.patch `_. -To do so, after you've downloaded and untar'd HDF5_ but before you've built -it, do something like the following using HDF5-1.8.14 as an example:: - gunzip < hdf5-1.8.14.tar.gz | tar xvf - - cd hdf5-1.8.14 - patch ./tools/h5repack/h5repack_parse.c /h5repack_parse.patch +.. warning:: + + Versions of HDF5_'s ``h5repack`` utility prior to 1.10.4 contain a bug that prevents proper parsing of the ``-f`` argument's option. + In order to use ``h5repack`` with ``-f`` argument as described here, you need to apply the patch from `h5repack_parse.patch `_. + To do so, after you've downloaded and untar'd HDF5_ but before you've built it, do something like the following using HDF5-1.8.14 as an example:: + + gunzip < hdf5-1.8.14.tar.gz | tar xvf - + cd hdf5-1.8.14 + patch ./tools/h5repack/h5repack_parse.c /h5repack_parse.patch ------------------------------------- Constructing an HDF5_ cd_values array ------------------------------------- HDF5_'s ``h5repack`` utility uses only the *generic* interface to HDF5_ filters. -Another challenge in using ``h5repack`` as described here is constructing the set -``unsigned int cd_values`` as is used in -`H5Pset_filter() `__ -required by the *generic* HDF5_ filter interface, especially because -of the type-punning (doubles as unsigned int) involved. - -**Note:** Querying an existing dataset using ``h5dump`` or ``h5ls`` to obtain -the *cd_values* stored with a ZFP_ compressed dataset -**will not provide the correct cd_values**. This is because the *cd_values* -stored in the file are different from those used in the *generic* interface -to invoke the ZFP_ filter. - -To facilitate constructing a valid ``-f`` argument to ``h5repack``, we have -created a utility program, ``print_h5repack_farg``, which is presently in the -``test`` directory and is built when tests are built. - -You can use the ``print_h5repack_farg`` utility to read a command-line -consisting of ZFP_ filter parameters you wish to use and it will output -part of the command-line needed for the ``-f`` argument to ``h5repack``. +Another challenge in using ``h5repack`` as described here is constructing the set ``unsigned int cd_values`` as is used in `H5Pset_filter() `__ required by the *generic* HDF5_ filter interface, especially because of the type-punning (doubles as unsigned int) which may be involved. + +.. note:: + + Querying an existing dataset using ``h5dump`` or ``h5ls`` to obtain the ``cd_values`` *stored* with a ZFP_ compressed dataset will not provide the correct ``cd_values`` needed to invoke the filter. + This is because the ``cd_values`` stored in the file are different from those needed in the *generic* interface to invoke the ZFP_ filter. + +To facilitate constructing a valid ``-f`` argument to ``h5repack``, there is a utility program, ``print_h5repack_farg``, which is presently in the ``test`` directory and is built when tests are built. +This program was originally written simply to facilitate testing of H5Z-ZFP filter. +It should eventually be made a first class *tool* installed with H5Z-ZFP. +However, it is presently made only as part of *testing* H5Z-ZFP. + +The ``print_h5repack_farg`` utility can be used to read a command-line consisting of ZFP_ filter parameters you wish to use and output part of the command-line needed for the ``-f`` argument to ``h5repack``. -------- Examples -------- -In the examples below, we use ``h5repack`` with the example data file, -``mesh.h5`` in the tests directory. +.. note:: + + The examples below assume H5Z-ZFP has been configured to run tests by either a preceding ``make check`` (if using a vanilla ``gmake`` build) or ``make test`` (if using a ``cmake`` build). + +In the examples below, we use ``h5repack`` with the example data file, ``mesh.h5`` in the tests directory. -To use ZFP_ filter in *rate* mode with a rate of ``4.5`` bits per value, -first, use the ``print_h5repack_farg``:: +To use ZFP_ filter in *rate* mode with a rate of ``4.5`` bits per value, first, use the ``print_h5repack_farg``:: % ./print_h5repack_farg zfpmode=1 rate=4.5 Print cdvals for set of ZFP compression paramaters... - zfpmode=1 set zfp mode (1=rate,2=prec,3=acc,4=expert) + zfpmode=1 set zfp mode (1=rate,2=prec,3=acc,4=expert,5=rev) rate=4.5 set rate for rate mode of filter acc=0 set accuracy for accuracy mode of filter - prec=11 set precision for precision mode of zfp filter + prec=0 set precision for precision mode of zfp filter minbits=0 set minbits for expert mode of zfp filter - maxbits=4171 set maxbits for expert mode of zfp filter - maxprec=64 set maxprec for expert mode of zfp filter - minexp=-1074 set minexp for expert mode of zfp filter + maxbits=0 set maxbits for expert mode of zfp filter + maxprec=0 set maxprec for expert mode of zfp filter + minexp=0 set minexp for expert mode of zfp filter help=0 this help message h5repack -f argument... -f UD=32013,0,4,1,0,0,1074921472 -Next, cut-n-paste the ``-f UD=32013,0,4,1,0,0,1074921472`` in a command -to ``h5repack`` like so:: +Next, cut-n-paste the ``-f UD=32013,0,4,1,0,0,1074921472`` in a command to ``h5repack`` like so:: env LD_LIBRARY_PATH=:$(LD_LIBRARY_PATH) \ HDF5_PLUGIN_PATH= \ @@ -80,30 +73,27 @@ to ``h5repack`` like so:: -l VelocityX_2D:CHUNK=21x31 \ mesh.h5 mesh_repack.h5 -where the ``-l`` arguments indicate the dataset(s) to be re-packed as well -as their (new) chunking. +where the ``-l`` arguments indicate the dataset(s) to be re-packed as well as their (new) chunking. -To use ZFP_ filter in *accuracy* mode with an accuracy of ``0.075``, -first, use the ``print_h5repack_farg``:: +To use ZFP_ filter in *accuracy* mode with an accuracy of ``0.075``, first, use the ``print_h5repack_farg``:: % ./print_h5repack_farg zfpmode=3 acc=0.075 Print cdvals for set of ZFP compression paramaters... - zfpmode=3 set zfp mode (1=rate,2=prec,3=acc,4=expert) - rate=4 set rate for rate mode of filter + zfpmode=3 set zfp mode (1=rate,2=prec,3=acc,4=expert,5=rev) + rate=3.5 set rate for rate mode of filter acc=0.075 set accuracy for accuracy mode of filter - prec=11 set precision for precision mode of zfp filter + prec=0 set precision for precision mode of zfp filter minbits=0 set minbits for expert mode of zfp filter - maxbits=4171 set maxbits for expert mode of zfp filter - maxprec=64 set maxprec for expert mode of zfp filter - minexp=-1074 set minexp for expert mode of zfp filter + maxbits=0 set maxbits for expert mode of zfp filter + maxprec=0 set maxprec for expert mode of zfp filter + minexp=0 set minexp for expert mode of zfp filter help=0 this help message h5repack -f argument... -f UD=32013,0,4,3,0,858993459,1068708659 -Next, cut-n-paste the ``-f UD=32013,0,4,3,0,858993459,1068708659`` in a command -to ``h5repack`` like so:: +Next, cut-n-paste the ``-f UD=32013,0,4,3,0,858993459,1068708659`` in a command to ``h5repack`` like so:: env LD_LIBRARY_PATH=:$(LD_LIBRARY_PATH) \ HDF5_PLUGIN_PATH= \ diff --git a/docs/index.rst b/docs/index.rst index c77a024..48d376a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,4 +21,3 @@ Contents: h5repack endian_issues tests - cmake diff --git a/docs/installation.rst b/docs/installation.rst index 7699b65..a0dc9ab 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -2,44 +2,23 @@ Installation ============ ------------------------------------------- -Installing via `Spack `_ ------------------------------------------- -The HDF5_ and ZFP_ libraries and the H5Z-ZFP_ plugin are all now part of the -Spack_ package manager. If you already have Spack_ installed, the easiest way to -install H5Z-ZFP_ is to simply use the Spack_ command ``spack install h5z-zfp``. -If you do not have Spack_ installed, it is very easy to install. +Three ways to install H5Z-ZFP_ are provided. +These are -:: - - git clone https://github.com/llnl/spack.git - . spack/share/spack/setup-env.sh - spack install h5z-zfp +* With generic (GNU) :ref:`make ` +* With :ref:`CMake ` +* With :ref:`Spack ` -By default, H5Z-ZFP_ will attempt to build with Fortran support which requires -a Fortran compiler. If you wish to exclude support for Fortran, use the command +For both generic make and CMake_, you are responsible for also installing (or knowing where the installations are) the dependencies, ZFP_ and HDF5_. +For Spack_ installations, Spack_ will handle installation of dependencies as well. -:: - - spack install h5z-zfp~fortran - -Note that these commands will build H5Z-ZFP_ **and** all of its dependencies including -the HDF5_ library (as well as a number of other dependencies you may not initially -expect. Be patient and let the build complete). In addition, by default, Spack_ installs -packages to directory *hashes* *within* the cloned Spack_ repository's directory tree, -``$spack/opt/spack``. You can find the resulting installed HDF5_ library with the command -``spack find -vp hdf5`` and your resulting H5Z-ZFP plugin installation with the command -``spack find -vp h5z-zfp``. If you wish to exercise more control over where Spack_ -installs things, have a look at -`configuring Spack `_ +.. _gnumake: -------------------- -Manual Installation -------------------- +---------------------------------- +Installiang via Generic (GNU) Make +---------------------------------- -If Spack_ is not an option for you, information on *manually* installing is provided -here. -H5Z-ZFP_ installation supports both vanilla (`GNU `__) Make (described below) as well as :ref:`CMake `. +H5Z-ZFP_ installation supports both vanilla (`GNU `__) Make (described below) as well as :ref:`CMake `. ^^^^^^^^^^^^^ Prerequisites @@ -49,47 +28,39 @@ Prerequisites * `HDF5 Library `_ * `H5Z-ZFP filter plugin `_ -^^^^^^^^^^^^^ -Compiling ZFP -^^^^^^^^^^^^^ - -* There is a ``Config`` file in top-level directory of the ZFP_ distribution that holds ``make`` variables - the ZFP_ Makefiles use. By default, this file is setup for a vanilla GNU compiler. If this is not the - appropriate compiler, edit ``Config`` as necessary to adjust the compiler and compilation flags. -* An important flag you **will** need to adjust in order to use the ZFP_ library with this HDF5_ filter is - the ``BIT_STREAM_WORD_TYPE`` CPP flag. To use ZFP_ with H5Z-ZFP_, the ZFP_ library **must** be compiled - with ``BIT_STREAM_WORD_TYPE`` of ``uint8``. Typically, this is achieved by including a line in ``Config`` - of the form ``DEFS += -DBIT_STREAM_WORD_TYPE=uint8``. If you attempt to use this filter with a ZFP_ - library compiled differently from this, the filter's ``can_apply`` method will always return - false. This will result in silently ignoring an HDF5_ client's request to compress data with - ZFP_. Also, be sure to see :ref:`endian-issues`. -* After you have setup ``Config``, simply run ``make`` and it will build the ZFP_ library placing - the library in a ``lib`` sub-directory and the necessary include files in ``inc[lude]`` sub-directory. +.. _zfp-config: + +^^^^^^^^^^^^^^ +Compiling ZFP_ +^^^^^^^^^^^^^^ + +* There is a ``Config`` file in top-level directory of the ZFP_ distribution that holds ``make`` variables the ZFP_ Makefiles use. By default, this file is setup for a vanilla GNU compiler. + If this is not the appropriate compiler, edit ``Config`` as necessary to adjust the compiler and compilation flags. +* An important flag you **will** need to adjust in order to use the ZFP_ library with this HDF5_ filter is the ``BIT_STREAM_WORD_TYPE`` CPP flag. + To use ZFP_ with H5Z-ZFP_, the ZFP_ library **must** be compiled with ``BIT_STREAM_WORD_TYPE`` of ``uint8``. + Typically, this is achieved by including a line in ``Config`` of the form ``DEFS += -DBIT_STREAM_WORD_TYPE=uint8``. + If you attempt to use this filter with a ZFP_ library compiled differently from this, the filter's ``can_apply`` method will always return false. + This will result in silently ignoring an HDF5_ client's request to compress data with ZFP_. + Also, be sure to see :ref:`endian-issues`. +* After you have setup ``Config``, simply run ``make`` and it will build the ZFP_ library placing the library in a ``lib`` sub-directory and the necessary include files in ``inc[lude]`` sub-directory. * For more information and details, please see the `ZFP README `_. ^^^^^^^^^^^^^^^ Compiling HDF5_ ^^^^^^^^^^^^^^^ -* If you want to be able to run the fortran tests for this filter, HDF5_ must be - configured with *both* the ``--enable-fortran`` and ``--enable-fortran2003`` - configuration switches. Otherwise, any vanilla installation of HDF5_ is acceptable. +* If you want to be able to run the fortran tests for this filter, HDF5_ must be configured with *both* the ``--enable-fortran`` and ``--enable-fortran2003`` configuration switches. + Otherwise, any vanilla installation of HDF5_ is acceptable. -* The Fortran interface to this filter *requires* a Fortran 2003 compiler - because it uses - `ISO_C_BINDING `_ - to define the Fortran interface. +* The Fortran interface to this filter *requires* a Fortran 2003 compiler because it uses `ISO_C_BINDING `_ to define the Fortran interface. -* If you are using HDF5-1.12 and wish to use the filter as a *library* (see :ref:`plugin-vs-library`), - you may need configure HDF5 with ``--disable-memory-alloc-sanity-check`` to work - around a memory management issue in HDF5. +* If you are using HDF5-1.12 and wish to use the filter as a *library* (see :ref:`plugin-vs-library`), you may need configure HDF5 with ``--disable-memory-alloc-sanity-check`` to work around a memory management issue in HDF5. ------------------ +^^^^^^^^^^^^^^^^^ Compiling H5Z-ZFP ------------------ +^^^^^^^^^^^^^^^^^ -H5Z-ZFP_ is designed to be compiled both as a standalone HDF5_ *plugin* and as a separate -*library* an application can explicitly link. See :ref:`plugin-vs-library`. +H5Z-ZFP_ is designed to be compiled both as a standalone HDF5_ *plugin* and as a separate *library* an application can explicitly link. See :ref:`plugin-vs-library`. Once you have installed the prerequisites, you can compile H5Z-ZFP_ using a command-line... @@ -99,14 +70,12 @@ Once you have installed the prerequisites, you can compile H5Z-ZFP_ using a comm ZFP_HOME= HDF5_HOME= \ PREFIX= -where ```` is a directory containing ZFP_ ``inc[lude]`` and ``lib`` dirs and -```` is a directory containing HDF5_ ``include`` and ``lib`` dirs. -If you don't specify a C compiler, it will try to guess one from your path. Fortran -compilation is optional. If you do not specify a Fortran compiler, it will not attempt -to build the Fortran interface. However, if the variable ``FC`` is already defined in -your enviornment (as in Spack_ for example), then H5Z-ZFP_ will attempt to build Fortran. -If this is not desired, the solution is to pass an *empty* ``FC`` on the make command -line as in... +where ```` is a directory containing ZFP_ ``inc[lude]`` and ``lib`` dirs and ```` is a directory containing HDF5_ ``include`` and ``lib`` dirs. +If you don't specify a C compiler, it will try to guess one from your path. +Fortran compilation is optional. +If you do not specify a Fortran compiler, it will not attempt to build the Fortran interface. +However, if the variable ``FC`` is already defined in your enviornment (as in Spack_ for example), then H5Z-ZFP_ will attempt to build Fortran. +If this is not desired, the solution is to pass an *empty* ``FC`` on the make command line as in... :: @@ -115,16 +84,12 @@ line as in... PREFIX= -The Makefile uses GNU Make syntax and is designed to work on OSX and -Linux. The filter has been tested on gcc, clang, xlc, icc and pgcc compilers -and checked with valgrind. +The Makefile uses GNU Make syntax and is designed to work on OSX and Linux. The filter has been tested on gcc, clang, xlc, icc and pgcc compilers and checked with valgrind. -The command ``make help`` will print useful information -about various make targets and variables. ``make check`` will compile everything -and run a handful of tests. +The command ``make help`` will print useful information about various make targets and variables. ``make check`` will compile everything and run a handful of tests. -If you don't specify a ``PREFIX``, it will install to ``./install``. The installed -package will look like... +If you don't specify a ``PREFIX``, it will install to ``./install``. +The installed package will look like... :: @@ -137,6 +102,136 @@ where ``$(PREFIX)`` resolves to whatever the full path of the installation is. To use the installed filter as an HDF5_ *plugin*, you would specify, for example, ``setenv HDF5_PLUGIN_PATH $(PREFIX)/plugin`` +.. _ceemake: + +-------------------- +Installing via CMake +-------------------- + +It is possible to build the H5Z-ZFP_ filter using the CMake_ build system. +To use CMake_ for H5Z-ZFP_, it is necessary to have also built ZFP_ with CMake. +This is necessary to get the correct dependencies from ZFP_. +For example, it is possible to build ZFP_ with OpenMP support. +The resulting CMake_ config files of ZFP_ build will make sure that this OpenMP dependency is correctly propagated to the build of H5Z-ZFP_ filter. +However, for HDF5_ it is not necessary to build it with its CMake_ build system but it is strongly recommended. + +ZFP_ must have been :ref:`configured ` with ``BIT_STREAM_WORD_TYPE`` of ``uint8`` as described above. + +Similar as for the Makefile installation, the CMake_ build system is designed such it compiles both the standalone HDF5_ *plugin* and a separate *library* an application can explicitly link. See :ref:`plugin-vs-library` + +Once both HDF5_ and ZFP_ have been installed, H5Z-ZFP_ can be compiled using a command=line... + +:: + + export HDF5_DIR= + export ZFP_DIR= + CC= FC= cmake -DCMAKE_INSTALL_PREFIX= + +where ```` is a directory containing ``zfp-config.cmake`` and ```` is a directory containing HDF5_ ``include`` and ``lib`` directories. +Furthermore, ``src-dir`` is the directory where the H5Z-ZFP_ source is located and ``path-to-install`` is the directory in which the resulting *plugin* and *library* will be installed. +Once ``cmake`` has finished successfully, you can build and install the filter using the command... + +:: + + make install + +This ``cmake`` and ``make`` combination builds both the C and Fortran interface. +In the case you want to specify the ```` and ``>`` via command-line to CMake_, the command looks like this... + +:: + + CC= FC= cmake -DCMAKE_INSTALL_PREFIX= + -DCMAKE_PREFIX_PATH=";" + +.. note:: + + The double quotes in the CMAKE_PREFIX_PATH expression are necessary to make sure that semicolon is interpreted as a semicolon instead of a new command. + +It is possible to build the filter without the Fortran interface. This is done as follows... + +:: + + export HDF5_DIR= + export ZFP_DIR= + CC= cmake -DCMAKE_INSTALL_PREFIX= -DFORTRAN_INTERFACE:BOOL=OFF + +followed by the same make command... + +:: + + make install + +------------------------------------------- +Including H5Z-ZFP filter in a CMake project +------------------------------------------- + +Suppose you have built the H5Z-ZFP_ filter using the CMake_ build system and installed it in ````. +To include it in another CMake_ project is done using the following steps. First edit the ``CMakeLists.txt`` +by adding the following two lines... + +:: + + cmake_policy(SET CMP0028 NEW) # Double colon in target name means ALIAS or IMPORTED target. + ... + set(H5Z_ZFP_USE_STATIC_LIBS OFF) + find_package(H5Z_ZFP 1.0.1 CONFIG) + ... + target_link_libraries( h5z_zfp::h5z_zfp) + ... + +where ```` in the target within the CMake_ project. +This could be, for example, an executable or library. +Furthermore, check if the ``cmake`` version is equal or greater than 3.9. +Next, you need to make sure that the filter can be found by CMake_, followed by ``cmake`` itself and ``make``... + +:: + + export H5Z_ZFP_DIR= + CC= cmake -DCMAKE_INSTALL_PREFIX= + make install + +The ``cmake`` command itself could be different depending on the CMake_ project you have created. +If you want to make use of the H5Z-ZFP_ *library* instead of the plugin, change cmake variable ``H5Z_ZFP_USE_STATIC_LIBS`` to ``ON`` and build the project. + +.. _spack2: + +--------------------- +Installing via Spack_ +--------------------- +If you already have experience with Spack_, one way to install H5Z-ZFP_ is to use the command ``spack install h5z-zfp``. +If you do not have Spack_ installed, it is easy to install. +Assuming you are working in a Bash shell...:: + + git clone https://github.com/llnl/spack.git + . spack/share/spack/setup-env.sh + spack install h5z-zfp + +If you are using a version of Spack_ very much older than the release of H5Z-ZFP_ you intend to use, you may have to *pin* various versions of H5Z-ZFP_, ZFP_ and/or HDF5_. +This is done by using Spack_'s ``@`` modifier to specify versions. +For example, to *pin* the version of the ZFP_ library to 0.5.5, the Spack_ command would look like:: + + spack install h5z-zfp ^zfp@0.5.5 + +To use the ``develop`` version of H5Z-ZFP_ with version 1.10.6 of HDF5_ :: + + spack install h5z-zfp@develop ^hdf5@1.10.6 + +By default, H5Z-ZFP_ will attempt to build with Fortran support which requires a Fortran compiler. +If you wish to exclude support for Fortran, use the command:: + + spack install h5z-zfp ~fortran + +.. note:: + + These commands will build H5Z-ZFP_ **and** all of its dependencies including the HDF5_ library *as well as a number of other dependencies you may not initially expect*. + Be patient and let the build complete. + It may take more than an hour. + +In addition, by default, Spack_ installs packages to directory *hashes within* the cloned Spack_ repository's directory tree, ``$spack/opt/spack``. +You can find the resulting installed HDF5_ library with the command ``spack find -vp hdf5`` and the resulting H5Z-ZFP_ plugin installation with the command ``spack find -vp h5z-zfp``. +If you wish to exercise more control over where Spack_ installs things, have a look at +`configuring Spack `_ + -------------------------------- H5Z-ZFP Source Code Organization -------------------------------- @@ -145,21 +240,14 @@ The source code is in two separate directories * ``src`` includes the ZFP_ filter and a few header files - * ``H5Zzfp_plugin.h`` is an optional header file applications *may* wish - to include because it contains several convenient macros for easily - controlling various compression modes of the ZFP_ library (*rate*, - *precision*, *accuracy*, *expert*) via the :ref:`generic-interface`. - * ``H5Zzfp_props.h`` is a header file that contains functions to control the - filter using *temporary* :ref:`properties-interface`. Fortran callers are - *required* to use this interface. - * ``H5Zzfp_lib.h`` is a header file for applications that wish to use the filter - explicitly as a library rather than a plugin. - * ``H5Zzfp.h`` is an *all-of-the-above* header file for applications that don't - care too much about separating out the above functionalities. - - * ``test`` includes various tests. In particular ``test_write.c`` includes examples - of using both the :ref:`generic-interface` and :ref:`properties-interface`. In - addition, there is an example of how to use the filter from Fortran in ``test_rw_fortran.F90``. + * ``H5Zzfp_plugin.h`` is an optional header file applications *may* wish to include because it contains several convenient macros for easily controlling various compression modes of the ZFP_ library (*rate*, *precision*, *accuracy*, *expert*) via the :ref:`generic-interface`. + * ``H5Zzfp_props.h`` is a header file that contains functions to control the filter using *temporary* :ref:`properties-interface`. + Fortran callers are *required* to use this interface. + * ``H5Zzfp_lib.h`` is a header file for applications that wish to use the filter explicitly as a library rather than a plugin. + * ``H5Zzfp.h`` is an *all-of-the-above* header file for applications that don't care too much about separating out the above functionalities. + + * ``test`` includes various tests. In particular ``test_write.c`` includes examples of using both the :ref:`generic-interface` and :ref:`properties-interface`. + In addition, there is an example of how to use the filter from Fortran in ``test_rw_fortran.F90``. ---------------- Silo Integration diff --git a/docs/interfaces.rst b/docs/interfaces.rst index 4069290..46f3352 100644 --- a/docs/interfaces.rst +++ b/docs/interfaces.rst @@ -2,32 +2,18 @@ Interfaces ========== -There are two interfaces to control the filter. One uses HDF5_'s -*generic* interface via an array of ``unsigned int cd_values`` as is used -in `H5Pset_filter() `__. The other -uses HDF5_ `property lists `__ -added to the `dataset creation property list `__ -used when the dataset to be compressed is being created. You can find examples of writing -HDF5_ data using both the -`generic `_ -and -`properties `__ -interfaces in -`test_write.c `_. - -The filter itself supports either interface. The filter also supports all of the -standard ZFP_ controls for affecting compression including *rate*, *precision*, -*accuracy*, *expert* and *reversible* modes. For more information and details about -these modes of controlling ZFP_ compression, please see the -`ZFP README `_. - -Finally, you should *not* attempt to combine the ZFP_ filter with any other -*byte order altering* filter such as, for example, HDF5_'s shuffle filter. -Space-performance will be ruined. This is in contrast to HDF5_'s -`deflate `__ -filter which often performs *better* when used in conjunction with the -`shuffle `__ -filter. +There are two interfaces to control the filter. +One uses HDF5_'s *generic* interface via an array of ``unsigned int cd_values`` as is used in `H5Pset_filter() `__. +The other uses HDF5_ `property lists `__ added to the `dataset creation property list `__ used when the dataset to be compressed is being created. +You can find examples of writing HDF5_ data using both the `generic `_ and `properties `__ interfaces in `test_write.c `_. + +The filter itself supports either interface. The filter also supports all of the standard ZFP_ controls for affecting compression including *rate*, *precision*, *accuracy*, *expert* and *reversible* modes. +For more information and details about these modes of controlling ZFP_ compression, please see the `ZFP README `_. + +Finally, you should *not* attempt to combine the ZFP_ filter with any other *byte order altering* filter such as, for example, HDF5_'s shuffle filter. +Space-performance will be ruined. +This is in contrast to HDF5_'s `deflate `__ filter which often performs *better* when used in conjunction with the `shuffle `__ filter. +To understand why, see the description of :ref:`endian issues `. .. _generic-interface: @@ -35,12 +21,9 @@ filter. Generic Interface ----------------- -The generic interface is the only means of controlling the H5Z-ZFP_ filter when it -is used as a -`dynamically loaded HDF5 plugin `_. +The generic interface is the only means of controlling the H5Z-ZFP_ filter when it is used as a `dynamically loaded HDF5 plugin `_. -For the generic interface, the following CPP macros are defined in -the ``H5Zzfp_plugin.h`` header file:: +For the generic interface, the following CPP macros are defined in the ``H5Zzfp_plugin.h`` header file:: H5Pset_zfp_rate_cdata(double rate, size_t cd_nelmts, unsigned int *cd_vals); H5Pset_zfp_precision_cdata(unsigned int prec, size_t cd_nelmts, unsigned int *cd_vals); @@ -50,14 +33,10 @@ the ``H5Zzfp_plugin.h`` header file:: size_t cd_nelmts, unsigned int *cd_vals); H5Pset_zfp_reversible_cdata(size_t cd_nelmts, unsigned int *cd_vals); -These macros utilize *type punning* to store the relevant ZFP_ parameters into a -sufficiently large array (>=6) of ``unsigned int cd_values``. It is up to -the caller to then call -`H5Pset_filter() `__ -with the array of cd_values constructed by one of these macros. +These macros utilize *type punning* to store the relevant ZFP_ parameters into a sufficiently large array (>=6) of ``unsigned int cd_values``. +It is up to the caller to then call `H5Pset_filter() `__ with the array of cd_values constructed by one of these macros. -Here is example code from -`test_write.c `_... +Here is example code from `test_write.c `_... .. literalinclude:: ../test/test_write.c :language: c @@ -65,11 +44,10 @@ Here is example code from :start-after: setup zfp filter via generic (cd_values) interface :end-before: #else -However, these macros are only a convenience. You do not **need** the -``H5Zzfp_plugin.h`` header file if you want to avoid using it. But, you are then -responsible for setting up the ``cd_values`` array correctly for the -filter. For reference, the ``cd_values`` array for this ZFP_ filter is -defined like so... +However, these macros are only a convenience. +You do not **need** the ``H5Zzfp_plugin.h`` header file if you want to avoid using it. +But, you are then responsible for setting up the ``cd_values`` array correctly for the filter. +For reference, the ``cd_values`` array for this ZFP_ filter is defined like so... +-----------+---------------------------------------------------------+ | | cd_values index | @@ -89,13 +67,9 @@ defined like so... A/B are high/low 32-bit words of a double. -Note that the cd_values used in the generic interface to ``H5Pset_filter()`` -are **not the same** cd_values ultimately stored to the HDF5_ dataset header -for a compressed dataset. The values are transformed in the set_local -method to use ZFP_'s internal routines for 'meta' and 'mode' data. So, -don't make the mistake of examining the values you find in a file and -think you can use those same values, for example, in an invokation of -h5repack. +Note that the cd_values used in the generic interface to ``H5Pset_filter()`` are **not the same** cd_values ultimately stored to the HDF5_ dataset header for a compressed dataset. +The values are transformed in the set_local method to use ZFP_'s internal routines for 'meta' and 'mode' data. +So, don't make the mistake of examining the values you find in a file and think you can use those same values, for example, in an invokation of h5repack. .. _properties-interface: @@ -103,8 +77,7 @@ h5repack. Properties Interface -------------------- -For the properties interface, the following functions are defined in -the ``H5Zzfp_props.h`` header file:: +For the properties interface, the following functions are defined in the ``H5Zzfp_props.h`` header file:: herr_t H5Pset_zfp_rate(hid_t dcpl_id, double rate); herr_t H5Pset_zfp_precision(hid_t dcpl_id, unsigned int prec); @@ -114,15 +87,11 @@ the ``H5Zzfp_props.h`` header file:: unsigned int maxprec, int minexp); herr_t H5Pset_zfp_reversible(hid_t dcpl_id); -These functions take a dataset creation property list, ``hid_t dcp_lid`` and -create temporary HDF5_ property -list entries to control the ZFP_ filter. Calling any of these functions -removes the effects of any previous call to any one of these functions. -In addition, calling any one of these functions also has the effect of -adding the filter to the pipeline. +These functions take a dataset creation property list, ``hid_t dcp_lid`` and create temporary HDF5_ property list entries to control the ZFP_ filter. +Calling any of these functions removes the effects of any previous call to any one of these functions. +In addition, calling any one of these functions also has the effect of adding the filter to the pipeline. -Here is example code from -`test_write.c `_... +Here is example code from `test_write.c `_... .. literalinclude:: ../test/test_write.c :language: c @@ -131,22 +100,19 @@ Here is example code from :end-before: #endif The properties interface is more type-safe than the generic interface. -However, there is no way for the implementation of the properties interface -to reside within the filter plugin itself. The properties interface requires that the caller link -with with the filter as a *library*, ``libh5zzfp.a``. The generic -interface does not require this. +However, there is no way for the implementation of the properties interface to reside within the filter plugin itself. +The properties interface requires that the caller link with with the filter as a *library*, ``libh5zzfp.a``. +The generic interface does not require this. -Note that either interface can be used whether the -filter is used as a plugin or as a library. The difference -is whether the application calls ``H5Z_zfp_initialize()`` or not. +Note that either interface can be used whether the filter is used as a plugin or as a library. +The difference is whether the application calls ``H5Z_zfp_initialize()`` or not. ----------------- Fortran Interface ----------------- -Fortran equivalents for both the properties and generic interfaces, described above, -has been added by Scot Breitenfeld of the HDF5_ group. The code that -implements the Fortran interfaces is in the file ``H5Zzfp_props_f.F90``. +Fortran equivalents for both the properties and generic interfaces, described above, has been added by Scot Breitenfeld of the HDF5_ group. +The code that implements the Fortran interfaces is in the file ``H5Zzfp_props_f.F90``. An example of its use is in ``test/test_rw_fortran.F90``. .. _plugin-vs-library: @@ -155,34 +121,21 @@ An example of its use is in ``test/test_rw_fortran.F90``. Plugin vs. Library Operation ---------------------------- -The filter is designed to be compiled for use as both a standalone HDF5_ -`dynamically loaded HDF5 plugin `_ -and as an explicitly linked *library*. -When it is used as a plugin, it is a best practice to link the ZFP_ library -into the plugin dynamic/shared object as a *static* library. Why? In so doing, -we ensure that all ZFP_ public namespace symbols remain *confined* to the plugin -so as not to interfere with any application that may be directly explicitly linking -to the ZFP_ library for other reasons. - -All HDF5_ applications are *required* -to *find* the plugin dynamic library (named ``lib*.{so,dylib}``) -in a directory specified by the enviornment -variable, ``HDF5_PLUGIN_PATH``. Currently, the HDF5 library offers -no mechanism for applications themselves to have pre-programmed -paths in which to search for a plugin. Applications are -then always vulnerable to an incorrectly specified or unspecified ``HDF5_PLUGIN_PATH`` -environment variable. - -However, the plugin can also be used explicitly as a *library*. In this case, -**do** **not** specify the ``HDF5_PLUGIN_PATH`` enviornment variable and instead -have the application link to ``libH5Zzfp.a`` in the ``lib`` dir of the installation. +The filter is designed to be compiled for use as both a standalone HDF5_ `dynamically loaded HDF5 plugin `_ and as an explicitly linked *library*. +When it is used as a plugin, it is a best practice to link the ZFP_ library into the plugin dynamic/shared object as a *static* library. +Why? In so doing, we ensure that all ZFP_ public namespace symbols remain *confined* to the plugin so as not to interfere with any application that may be directly explicitly linking to the ZFP_ library for other reasons. + +All HDF5_ applications are *required* to *find* the plugin dynamic library (named ``lib*.{so,dylib}``) in a directory specified by the enviornment variable, ``HDF5_PLUGIN_PATH``. +Currently, the HDF5 library offers no mechanism for applications themselves to have pre-programmed paths in which to search for a plugin. +Applications are then always vulnerable to an incorrectly specified or unspecified ``HDF5_PLUGIN_PATH`` environment variable. + +However, the plugin can also be used explicitly as a *library*. +In this case, **do** **not** specify the ``HDF5_PLUGIN_PATH`` enviornment variable and instead have the application link to ``libH5Zzfp.a`` in the ``lib`` dir of the installation. Instead two initialization and finalization routines are defined:: int H5Z_zfp_initialize(void); int H5Z_zfp_finalize(void); These functions are defined in the ``H5Zzfp_lib.h`` header file. -Any applications that wish to use the filter as a *library* are required to call -the initialization routine, ``H5Z_zfp_initialize()`` before the filter can be -referenced. In addition, to free up resources used by the filter, applications may -call ``H5Z_zfp_finalize()`` when they are done using the filter. +Any applications that wish to use the filter as a *library* are required to call the initialization routine, ``H5Z_zfp_initialize()`` before the filter can be referenced. +In addition, to free up resources used by the filter, applications may call ``H5Z_zfp_finalize()`` when they are done using the filter. diff --git a/src/H5Zzfp_version.h b/src/H5Zzfp_version.h index 9bd327d..a79179c 100644 --- a/src/H5Zzfp_version.h +++ b/src/H5Zzfp_version.h @@ -5,7 +5,7 @@ #define H5Z_FILTER_ZFP_VERSION_MAJOR 1 #define H5Z_FILTER_ZFP_VERSION_MINOR 1 -#define H5Z_FILTER_ZFP_VERSION_PATCH 0 +#define H5Z_FILTER_ZFP_VERSION_PATCH 1 #define H5Z_ZFP_MODE_RATE 1 #define H5Z_ZFP_MODE_PRECISION 2