Merge pull request #140 from mulimoen/feature/dcpl-2021

The DCPL branch

Author: mulimoen

.github/workflows/ci.yml

@@ -162,9 +162,6 @@ jobs:
         run: |
           [ "${{matrix.mpi}}" != "serial" ] && FEATURES=mpio
           cargo test -vv --features="$FEATURES"
-      - name: Test const generics
-        if: matrix.rust == 'nightly'
-        run: cargo test -p hdf5-types --features hdf5-types/const_generics
 
   msi:
     name: msi
@@ -209,6 +206,22 @@ jobs:
       - name: Build and test all crates
         run: cargo test -vv
 
+  msrv:
+    name: Minimal Supported Rust Version
+    runs-on: ubuntu-18.04
+    strategy:
+      fail-fast: false
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+        with: {submodules: true}
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
+        with: {toolchain: 1.51, profile: minimal, override: true}
+      - name: Build and test all crates
+        run:
+          cargo test --workspace -vv --features=hdf5-sys/static --exclude=hdf5-derive
+
   wine:
     name: wine
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -2,6 +2,65 @@
 
 ## Unreleased
 
+### Added
+
+- Complete rewrite of `DatasetBuilder`; dataset creation API is now different and not
+  backwards-compatible (however, it integrates all of the new features and is more
+  flexible and powerful). It is now possible to create and write the datasets in one step.
+  Refer to the API docs for full reference.
+- New `Extents` type matching HDF5 extents types: null (no elements), scalar, simple (fixed
+  dimensionality); it is used to query and specify shapes of datasets. Extents objects
+  are convertible from numbers and also tuples, slices and vectors of indices - all of
+  which can be used whenever extents are required (e.g., when creating a new dataset).
+- New `Selection` type and the API around it closely matching HDF5 selection API - this
+  includes 'all' selection, point-wise selection and hyperslab selection (only 'regular'
+  hyperslabs are supported - that is, hyperslabs that can be represented as a single
+  multi-dimensional box some of whose dimensions may be infinite). Selection objects
+  are convertible from numbers and ranges and tuples and arrays of numbers and ranges;
+  one can also use `s!` macro from `ndarray` crate. Selections can be provided when
+  reading and writing slices.
+- LZF / Blosc filters have been added, they have to be enabled by `"lzf"` / `"blosc"`
+  cargo features and depend on `lzf-sys` / `blosc-sys` crates respectively. Blosc filter
+  is a meta-filter providing multi-threaded access to the best-in-class compression codecs
+  like Zstd and LZ4 and is recommended to use as a default when compression is critical.
+- New `Filter` type to unify all of the filters API; if LZF / Blosc filters are enabled,
+  this enum also contains the corresponding variants. It is now also possible to provide
+  user-defined filters with custom filter IDs and parameters.
+- Dataset creation property list (DCPL) API is now supported; this provides access to all
+  of the properties that can be specified at dataset creation time (e.g., layout, chunking,
+  fill values, external file linking, virtual maps, object time tracking, attribute
+  creation order, and a few other low-level settings).
+- As part of DCPL change, virtual dataset maps (VDS API in HDF5 1.10+) are now supported.
+- Link creation property list (LCPL) API is now also wrapped.
+- File creation property list (FCPL) API has been extended to include a few previously
+  missing properties (object time tracking, attribute creation order and few other
+  low-level settings).
+- Added `h5-alloc` feature to `hdf5-types` crate - uses the HDF5 allocator for
+  varlen types and dynamic values. This may be necessary on platforms where different
+  allocators may be used in different libraries (e.g. dynamic libraries on Windows),
+  or if `libhdf5` is compiled with the memchecker option enabled. This option is
+  force-enabled by default if using a dll version of the library on Windows.
+- New `DynValue` type which represents a dynamic self-describing HDF5 object that
+  also knows how to deallocate itself; it supports all of the HDF5 types including
+  compound types, strings and arrays.
+- Added methods to `Dataset`: `layout`, `dapl`, `access_plist`, `dcpl`, `create_plist`.
+  
+ ### Changed
+  
+- `Dataspace` type has been reworked and can be now constructed from an extents object 
+  and sliced with a selection object.
+- `Dataset::fill_value` now returns an object of the newly added `DynValue` type; this
+  object is self-describing and knows how to free itself.
+- Automatic chunking now uses a fill-from-back approach instead of the previously
+  used method which is used in `h5py`.
+- Removed `Filters` type (there's now `Filter` that represents a single filter).
+- `write_slice`, `read_slice`, `read_slice_1d`, `read_slice_2d` now take any object
+  convertible to `Selection` (instead of `SliceInfo`).
+- `Dataset::chunks` has been renamed to `Dataset::chunk`
+- Const generics support (MSRV 1.51): `hdf5-types` now uses const generics for array types,
+  allowing fixed-size arrays of arbitrary sizes.
+- The `ndarray` dependency has been updated to `0.15`.
+
 ## 0.7.1
 
 ### Added

Cargo.toml

@@ -14,6 +14,8 @@ edition = "2018"
 [features]
 default = []
 mpio = ["mpi-sys", "hdf5-sys/mpio"]
+lzf = ["lzf-sys", "errno"]
+blosc = ["blosc-sys"]
 
 [workspace]
 members = [".", "hdf5-types", "hdf5-derive", "hdf5-sys", "hdf5-src"]
@@ -24,21 +26,25 @@ bitflags = "1.2"
 lazy_static = "1.4"
 libc = "0.2"
 parking_lot = "0.11"
-ndarray = ">=0.13,<0.15"
+ndarray = "0.15"
 num-integer = "0.1"
 num-traits = "0.2"
 mpi-sys = { version = "0.1", optional = true }
+errno = { version = "0.2", optional = true }
 hdf5-sys = { path = "hdf5-sys", version = "0.7.1" }  # !V
 hdf5-types = { path = "hdf5-types", version = "0.7.1" }  # !V
 hdf5-derive = { path = "hdf5-derive", version = "0.7.1" }  # !V
+blosc-sys = { version = "0.1", package = "blosc-src", optional = true }
+lzf-sys = { version = "0.1", optional = true }
+cfg-if = "1.0"
 
 [dev-dependencies]
 paste = "1.0"
-pretty_assertions = "0.6"
+pretty_assertions = "0.7"
 rand = { version = "0.8", features = ["small_rng"] }
 regex = "1.3"
 scopeguard = "1.0"
 tempfile = "3.2"
 
 [package.metadata.docs.rs]
-features = ["hdf5-sys/static", "hdf5-sys/zlib"]
+features = ["hdf5-sys/static", "hdf5-sys/zlib", "blosc", "lzf"]

README.md

@@ -49,10 +49,10 @@ fn main() -> hdf5::Result<()> {
     {
         // write
         let file = hdf5::File::create("pixels.h5")?;
-        let colors = file.new_dataset::<Color>().create("colors", 2)?;
+        let colors = file.new_dataset::<Color>().shape(2).create("colors")?;
         colors.write(&[RED, BLUE])?;
         let group = file.create_group("dir")?;
-        let pixels = group.new_dataset::<Pixel>().create("pixels", (2, 2))?;
+        let pixels = group.new_dataset::<Pixel>().shape((2, 2)).create("pixels")?;
         pixels.write(&arr2(&[
             [Pixel { xy: (1, 2), color: RED }, Pixel { xy: (3, 4), color: BLUE }],
             [Pixel { xy: (5, 6), color: GREEN }, Pixel { xy: (7, 8), color: RED }],

examples/simple.rs

@@ -23,10 +23,10 @@ fn main() -> hdf5::Result<()> {
     {
         // write
         let file = hdf5::File::create("pixels.h5")?;
-        let colors = file.new_dataset::<Color>().create("colors", 2)?;
+        let colors = file.new_dataset::<Color>().shape(2).create("colors")?;
         colors.write(&[RED, BLUE])?;
         let group = file.create_group("dir")?;
-        let pixels = group.new_dataset::<Pixel>().create("pixels", (2, 2))?;
+        let pixels = group.new_dataset::<Pixel>().shape((2, 2)).create("pixels")?;
         pixels.write(&arr2(&[
             [Pixel { xy: (1, 2), color: RED }, Pixel { xy: (3, 4), color: BLUE }],
             [Pixel { xy: (5, 6), color: GREEN }, Pixel { xy: (7, 8), color: RED }],

hdf5-sys/Cargo.toml

@@ -29,7 +29,7 @@ static = ["hdf5-src"]
 deprecated = ["hdf5-src/deprecated"]
 
 [build-dependencies]
-libloading = "0.6"
+libloading = "0.7"
 regex = { version = "1.3", features = ["std"] }
 
 [target.'cfg(all(unix, not(target_os = "macos")))'.build-dependencies]

hdf5-sys/build.rs

@@ -85,7 +85,7 @@ impl Display for RuntimeError {
 
 #[allow(non_snake_case, non_camel_case_types)]
 fn get_runtime_version_single<P: AsRef<Path>>(path: P) -> Result<Version, Box<dyn Error>> {
-    let lib = libloading::Library::new(path.as_ref())?;
+    let lib = unsafe { libloading::Library::new(path.as_ref()) }?;
 
     type H5open_t = unsafe extern "C" fn() -> c_int;
     let H5open = unsafe { lib.get::<H5open_t>(b"H5open")? };
@@ -124,28 +124,26 @@ fn validate_runtime_version(config: &Config) {
     }
     for link_path in &link_paths {
         if let Ok(paths) = fs::read_dir(link_path) {
-            for path in paths {
-                if let Ok(path) = path {
-                    let path = path.path();
-                    if let Some(filename) = path.file_name() {
-                        let filename = filename.to_str().unwrap_or("");
-                        if path.is_file() && libfiles.contains(&filename) {
-                            println!("Attempting to load: {:?}", path);
-                            match get_runtime_version_single(&path) {
-                                Ok(version) => {
-                                    println!("    => runtime version = {:?}", version);
-                                    if version == config.header.version {
-                                        println!("HDF5 library runtime version matches headers.");
-                                        return;
-                                    }
-                                    panic!(
-                                        "Invalid HDF5 runtime version (expected: {:?}).",
-                                        config.header.version
-                                    );
-                                }
-                                Err(err) => {
-                                    println!("    => {}", err);
+            for path in paths.flatten() {
+                let path = path.path();
+                if let Some(filename) = path.file_name() {
+                    let filename = filename.to_str().unwrap_or("");
+                    if path.is_file() && libfiles.contains(&filename) {
+                        println!("Attempting to load: {:?}", path);
+                        match get_runtime_version_single(&path) {
+                            Ok(version) => {
+                                println!("    => runtime version = {:?}", version);
+                                if version == config.header.version {
+                                    println!("HDF5 library runtime version matches headers.");
+                                    return;
                                 }
+                                panic!(
+                                    "Invalid HDF5 runtime version (expected: {:?}).",
+                                    config.header.version
+                                );
+                            }
+                            Err(err) => {
+                                println!("    => {}", err);
                             }
                         }
                     }

hdf5-sys/src/lib.rs

@@ -2,6 +2,7 @@
 #![cfg_attr(feature = "cargo-clippy", allow(clippy::unreadable_literal))]
 #![cfg_attr(feature = "cargo-clippy", allow(clippy::missing_safety_doc))]
 #![cfg_attr(feature = "cargo-clippy", allow(clippy::cognitive_complexity))]
+#![cfg_attr(feature = "cargo-clippy", allow(clippy::upper_case_acronyms))]
 
 macro_rules! extern_static {
     ($dest:ident, $src:ident) => {

hdf5-types/Cargo.toml

@@ -8,13 +8,17 @@ description = "Native Rust equivalents of HDF5 types."
 repository = "https://github.com/aldanor/hdf5-rust"
 homepage = "https://github.com/aldanor/hdf5-rust"
 edition = "2018"
+build = "build.rs"
 
 [features]
-const_generics = []
+h5-alloc = []
 
 [dependencies]
 ascii = "1.0"
 libc = "0.2"
+hdf5-sys = { version = "0.7.1", path = "../hdf5-sys" }  # !V
+cfg-if = "1.0.0"
 
 [dev-dependencies]
 quickcheck = { version = "1.0", default-features = false }
+unindent = "0.1"

hdf5-types/build.rs

@@ -0,0 +1,6 @@
+fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+    if std::env::var_os("DEP_HDF5_MSVC_DLL_INDIRECTION").is_some() {
+        println!("cargo:rustc-cfg=windows_dll");
+    }
+}

hdf5-types/src/array.rs

@@ -14,76 +14,22 @@ pub unsafe trait Array: 'static {
     fn capacity() -> usize;
 }
 
-#[cfg(not(feature = "const_generics"))]
-mod impl_array {
-    use super::*;
-
-    macro_rules! impl_array {
-        () => ();
-
-        ($n:expr, $($ns:expr,)*) => (
-            unsafe impl<T: 'static> Array for [T; $n] {
-                type Item = T;
-
-                #[inline(always)]
-                fn as_ptr(&self) -> *const T {
-                    self as *const _ as *const _
-                }
-
-                #[inline(always)]
-                fn as_mut_ptr(&mut self) -> *mut T {
-                    self as *mut _ as *mut _
-                }
-
-                #[inline(always)]
-                fn capacity() -> usize {
-                    $n
-                }
-            }
+unsafe impl<T: 'static, const N: usize> Array for [T; N] {
+    type Item = T;
 
-            impl_array!($($ns,)*);
-        );
+    #[inline(always)]
+    fn as_ptr(&self) -> *const T {
+        self as *const _
     }
 
-    impl_array!(
-        0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
-        25, 26, 27, 28, 29, 30, 31,
-    );
-    impl_array!(
-        32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54,
-        55, 56, 57, 58, 59, 60, 61, 62, 63,
-    );
-    impl_array!(
-        64, 70, 72, 80, 90, 96, 100, 110, 120, 128, 130, 140, 150, 160, 170, 180, 190, 192, 200,
-        210, 220, 224, 230, 240, 250,
-    );
-    impl_array!(
-        256, 300, 365, 366, 384, 400, 500, 512, 600, 700, 768, 800, 900, 1000, 1024, 2048, 4096,
-        8192, 16384, 32768,
-    );
-}
-
-#[cfg(feature = "const_generics")]
-mod impl_array {
-    use super::*;
-
-    unsafe impl<T: 'static, const N: usize> Array for [T; N] {
-        type Item = T;
-
-        #[inline(always)]
-        fn as_ptr(&self) -> *const T {
-            self as *const _
-        }
-
-        #[inline(always)]
-        fn as_mut_ptr(&mut self) -> *mut T {
-            self as *mut _ as *mut _
-        }
+    #[inline(always)]
+    fn as_mut_ptr(&mut self) -> *mut T {
+        self as *mut _ as *mut _
+    }
 
-        #[inline(always)]
-        fn capacity() -> usize {
-            N
-        }
+    #[inline(always)]
+    fn capacity() -> usize {
+        N
     }
 }
 
@@ -97,7 +43,7 @@ pub struct VarLenArray<T: Copy> {
 impl<T: Copy> VarLenArray<T> {
     pub unsafe fn from_parts(p: *const T, len: usize) -> VarLenArray<T> {
         let (len, ptr) = if !p.is_null() && len != 0 {
-            let dst = libc::malloc(len * mem::size_of::<T>());
+            let dst = crate::malloc(len * mem::size_of::<T>());
             ptr::copy_nonoverlapping(p, dst as *mut _, len);
             (len, dst)
         } else {
@@ -136,7 +82,7 @@ impl<T: Copy> Drop for VarLenArray<T> {
     fn drop(&mut self) {
         if !self.ptr.is_null() {
             unsafe {
-                libc::free(self.ptr as *mut _);
+                crate::free(self.ptr as *mut _);
             }
             self.ptr = ptr::null();
             if self.len != 0 {
@@ -173,10 +119,10 @@ impl<'a, T: Copy> From<&'a [T]> for VarLenArray<T> {
     }
 }
 
-impl<T: Copy> Into<Vec<T>> for VarLenArray<T> {
+impl<T: Copy> From<VarLenArray<T>> for Vec<T> {
     #[inline]
-    fn into(self) -> Vec<T> {
-        self.iter().cloned().collect()
+    fn from(v: VarLenArray<T>) -> Self {
+        v.iter().cloned().collect()
     }
 }