Add filespace management API calls (#924)

* Add low level API bindings for file space

* Add helpers

* Add refs

* Add file space strategy helpers

* add tests, simplify

* Add file_space_page_size to FileCreateProperties

* Simplify h5f_get_free_sections

* h5f_get_free_sections: return empty vector

* Revise h5f_get_free_sections

docs/src/api_bindings.md

@@ -174,6 +174,8 @@ h5e_walk
 - [`h5f_flush`](@ref h5f_flush)
 - [`h5f_get_access_plist`](@ref h5f_get_access_plist)
 - [`h5f_get_create_plist`](@ref h5f_get_create_plist)
+- [`h5f_get_free_sections`](@ref h5f_get_free_sections)
+- [`h5f_get_freespace`](@ref h5f_get_freespace)
 - [`h5f_get_intent`](@ref h5f_get_intent)
 - [`h5f_get_name`](@ref h5f_get_name)
 - [`h5f_get_obj_count`](@ref h5f_get_obj_count)
@@ -188,6 +190,8 @@ h5f_create
 h5f_flush
 h5f_get_access_plist
 h5f_get_create_plist
+h5f_get_free_sections
+h5f_get_freespace
 h5f_get_intent
 h5f_get_name
 h5f_get_obj_count
@@ -331,6 +335,8 @@ h5pl_size
 - [`h5p_get_fapl_multi`](@ref h5p_get_fapl_multi)
 - [`h5p_get_fapl_splitter`](@ref h5p_get_fapl_splitter)
 - [`h5p_get_fclose_degree`](@ref h5p_get_fclose_degree)
+- [`h5p_get_file_space_page_size`](@ref h5p_get_file_space_page_size)
+- [`h5p_get_file_space_strategy`](@ref h5p_get_file_space_strategy)
 - [`h5p_get_fill_time`](@ref h5p_get_fill_time)
 - [`h5p_get_fill_value`](@ref h5p_get_fill_value)
 - [`h5p_get_filter`](@ref h5p_get_filter)
@@ -368,6 +374,8 @@ h5pl_size
 - [`h5p_set_fapl_splitter`](@ref h5p_set_fapl_splitter)
 - [`h5p_set_fapl_windows`](@ref h5p_set_fapl_windows)
 - [`h5p_set_fclose_degree`](@ref h5p_set_fclose_degree)
+- [`h5p_set_file_space_page_size`](@ref h5p_set_file_space_page_size)
+- [`h5p_set_file_space_strategy`](@ref h5p_set_file_space_strategy)
 - [`h5p_set_fill_time`](@ref h5p_set_fill_time)
 - [`h5p_set_fill_value`](@ref h5p_set_fill_value)
 - [`h5p_set_filter`](@ref h5p_set_filter)
@@ -408,6 +416,8 @@ h5p_get_fapl_mpio64
 h5p_get_fapl_multi
 h5p_get_fapl_splitter
 h5p_get_fclose_degree
+h5p_get_file_space_page_size
+h5p_get_file_space_strategy
 h5p_get_fill_time
 h5p_get_fill_value
 h5p_get_filter
@@ -445,6 +455,8 @@ h5p_set_fapl_split
 h5p_set_fapl_splitter
 h5p_set_fapl_windows
 h5p_set_fclose_degree
+h5p_set_file_space_page_size
+h5p_set_file_space_strategy
 h5p_set_fill_time
 h5p_set_fill_value
 h5p_set_filter

gen/api_defs.jl

@@ -105,6 +105,8 @@
 @bind h5f_flush(object_id::hid_t, scope::Cint)::herr_t "Error flushing object to file"
 @bind h5f_get_access_plist(file_id::hid_t)::hid_t "Error getting file access property list"
 @bind h5f_get_create_plist(file_id::hid_t)::hid_t "Error getting file create property list"
+@bind h5f_get_free_sections(file_id::hid_t, type::H5F_mem_t, nsects::Csize_t, sect_info::Ptr{H5F_sect_info_t})::Cssize_t "Error in h5f_get_free_sections (not annotated)"
+@bind h5f_get_freespace(file_id::hid_t)::hssize_t "Error in h5f_get_freespace (not annotated)"
 @bind h5f_get_intent(file_id::hid_t, intent::Ptr{Cuint})::herr_t "Error getting file intent"
 @bind h5f_get_name(obj_id::hid_t, buf::Ptr{UInt8}, buf_size::Csize_t)::Cssize_t "Error getting file name"
 @bind h5f_get_obj_count(file_id::hid_t, types::Cuint)::Cssize_t "Error getting object count"
@@ -192,6 +194,8 @@
 @bind h5p_get_fapl_splitter(fapl_id::hid_t, config_ptr::Ptr{H5FD_splitter_vfd_config_t})::herr_t "Error in h5p_get_fapl_splitter (not annotated)"
 @bind h5p_get_fapl_mpio32(fapl_id::hid_t, comm::Ptr{Hmpih32}, info::Ptr{Hmpih32})::herr_t "Error getting MPIO properties"
 @bind h5p_get_fapl_mpio64(fapl_id::hid_t, comm::Ptr{Hmpih64}, info::Ptr{Hmpih64})::herr_t "Error getting MPIO properties"
+@bind h5p_get_file_space_strategy(plist_id::hid_t, strategy::Ptr{H5F_fspace_strategy_t}, persist::Ptr{hbool_t}, threshold::Ptr{hsize_t})::herr_t "Error in h5p_get_file_space_strategy (not annotated)"
+@bind h5p_get_file_space_page_size(plist_id::hid_t, fsp_size::Ptr{hsize_t})::herr_t "Error in h5p_get_file_space_page_size (not annotated)"
 @bind h5p_get_fclose_degree(fapl_id::hid_t, fc_degree::Ref{Cint})::herr_t "Error getting close degree"
 @bind h5p_get_fill_time(plist_id::hid_t, fill_time::Ptr{H5D_fill_time_t})::herr_t "Error in h5p_get_fill_time (not annotated)"
 @bind h5p_get_fill_value(plist_id::hid_t, type_id::hid_t, value::Ptr{Cvoid})::herr_t "Error in h5p_get_fill_value (not annotated)"
@@ -218,8 +222,6 @@
 @bind h5p_set_dxpl_mpio(dxpl_id::hid_t, xfer_mode::Cint)::herr_t "Error setting MPIO transfer mode"
 @bind h5p_set_external(plist_id::hid_t, name::Ptr{UInt8}, offset::off_t, size::Csize_t)::herr_t "Error setting external property"
 @bind h5p_set_efile_prefix(plist_id::hid_t, prefix::Ptr{UInt8})::herr_t "Error setting external file prefix"
-@bind h5p_set_fill_time(plist_id::hid_t, fill_time::H5D_fill_time_t)::herr_t "Error in h5p_set_fill_time (not annotated)"
-@bind h5p_set_fill_value(plist_id::hid_t, type_id::hid_t, value::Ptr{Cvoid})::herr_t "Error in h5p_set_fill_value (not annotated)"
 @bind h5p_set_fapl_core(fapl_id::hid_t, increment::Csize_t, backing_store::hbool_t)::herr_t "Error in h5p_set_fapl_core (not annotated)"
 @bind h5p_set_fapl_family(fapl_id::hid_t, memb_size::hsize_t, memb_fapl_id::hid_t)::herr_t "Error in h5p_set_fapl_family (not annotated)"
 @bind h5p_set_fapl_hdfs(fapl_id::hid_t, fa::Ptr{H5FD_hdfs_fapl_t})::herr_t "Error in h5p_set_fapl_hdfs (not annotated)"
@@ -232,6 +234,10 @@
 @bind h5p_set_fapl_splitter(fapl_id::hid_t, config_ptr::Ptr{H5FD_splitter_vfd_config_t})::herr_t "Error in h5p_set_fapl_splitter (not annotated)"
 @bind h5p_set_fapl_windows(fapl_id::hid_t)::herr_t "Error in h5p_set_fapl_windows (not annotated)"
 @bind h5p_set_fclose_degree(plist_id::hid_t, fc_degree::Cint)::herr_t "Error setting close degree"
+@bind h5p_set_file_space_strategy(plist_id::hid_t, strategy::H5F_fspace_strategy_t, persist::hbool_t, threshold::hsize_t)::herr_t "Error in h5p_set_file_space_strategy (not annotated)"
+@bind h5p_set_file_space_page_size(plist_id::hid_t, fsp_size::hsize_t)::herr_t "Error in h5p_set_file_space_page_size (not annotated)"
+@bind h5p_set_fill_time(plist_id::hid_t, fill_time::H5D_fill_time_t)::herr_t "Error in h5p_set_fill_time (not annotated)"
+@bind h5p_set_fill_value(plist_id::hid_t, type_id::hid_t, value::Ptr{Cvoid})::herr_t "Error in h5p_set_fill_value (not annotated)"
 @bind h5p_set_filter(plist_id::hid_t, filter_id::H5Z_filter_t, flags::Cuint, cd_nelmts::Csize_t, cd_values::Ptr{Cuint})::herr_t "Error setting filter"
 @bind h5p_set_fletcher32(plist_id::hid_t)::herr_t "Error enabling Fletcher32 filter"
 @bind h5p_set_layout(plist_id::hid_t, setting::Cint)::herr_t "Error setting layout"

src/api/functions.jl

@@ -740,6 +740,28 @@ function h5f_get_create_plist(file_id)
     return var"#status#"
 end
 
+"""
+    h5f_get_free_sections(file_id::hid_t, type::H5F_mem_t, nsects::Csize_t, sect_info::Ptr{H5F_sect_info_t}) -> Cssize_t
+
+See `libhdf5` documentation for [`H5Fget_free_sections`](https://portal.hdfgroup.org/display/HDF5/H5F_GET_FREE_SECTIONS).
+"""
+function h5f_get_free_sections(file_id, type, nsects, sect_info)
+    var"#status#" = ccall((:H5Fget_free_sections, libhdf5), Cssize_t, (hid_t, H5F_mem_t, Csize_t, Ptr{H5F_sect_info_t}), file_id, type, nsects, sect_info)
+    var"#status#" < 0 && @h5error("Error in h5f_get_free_sections (not annotated)")
+    return var"#status#"
+end
+
+"""
+    h5f_get_freespace(file_id::hid_t) -> hssize_t
+
+See `libhdf5` documentation for [`H5Fget_freespace`](https://portal.hdfgroup.org/display/HDF5/H5F_GET_FREESPACE).
+"""
+function h5f_get_freespace(file_id)
+    var"#status#" = ccall((:H5Fget_freespace, libhdf5), hssize_t, (hid_t,), file_id)
+    var"#status#" < 0 && @h5error("Error in h5f_get_freespace (not annotated)")
+    return var"#status#"
+end
+
 """
     h5f_get_intent(file_id::hid_t, intent::Ptr{Cuint})
 
@@ -1393,6 +1415,28 @@ function h5p_get_fapl_mpio64(fapl_id, comm, info)
     return nothing
 end
 
+"""
+    h5p_get_file_space_strategy(plist_id::hid_t, strategy::Ptr{H5F_fspace_strategy_t}, persist::Ptr{hbool_t}, threshold::Ptr{hsize_t})
+
+See `libhdf5` documentation for [`H5Pget_file_space_strategy`](https://portal.hdfgroup.org/display/HDF5/H5P_GET_FILE_SPACE_STRATEGY).
+"""
+function h5p_get_file_space_strategy(plist_id, strategy, persist, threshold)
+    var"#status#" = ccall((:H5Pget_file_space_strategy, libhdf5), herr_t, (hid_t, Ptr{H5F_fspace_strategy_t}, Ptr{hbool_t}, Ptr{hsize_t}), plist_id, strategy, persist, threshold)
+    var"#status#" < 0 && @h5error("Error in h5p_get_file_space_strategy (not annotated)")
+    return nothing
+end
+
+"""
+    h5p_get_file_space_page_size(plist_id::hid_t, fsp_size::Ptr{hsize_t})
+
+See `libhdf5` documentation for [`H5Pget_file_space_page_size`](https://portal.hdfgroup.org/display/HDF5/H5P_GET_FILE_SPACE_PAGE_SIZE).
+"""
+function h5p_get_file_space_page_size(plist_id, fsp_size)
+    var"#status#" = ccall((:H5Pget_file_space_page_size, libhdf5), herr_t, (hid_t, Ptr{hsize_t}), plist_id, fsp_size)
+    var"#status#" < 0 && @h5error("Error in h5p_get_file_space_page_size (not annotated)")
+    return nothing
+end
+
 """
     h5p_get_fclose_degree(fapl_id::hid_t, fc_degree::Ref{Cint})
 
@@ -1657,17 +1701,6 @@ function h5p_set_efile_prefix(plist_id, prefix)
     return nothing
 end
 
-"""
-    h5p_set_fill_time(plist_id::hid_t, fill_time::H5D_fill_time_t)
-
-See `libhdf5` documentation for [`H5Pset_fill_time`](https://portal.hdfgroup.org/display/HDF5/H5P_SET_FILL_TIME).
-"""
-function h5p_set_fill_time(plist_id, fill_time)
-    var"#status#" = ccall((:H5Pset_fill_time, libhdf5), herr_t, (hid_t, H5D_fill_time_t), plist_id, fill_time)
-    var"#status#" < 0 && @h5error("Error in h5p_set_fill_time (not annotated)")
-    return nothing
-end
-
 """
     h5p_set_fapl_core(fapl_id::hid_t, increment::Csize_t, backing_store::hbool_t)
 
@@ -1679,17 +1712,6 @@ function h5p_set_fapl_core(fapl_id, increment, backing_store)
     return nothing
 end
 
-"""
-    h5p_set_fill_value(plist_id::hid_t, type_id::hid_t, value::Ptr{Cvoid})
-
-See `libhdf5` documentation for [`H5Pset_fill_value`](https://portal.hdfgroup.org/display/HDF5/H5P_SET_FILL_VALUE).
-"""
-function h5p_set_fill_value(plist_id, type_id, value)
-    var"#status#" = ccall((:H5Pset_fill_value, libhdf5), herr_t, (hid_t, hid_t, Ptr{Cvoid}), plist_id, type_id, value)
-    var"#status#" < 0 && @h5error("Error in h5p_set_fill_value (not annotated)")
-    return nothing
-end
-
 """
     h5p_set_fapl_family(fapl_id::hid_t, memb_size::hsize_t, memb_fapl_id::hid_t)
 
@@ -1811,6 +1833,50 @@ function h5p_set_fclose_degree(plist_id, fc_degree)
     return nothing
 end
 
+"""
+    h5p_set_file_space_strategy(plist_id::hid_t, strategy::H5F_fspace_strategy_t, persist::hbool_t, threshold::hsize_t)
+
+See `libhdf5` documentation for [`H5Pset_file_space_strategy`](https://portal.hdfgroup.org/display/HDF5/H5P_SET_FILE_SPACE_STRATEGY).
+"""
+function h5p_set_file_space_strategy(plist_id, strategy, persist, threshold)
+    var"#status#" = ccall((:H5Pset_file_space_strategy, libhdf5), herr_t, (hid_t, H5F_fspace_strategy_t, hbool_t, hsize_t), plist_id, strategy, persist, threshold)
+    var"#status#" < 0 && @h5error("Error in h5p_set_file_space_strategy (not annotated)")
+    return nothing
+end
+
+"""
+    h5p_set_file_space_page_size(plist_id::hid_t, fsp_size::hsize_t)
+
+See `libhdf5` documentation for [`H5Pset_file_space_page_size`](https://portal.hdfgroup.org/display/HDF5/H5P_SET_FILE_SPACE_PAGE_SIZE).
+"""
+function h5p_set_file_space_page_size(plist_id, fsp_size)
+    var"#status#" = ccall((:H5Pset_file_space_page_size, libhdf5), herr_t, (hid_t, hsize_t), plist_id, fsp_size)
+    var"#status#" < 0 && @h5error("Error in h5p_set_file_space_page_size (not annotated)")
+    return nothing
+end
+
+"""
+    h5p_set_fill_time(plist_id::hid_t, fill_time::H5D_fill_time_t)
+
+See `libhdf5` documentation for [`H5Pset_fill_time`](https://portal.hdfgroup.org/display/HDF5/H5P_SET_FILL_TIME).
+"""
+function h5p_set_fill_time(plist_id, fill_time)
+    var"#status#" = ccall((:H5Pset_fill_time, libhdf5), herr_t, (hid_t, H5D_fill_time_t), plist_id, fill_time)
+    var"#status#" < 0 && @h5error("Error in h5p_set_fill_time (not annotated)")
+    return nothing
+end
+
+"""
+    h5p_set_fill_value(plist_id::hid_t, type_id::hid_t, value::Ptr{Cvoid})
+
+See `libhdf5` documentation for [`H5Pset_fill_value`](https://portal.hdfgroup.org/display/HDF5/H5P_SET_FILL_VALUE).
+"""
+function h5p_set_fill_value(plist_id, type_id, value)
+    var"#status#" = ccall((:H5Pset_fill_value, libhdf5), herr_t, (hid_t, hid_t, Ptr{Cvoid}), plist_id, type_id, value)
+    var"#status#" < 0 && @h5error("Error in h5p_set_fill_value (not annotated)")
+    return nothing
+end
+
 """
     h5p_set_filter(plist_id::hid_t, filter_id::H5Z_filter_t, flags::Cuint, cd_nelmts::Csize_t, cd_values::Ptr{Cuint})
 

src/api/helpers.jl

@@ -224,6 +224,49 @@ function h5f_get_vfd_handle(file_id, fapl)
     return file_handle[]
 end
 
+"""
+    h5f_get_free_sections(file_id, type, [sect_info::AbstractVector{H5F_sect_info_t}])::AbstractVector{H5F_sect_info_t}
+
+Return an `AbstractVector` of the free section information. If `sect_info` is not provided a new `Vector` will be allocated and returned.
+If `sect_info` is provided, a view, a `SubArray`, will be returned.
+"""
+function h5f_get_free_sections(file_id, type)
+    nsects = h5f_get_free_sections(file_id, type, 0, C_NULL)
+    sect_info = Vector{H5F_sect_info_t}(undef, nsects)
+    if nsects > 0
+        h5f_get_free_sections(file_id, type, nsects, sect_info)
+    end
+    return sect_info
+end
+
+function h5f_get_free_sections(file_id, type, sect_info::AbstractVector{H5F_sect_info_t})
+    nsects = length(sect_info)
+    nsects = h5f_get_free_sections(file_id, type, nsects, sect_info)
+    return @view(sect_info[1:nsects])
+end
+
+function h5p_get_file_space_strategy(plist_id)
+    strategy = Ref{H5F_fspace_strategy_t}()
+    persist = Ref{hbool_t}(0)
+    threshold = Ref{hsize_t}()
+    h5p_get_file_space_strategy(plist_id, strategy, persist, threshold)
+    return (strategy = strategy[], persist = persist[], threshold = threshold[])
+end
+
+function h5p_get_file_space_page_size(plist_id)
+    fsp_size = Ref{hsize_t}()
+    h5p_get_file_space_page_size(plist_id, fsp_size)
+    return fsp_size[]
+end
+
+function h5p_set_file_space_strategy(plist_id; strategy = nothing, persist = nothing, threshold = nothing)
+    current = h5p_get_file_space_strategy(plist_id)
+    strategy = isnothing(strategy) ? current[:strategy] : strategy
+    persist = isnothing(persist) ? current[:persist] : persist
+    threshold = isnothing(threshold) ? current[:threshold] : threshold
+    return h5p_set_file_space_strategy(plist_id, strategy, persist, threshold)
+end
+
 ###
 ### Group Interface
 ###

src/api/types.jl

@@ -453,3 +453,25 @@ end
 end
 
 const H5FD_mem_t = H5F_mem_t
+
+struct H5F_sect_info_t
+    addr::haddr_t
+    size::hsize_t
+end
+
+@enum H5F_fspace_strategy_t::UInt32 begin
+    H5F_FSPACE_STRATEGY_FSM_AGGR = 0
+    H5F_FSPACE_STRATEGY_PAGE = 1
+    H5F_FSPACE_STRATEGY_AGGR = 2
+    H5F_FSPACE_STRATEGY_NONE = 3
+    H5F_FSPACE_STRATEGY_NTYPES = 4
+end
+
+@enum H5F_file_space_type_t::UInt32 begin
+    H5F_FILE_SPACE_DEFAULT = 0
+    H5F_FILE_SPACE_ALL_PERSIST = 1
+    H5F_FILE_SPACE_ALL = 2
+    H5F_FILE_SPACE_AGGR_VFD = 3
+    H5F_FILE_SPACE_VFD = 4
+    H5F_FILE_SPACE_NTYPES = 5
+end

src/properties.jl

@@ -308,15 +308,48 @@ superclass(::Type{FileCreateProperties}) = ObjectCreateProperties
 class_propertynames(::Type{FileCreateProperties}) = (
     :userblock,
     :track_order,
+    :strategy,
+    :persist,
+    :threshold,
+    :file_space_page_size
     )
+
+const FSPACE_STRATEGY_SYMBOLS = Dict(
+    :fsm_aggr => API.H5F_FSPACE_STRATEGY_FSM_AGGR,
+    :page => API.H5F_FSPACE_STRATEGY_PAGE,
+    :aggr => API.H5F_FSPACE_STRATEGY_AGGR,
+    :none => API.H5F_FSPACE_STRATEGY_NONE,
+    :ntypes => API.H5F_FSPACE_STRATEGY_NTYPES
+)
+
+set_strategy!(p::FileCreateProperties, val) = API.h5p_set_file_space_strategy(p, strategy = val)
+set_strategy!(p::FileCreateProperties, val::Symbol) = API.h5p_set_file_space_strategy(p, strategy = FSPACE_STRATEGY_SYMBOLS[val])
+function get_strategy(p::FileCreateProperties)
+    strategy = API.h5p_get_file_space_strategy(p)[:strategy]
+    for (k, v) in FSPACE_STRATEGY_SYMBOLS
+        if v == strategy
+            return k
+        end
+    end
+    return :unknown
+end
+
 function class_getproperty(::Type{FileCreateProperties}, p::Properties, name::Symbol)
     name === :userblock   ? API.h5p_get_userblock(p) :
     name === :track_order ? get_track_order(p) :
+    name === :strategy    ? get_strategy(p) :
+    name === :persist     ? API.h5p_get_file_space_strategy(p)[:persist] :
+    name === :threshold   ? API.h5p_get_file_space_strategy(p)[:threshold] :
+    name === :file_space_page_size ? API.h5p_get_file_space_page_size(p) :
     class_getproperty(superclass(FileCreateProperties), p, name)
 end
 function class_setproperty!(::Type{FileCreateProperties}, p::Properties, name::Symbol, val)
     name === :userblock   ? API.h5p_set_userblock(p, val) :
     name === :track_order ? set_track_order(p, val) : 
+    name === :strategy ? set_strategy!(p, val) :
+    name === :persist ? API.h5p_set_file_space_strategy(p, persist = val) :
+    name === :threshold ? API.h5p_set_file_space_strategy(p, threshold = val) :
+    name === :file_space_page_size ? API.h5p_set_file_space_page_size(p, val) :
     class_setproperty!(superclass(FileCreateProperties), p, name, val)
 end
 

test/properties.jl

@@ -8,6 +8,10 @@ h5open(fn, "w";
        userblock = 1024,
        alignment = (0, sizeof(Int)),
        libver_bounds = (:earliest, :latest),
+       strategy = :fsm_aggr,
+       persist = 1,
+       threshold = 2,
+       file_space_page_size = 0x800
       ) do hfile
     # generic
     g = create_group(hfile, "group")
@@ -37,6 +41,10 @@ h5open(fn, "w";
     # Retrieving property values:
     @test fcpl.userblock == 1024
     @test fcpl.obj_track_times
+    @test fcpl.file_space_page_size == 0x800
+    @test fcpl.strategy == :fsm_aggr
+    @test fcpl.persist == 1
+    @test fcpl.threshold == 2
 
     @test fapl.alignment == (0, sizeof(Int))
     @test fapl.driver == Drivers.POSIX()