Provide high level syntax for dealing with HDF5 files in memory (#1077)

* Fix get_driver for Core

* Enhance creating a file from memory using the Core driver

* Check for file image callbacks

* Formatting

* Use h5_free_memory for h5p_get_file_image helper

src/api/helpers.jl

@@ -30,6 +30,27 @@ function h5f_get_dset_no_attrs_hint(file_id)::Bool
     return minimize[]
 end
 
+"""
+    h5f_get_file_image(file_id)
+
+Return a `Vector{UInt8}` containing the file image. Does not include the user block.
+"""
+function h5f_get_file_image(file_id)
+    buffer_length = h5f_get_file_image(file_id, C_NULL, 0)
+    buffer = Vector{UInt8}(undef, buffer_length)
+    h5f_get_file_image(file_id, buffer, buffer_length)
+    return buffer
+end
+
+"""
+    h5f_get_file_image(file_id, buffer::Vector{UInt8})
+
+Store the file image in the provided buffer.
+"""
+function h5f_get_file_image(file_id, buffer::Vector{UInt8})
+    h5f_get_file_image(fild_id, buffer, length(buffer))
+end
+
 ###
 ### Attribute Interface
 ###
@@ -790,6 +811,52 @@ function h5p_get_virtual_view(dapl_id)
     return view[]
 end
 
+"""
+    h5p_get_file_image(fapl_id)::Vector{UInt8}
+
+Retrieve a file image of the appropriate size in a `Vector{UInt8}`.
+"""
+function h5p_get_file_image(fapl_id)::Vector{UInt8}
+    cb = h5p_get_file_image_callbacks(fapl_id)
+    if cb.image_free != C_NULL
+        # The user has configured their own memory deallocation routines.
+        # The user should use a lower level call to properly handle deallocation
+        error(
+            "File image callback image_free is not C_NULL. Use the three argument method of h5p_get_file_image when setting file image callbacks."
+        )
+    end
+    buf_ptr_ref = Ref{Ptr{Nothing}}()
+    buf_len_ref = Ref{Csize_t}(0)
+    h5p_get_file_image(fapl_id, buf_ptr_ref, buf_len_ref)
+    image = unsafe_wrap(Array{UInt8}, Ptr{UInt8}(buf_ptr_ref[]), buf_len_ref[]; own=false)
+    finalizer(image) do image
+        # Use h5_free_memory to ensure we are using the correct free
+        h5_free_memory(image)
+    end
+    return image
+end
+
+"""
+    h5p_set_file_image(fapl_id, image::Vector{UInt8})
+
+Set the file image from a `Vector{UInt8}`.
+"""
+function h5p_set_file_image(fapl_id, image::Vector{UInt8})
+    h5p_set_file_image(fapl_id, image, length(image))
+end
+
+"""
+    h5p_get_file_image_callbacks(fapl_id)
+
+Retrieve the file image callbacks for memory operations
+"""
+function h5p_get_file_image_callbacks(fapl_id)
+    cb = H5FD_file_image_callbacks_t(C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL)
+    r = Ref(cb)
+    h5p_get_file_image_callbacks(fapl_id, r)
+    return r[]
+end
+
 # Note: The following function(s) implement direct ccalls because the binding generator
 # cannot (yet) do the string wrapping and memory freeing.
 

src/drivers/drivers.jl

@@ -45,8 +45,8 @@ Core(; increment=8192, backing_store=true, write_tracking=false, page_size=52428
 
 function get_driver(p::Properties, ::Type{Core})
     r_increment = Ref{Csize_t}(0)
-    r_backing_store = Ref{Cuint}(0)
-    r_write_tracking = Ref{Cuint}(0)
+    r_backing_store = Ref{Bool}(0)
+    r_write_tracking = Ref{Bool}(0)
     r_page_size = Ref{Csize_t}(0)
     API.h5p_get_fapl_core(p, r_increment, r_backing_store)
     API.h5p_get_core_write_tracking(p, r_write_tracking, r_page_size)

src/drivers/ros3.jl

@@ -54,7 +54,7 @@ function Base.convert(::Type{API.H5FD_ros3_fapl_t}, driver::ROS3)
 end
 
 function get_driver(fapl::Properties, ::Type{ROS3})
-    r_fa = Ref{H5FD_ros3_fapl_t}()
+    r_fa = Ref{API.H5FD_ros3_fapl_t}()
     API.h5p_get_fapl_ros3(fapl, r_fa)
     return ROS3(r_fa[])
 end

src/file.jl

@@ -41,6 +41,9 @@ function h5open(
     if cr && (tr || !isfile(filename))
         flag = swmr ? API.H5F_ACC_TRUNC | API.H5F_ACC_SWMR_WRITE : API.H5F_ACC_TRUNC
         fid = API.h5f_create(filename, flag, fcpl, fapl)
+    elseif fapl.driver isa Drivers.Core && fapl.driver.backing_store == 0
+        flag = wr ? API.H5F_ACC_RDWR : API.H5F_ACC_RDONLY
+        fid = API.h5f_open(filename, flag, fapl)
     else
         occursin(r"(s3a?|https?)://", filename) ||
             ishdf5(filename) ||
@@ -77,7 +80,7 @@ function h5open(
 end
 
 """
-    function h5open(f::Function, args...; pv...)
+    h5open(f::Function, args...; pv...)
 
 Apply the function f to the result of `h5open(args...; kwargs...)` and close the resulting
 `HDF5.File` upon completion.
@@ -103,6 +106,46 @@ function h5open(f::Function, args...; context=copy(CONTEXT), pv...)
     end
 end
 
+"""
+    h5open(file_image::Vector{UInt8}, mode::AbstractString="r";
+        name=nothing,
+        fapl=FileAccessProperties(),
+        increment=8192,
+        backing_store=false,
+        write_tracking=false,
+        page_size=524288,
+        pv...
+    )
+
+Open a file image contained in a `Vector{UInt8}`. See [`API.h5p_set_file_image`](@ref).
+
+Unlike [`Drivers.Core`](@ref) the default here is not to use a backing store.
+"""
+function h5open(
+    file_image::Vector{UInt8},
+    mode::AbstractString="r";
+    name=nothing,
+    fapl=FileAccessProperties(),
+    increment=8192,
+    backing_store=false,
+    write_tracking=false,
+    page_size=524288,
+    pv...
+)
+    fapl.driver = Drivers.Core(; increment, backing_store, write_tracking, page_size)
+    fapl.file_image = file_image
+    if isnothing(name)
+        if fapl.driver.backing_store != 0
+            # The temporary file will be used as a backing store
+            name = tempname()
+        else
+            # Provide it with a unique name based on the objectid
+            name = "<memory objectid>: " * repr(objectid(file_image))
+        end
+    end
+    h5open(name, mode; fapl, pv...)
+end
+
 function h5rewrite(f::Function, filename::AbstractString, args...)
     tmppath, tmpio = mktemp(dirname(filename))
     close(tmpio)
@@ -169,3 +212,15 @@ start_swmr_write(h5::File) = API.h5f_start_swmr_write(h5)
 # Flush buffers
 Base.flush(f::Union{Object,Attribute,Datatype,File}, scope=API.H5F_SCOPE_GLOBAL) =
     API.h5f_flush(checkvalid(f), scope)
+
+# File image conversion
+
+function Vector{UInt8}(h5f::File)
+    flush(h5f)
+    API.h5f_get_file_image(h5f)
+end
+function File(file_image::Vector{UInt8}, name=nothing)
+    h5open(file_image; name)
+end
+Base.convert(::Type{Vector{UInt8}}, h5f::File) = Vector{UInt8}(h5f)
+Base.convert(::Type{File}, file_image::Vector{UInt8}) = File(file_image)

src/properties.jl

@@ -745,6 +745,7 @@ class_propertynames(::Type{FileAccessProperties}) = (
     :file_locking,
     :libver_bounds,
     :meta_block_size,
+    :file_image,
     )
 
 function class_getproperty(::Type{FileAccessProperties}, p::Properties, name::Symbol)
@@ -755,6 +756,7 @@ function class_getproperty(::Type{FileAccessProperties}, p::Properties, name::Sy
     name === :file_locking  ? API.h5p_get_file_locking(p) :
     name === :libver_bounds ? get_libver_bounds(p) :
     name === :meta_block_size ? API.h5p_get_meta_block_size(p) :
+    name === :file_image      ? API.h5p_get_file_image(p) :
     # deprecated
     name === :fapl_mpio     ? (depwarn("The `fapl_mpio` property is deprecated, use `driver=HDF5.Drivers.MPIO(...)` instead.", :fapl_mpio); drv = get_driver(p, MPIO); (drv.comm, drv.info)) :
     class_getproperty(superclass(FileAccessProperties), p, name)
@@ -766,6 +768,7 @@ function class_setproperty!(::Type{FileAccessProperties}, p::Properties, name::S
     name === :file_locking  ? API.h5p_set_file_locking(p, val...) :
     name === :libver_bounds ? set_libver_bounds!(p, val) :
     name === :meta_block_size ? API.h5p_set_meta_block_size(p, val) :
+    name === :file_image      ? API.h5p_set_file_image(p, val) :
     # deprecated
     name === :fapl_mpio     ? (depwarn("The `fapl_mpio` property is deprecated, use `driver=HDF5.Drivers.MPIO(...)` instead.", :fapl_mpio); Drivers.set_driver!(p, Drivers.MPIO(val...))) :
     class_setproperty!(superclass(FileAccessProperties), p, name, val)

src/show.jl

@@ -36,7 +36,20 @@ function Base.show(io::IO, prop::Properties)
             # or always well-defined (e.g. chunk if layout != :chunked, dxpl_mpio if no MPI)
             try
                 val = getproperty(prop, name)
-                print(io, "\n  ", rpad(name, 15), " = ", repr(val), ",")
+                if name == :file_image
+                    print(
+                        io,
+                        "\n  ",
+                        rpad(name, 15),
+                        " = ",
+                        repr(typeof(val)),
+                        ", length = ",
+                        length(val),
+                        ","
+                    )
+                else
+                    print(io, "\n  ", rpad(name, 15), " = ", repr(val), ",")
+                end
             catch e
             end
         end

test/drivers.jl

@@ -14,8 +14,18 @@ using Test
     end
 
     fn = tempname()
-    h5open(fn, "w"; driver=Drivers.Core(; backing_store=false)) do f
+    file_image = h5open(fn, "w"; driver=Drivers.Core(; backing_store=false)) do f
         ds = write_dataset(f, "core_dataset", A)
+        Vector{UInt8}(f)
     end
     @test !isfile(fn)
+
+    h5open(file_image) do f
+        @test f["core_dataset"][] == A
+    end
+
+    fn = tempname()
+    h5open(fn, "r"; driver=Drivers.Core(; backing_store=false), file_image) do f
+        @test f["core_dataset"][] == A
+    end
 end

test/properties.jl

@@ -121,5 +121,20 @@ using Test
         nothing
     end
 
+    file_image = read(fn)
+
+    rm(fn; force=true)
+
+    fapl = HDF5.FileAccessProperties()
+    fapl.driver = HDF5.Drivers.Core(; backing_store=false)
+    fapl.file_image = copy(file_image)
+    @test fapl.file_image == file_image
+    file_image2 = h5open(fn, "r"; fapl) do f
+        @test haskey(f, "group")
+        convert(Vector{UInt8}, f)
+    end
+    # file_image2 does not include user block
+    @test file_image2 == @view(file_image[1025:end])
+
     rm(fn; force=true)
 end