From 0470a0c41283c4ffa4306e1a0fcefd342d869142 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Fri, 2 Jun 2023 23:04:25 -0400 Subject: [PATCH 1/5] Fix get_driver for Core --- src/drivers/drivers.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/drivers/drivers.jl b/src/drivers/drivers.jl index dd788b51d..a985cabaa 100644 --- a/src/drivers/drivers.jl +++ b/src/drivers/drivers.jl @@ -43,8 +43,8 @@ Core(; increment=8192, backing_store=true, write_tracking=false, page_size=52428 function get_driver(p::Properties, ::Type{Core}) r_increment = Ref{Csize_t}(0) - r_backing_store = Ref{Cuint}(0) - r_write_tracking = Ref{Cuint}(0) + r_backing_store = Ref{Bool}(0) + r_write_tracking = Ref{Bool}(0) r_page_size = Ref{Csize_t}(0) API.h5p_get_fapl_core(p, r_increment, r_backing_store) API.h5p_get_core_write_tracking(p, r_write_tracking, r_page_size) From c6e3ea32e647303830ec0c09f994b651a1a812f7 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Sat, 3 Jun 2023 01:52:44 -0400 Subject: [PATCH 2/5] Enhance creating a file from memory using the Core driver --- src/api/helpers.jl | 42 +++++++++++++++++++++++++++++++++ src/drivers/ros3.jl | 2 +- src/file.jl | 57 ++++++++++++++++++++++++++++++++++++++++++++- src/properties.jl | 3 +++ src/show.jl | 15 +++++++++++- test/drivers.jl | 12 +++++++++- test/properties.jl | 15 ++++++++++++ 7 files changed, 142 insertions(+), 4 deletions(-) diff --git a/src/api/helpers.jl b/src/api/helpers.jl index 441c1aa4e..42730aa26 100644 --- a/src/api/helpers.jl +++ b/src/api/helpers.jl @@ -30,6 +30,27 @@ function h5f_get_dset_no_attrs_hint(file_id)::Bool return minimize[] end +""" + h5f_get_file_image(file_id) + +Return a `Vector{UInt8}` containing the file image. Does not include the user block. +""" +function h5f_get_file_image(file_id) + buffer_length = h5f_get_file_image(file_id, C_NULL, 0) + buffer = Vector{UInt8}(undef, buffer_length) + h5f_get_file_image(file_id, buffer, buffer_length) + return buffer +end + +""" + h5f_get_file_image(file_id, buffer::Vector{UInt8}) + +Store the file image in the provided buffer. +""" +function h5f_get_file_image(file_id, buffer::Vector{UInt8}) + h5f_get_file_image(fild_id, buffer, length(buffer)) +end + ### ### Attribute Interface ### @@ -790,6 +811,27 @@ function h5p_get_virtual_view(dapl_id) return view[] end +""" + h5p_get_file_image(fapl_id)::Vector{UInt8} + +Retrieve a file image of the appropriate size in a `Vector{UInt8}`. +""" +function h5p_get_file_image(fapl_id)::Vector{UInt8} + buf_ptr_ref = Ref{Ptr{Nothing}}() + buf_len_ref = Ref{Csize_t}(0) + h5p_get_file_image(fapl_id, buf_ptr_ref, buf_len_ref) + unsafe_wrap(Array{UInt8}, Ptr{UInt8}(buf_ptr_ref[]), buf_len_ref[]; own=true) +end + +""" + h5p_set_file_image(fapl_id, image::Vector{UInt8}) + +Set the file image from a `Vector{UInt8}`. +""" +function h5p_set_file_image(fapl_id, image::Vector{UInt8}) + h5p_set_file_image(fapl_id, image, length(image)) +end + # Note: The following function(s) implement direct ccalls because the binding generator # cannot (yet) do the string wrapping and memory freeing. diff --git a/src/drivers/ros3.jl b/src/drivers/ros3.jl index 85f762a26..33090d97b 100644 --- a/src/drivers/ros3.jl +++ b/src/drivers/ros3.jl @@ -54,7 +54,7 @@ function Base.convert(::Type{API.H5FD_ros3_fapl_t}, driver::ROS3) end function get_driver(fapl::Properties, ::Type{ROS3}) - r_fa = Ref{H5FD_ros3_fapl_t}() + r_fa = Ref{API.H5FD_ros3_fapl_t}() API.h5p_get_fapl_ros3(fapl, r_fa) return ROS3(r_fa[]) end diff --git a/src/file.jl b/src/file.jl index d8c151b53..6d1d9875a 100644 --- a/src/file.jl +++ b/src/file.jl @@ -41,6 +41,9 @@ function h5open( if cr && (tr || !isfile(filename)) flag = swmr ? API.H5F_ACC_TRUNC | API.H5F_ACC_SWMR_WRITE : API.H5F_ACC_TRUNC fid = API.h5f_create(filename, flag, fcpl, fapl) + elseif fapl.driver isa Drivers.Core && fapl.driver.backing_store == 0 + flag = wr ? API.H5F_ACC_RDWR : API.H5F_ACC_RDONLY + fid = API.h5f_open(filename, flag, fapl) else occursin(r"(s3a?|https?)://", filename) || ishdf5(filename) || @@ -77,7 +80,7 @@ function h5open( end """ - function h5open(f::Function, args...; pv...) + h5open(f::Function, args...; pv...) Apply the function f to the result of `h5open(args...; kwargs...)` and close the resulting `HDF5.File` upon completion. @@ -103,6 +106,46 @@ function h5open(f::Function, args...; context=copy(CONTEXT), pv...) end end +""" + h5open(file_image::Vector{UInt8}, mode::AbstractString="r"; + name=nothing, + fapl=FileAccessProperties(), + increment=8192, + backing_store=false, + write_tracking=false, + page_size=524288, + pv... + ) + +Open a file image contained in a `Vector{UInt8}`. See [`API.h5p_set_file_image`](@ref). + +Unlike [`Drivers.Core`](@ref) the default here is not to use a backing store. +""" +function h5open( + file_image::Vector{UInt8}, + mode::AbstractString="r"; + name=nothing, + fapl=FileAccessProperties(), + increment=8192, + backing_store=false, + write_tracking=false, + page_size=524288, + pv... +) + fapl.driver = Drivers.Core(; increment, backing_store, write_tracking, page_size) + fapl.file_image = file_image + if isnothing(name) + if fapl.driver.backing_store != 0 + # The temporary file will be used as a backing store + name = tempname() + else + # Provide it with a unique name based on the objectid + name = ": " * repr(objectid(file_image)) + end + end + h5open(name, mode; fapl, pv...) +end + function h5rewrite(f::Function, filename::AbstractString, args...) tmppath, tmpio = mktemp(dirname(filename)) close(tmpio) @@ -169,3 +212,15 @@ start_swmr_write(h5::File) = API.h5f_start_swmr_write(h5) # Flush buffers Base.flush(f::Union{Object,Attribute,Datatype,File}, scope=API.H5F_SCOPE_GLOBAL) = API.h5f_flush(checkvalid(f), scope) + +# File image conversion + +function Vector{UInt8}(h5f::File) + flush(h5f) + API.h5f_get_file_image(h5f) +end +function File(file_image::Vector{UInt8}, name=nothing) + h5open(file_image; name) +end +Base.convert(::Type{Vector{UInt8}}, h5f::File) = Vector{UInt8}(h5f) +Base.convert(::Type{File}, file_image::Vector{UInt8}) = File(file_image) diff --git a/src/properties.jl b/src/properties.jl index 48a57d14a..01df0d457 100644 --- a/src/properties.jl +++ b/src/properties.jl @@ -743,6 +743,7 @@ class_propertynames(::Type{FileAccessProperties}) = ( :file_locking, :libver_bounds, :meta_block_size, + :file_image, ) function class_getproperty(::Type{FileAccessProperties}, p::Properties, name::Symbol) @@ -753,6 +754,7 @@ function class_getproperty(::Type{FileAccessProperties}, p::Properties, name::Sy name === :file_locking ? API.h5p_get_file_locking(p) : name === :libver_bounds ? get_libver_bounds(p) : name === :meta_block_size ? API.h5p_get_meta_block_size(p) : + name === :file_image ? API.h5p_get_file_image(p) : # deprecated name === :fapl_mpio ? (depwarn("The `fapl_mpio` property is deprecated, use `driver=HDF5.Drivers.MPIO(...)` instead.", :fapl_mpio); drv = get_driver(p, MPIO); (drv.comm, drv.info)) : class_getproperty(superclass(FileAccessProperties), p, name) @@ -764,6 +766,7 @@ function class_setproperty!(::Type{FileAccessProperties}, p::Properties, name::S name === :file_locking ? API.h5p_set_file_locking(p, val...) : name === :libver_bounds ? set_libver_bounds!(p, val) : name === :meta_block_size ? API.h5p_set_meta_block_size(p, val) : + name === :file_image ? API.h5p_set_file_image(p, val) : # deprecated name === :fapl_mpio ? (depwarn("The `fapl_mpio` property is deprecated, use `driver=HDF5.Drivers.MPIO(...)` instead.", :fapl_mpio); Drivers.set_driver!(p, Drivers.MPIO(val...))) : class_setproperty!(superclass(FileAccessProperties), p, name, val) diff --git a/src/show.jl b/src/show.jl index 833c42652..51f7b49c1 100644 --- a/src/show.jl +++ b/src/show.jl @@ -36,7 +36,20 @@ function Base.show(io::IO, prop::Properties) # or always well-defined (e.g. chunk if layout != :chunked, dxpl_mpio if no MPI) try val = getproperty(prop, name) - print(io, "\n ", rpad(name, 15), " = ", repr(val), ",") + if name == :file_image + print( + io, + "\n ", + rpad(name, 15), + " = ", + repr(typeof(val)), + ", length = ", + length(val), + "," + ) + else + print(io, "\n ", rpad(name, 15), " = ", repr(val), ",") + end catch e end end diff --git a/test/drivers.jl b/test/drivers.jl index 709871b60..16ad93a15 100644 --- a/test/drivers.jl +++ b/test/drivers.jl @@ -14,8 +14,18 @@ using Test end fn = tempname() - h5open(fn, "w"; driver=Drivers.Core(; backing_store=false)) do f + file_image = h5open(fn, "w"; driver=Drivers.Core(; backing_store=false)) do f ds = write_dataset(f, "core_dataset", A) + Vector{UInt8}(f) end @test !isfile(fn) + + h5open(file_image) do f + @test f["core_dataset"][] == A + end + + fn = tempname() + h5open(fn, "r"; driver=Drivers.Core(; backing_store=false), file_image) do f + @test f["core_dataset"][] == A + end end diff --git a/test/properties.jl b/test/properties.jl index ac0ffb32c..81353ff5f 100644 --- a/test/properties.jl +++ b/test/properties.jl @@ -121,5 +121,20 @@ using Test nothing end + file_image = read(fn) + + rm(fn; force=true) + + fapl = HDF5.FileAccessProperties() + fapl.driver = HDF5.Drivers.Core(; backing_store=false) + fapl.file_image = copy(file_image) + @test fapl.file_image == file_image + file_image2 = h5open(fn, "r"; fapl) do f + @test haskey(f, "group") + convert(Vector{UInt8}, f) + end + # file_image2 does not include user block + @test file_image2 == @view(file_image[1025:end]) + rm(fn; force=true) end From 07f80f10a0e8224ea63081889809550b82ba32e8 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Fri, 1 Sep 2023 04:12:50 -0400 Subject: [PATCH 3/5] Check for file image callbacks --- src/api/helpers.jl | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/api/helpers.jl b/src/api/helpers.jl index 42730aa26..889838703 100644 --- a/src/api/helpers.jl +++ b/src/api/helpers.jl @@ -817,6 +817,10 @@ end Retrieve a file image of the appropriate size in a `Vector{UInt8}`. """ function h5p_get_file_image(fapl_id)::Vector{UInt8} + cb = h5p_get_file_image_callbacks(fapl_id) + if cb.image_free != C_NULL + error("File image callback image_free is not C_NULL. Use the three argument method of h5p_get_file_image when setting file image callbacks.") + end buf_ptr_ref = Ref{Ptr{Nothing}}() buf_len_ref = Ref{Csize_t}(0) h5p_get_file_image(fapl_id, buf_ptr_ref, buf_len_ref) @@ -832,6 +836,26 @@ function h5p_set_file_image(fapl_id, image::Vector{UInt8}) h5p_set_file_image(fapl_id, image, length(image)) end +""" + h5p_get_file_image_callbacks(fapl_id) + +Retrieve the file image callbacks for memory operations +""" +function h5p_get_file_image_callbacks(fapl_id) + cb = H5FD_file_image_callbacks_t( + C_NULL, + C_NULL, + C_NULL, + C_NULL, + C_NULL, + C_NULL, + C_NULL) + r = Ref(cb) + h5p_get_file_image_callbacks(fapl_id, r) + return r[] +end + + # Note: The following function(s) implement direct ccalls because the binding generator # cannot (yet) do the string wrapping and memory freeing. From f616e00581bcaafc58e8538f341efda7664a9844 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Fri, 1 Sep 2023 04:17:53 -0400 Subject: [PATCH 4/5] Formatting --- src/api/helpers.jl | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/api/helpers.jl b/src/api/helpers.jl index 889838703..6e4d0008c 100644 --- a/src/api/helpers.jl +++ b/src/api/helpers.jl @@ -819,7 +819,9 @@ Retrieve a file image of the appropriate size in a `Vector{UInt8}`. function h5p_get_file_image(fapl_id)::Vector{UInt8} cb = h5p_get_file_image_callbacks(fapl_id) if cb.image_free != C_NULL - error("File image callback image_free is not C_NULL. Use the three argument method of h5p_get_file_image when setting file image callbacks.") + error( + "File image callback image_free is not C_NULL. Use the three argument method of h5p_get_file_image when setting file image callbacks." + ) end buf_ptr_ref = Ref{Ptr{Nothing}}() buf_len_ref = Ref{Csize_t}(0) @@ -842,20 +844,12 @@ end Retrieve the file image callbacks for memory operations """ function h5p_get_file_image_callbacks(fapl_id) - cb = H5FD_file_image_callbacks_t( - C_NULL, - C_NULL, - C_NULL, - C_NULL, - C_NULL, - C_NULL, - C_NULL) + cb = H5FD_file_image_callbacks_t(C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL) r = Ref(cb) h5p_get_file_image_callbacks(fapl_id, r) return r[] end - # Note: The following function(s) implement direct ccalls because the binding generator # cannot (yet) do the string wrapping and memory freeing. From 5acf80449b7fa619a33f31f78669bc9165dce14b Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Fri, 1 Sep 2023 22:14:54 -0400 Subject: [PATCH 5/5] Use h5_free_memory for h5p_get_file_image helper --- src/api/helpers.jl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/api/helpers.jl b/src/api/helpers.jl index 6e4d0008c..b97bced26 100644 --- a/src/api/helpers.jl +++ b/src/api/helpers.jl @@ -819,6 +819,8 @@ Retrieve a file image of the appropriate size in a `Vector{UInt8}`. function h5p_get_file_image(fapl_id)::Vector{UInt8} cb = h5p_get_file_image_callbacks(fapl_id) if cb.image_free != C_NULL + # The user has configured their own memory deallocation routines. + # The user should use a lower level call to properly handle deallocation error( "File image callback image_free is not C_NULL. Use the three argument method of h5p_get_file_image when setting file image callbacks." ) @@ -826,7 +828,12 @@ function h5p_get_file_image(fapl_id)::Vector{UInt8} buf_ptr_ref = Ref{Ptr{Nothing}}() buf_len_ref = Ref{Csize_t}(0) h5p_get_file_image(fapl_id, buf_ptr_ref, buf_len_ref) - unsafe_wrap(Array{UInt8}, Ptr{UInt8}(buf_ptr_ref[]), buf_len_ref[]; own=true) + image = unsafe_wrap(Array{UInt8}, Ptr{UInt8}(buf_ptr_ref[]), buf_len_ref[]; own=false) + finalizer(image) do image + # Use h5_free_memory to ensure we are using the correct free + h5_free_memory(image) + end + return image end """