diff --git a/src/api/helpers.jl b/src/api/helpers.jl index 441c1aa4e..42730aa26 100644 --- a/src/api/helpers.jl +++ b/src/api/helpers.jl @@ -30,6 +30,27 @@ function h5f_get_dset_no_attrs_hint(file_id)::Bool return minimize[] end +""" + h5f_get_file_image(file_id) + +Return a `Vector{UInt8}` containing the file image. Does not include the user block. +""" +function h5f_get_file_image(file_id) + buffer_length = h5f_get_file_image(file_id, C_NULL, 0) + buffer = Vector{UInt8}(undef, buffer_length) + h5f_get_file_image(file_id, buffer, buffer_length) + return buffer +end + +""" + h5f_get_file_image(file_id, buffer::Vector{UInt8}) + +Store the file image in the provided buffer. +""" +function h5f_get_file_image(file_id, buffer::Vector{UInt8}) + h5f_get_file_image(fild_id, buffer, length(buffer)) +end + ### ### Attribute Interface ### @@ -790,6 +811,27 @@ function h5p_get_virtual_view(dapl_id) return view[] end +""" + h5p_get_file_image(fapl_id)::Vector{UInt8} + +Retrieve a file image of the appropriate size in a `Vector{UInt8}`. +""" +function h5p_get_file_image(fapl_id)::Vector{UInt8} + buf_ptr_ref = Ref{Ptr{Nothing}}() + buf_len_ref = Ref{Csize_t}(0) + h5p_get_file_image(fapl_id, buf_ptr_ref, buf_len_ref) + unsafe_wrap(Array{UInt8}, Ptr{UInt8}(buf_ptr_ref[]), buf_len_ref[]; own=true) +end + +""" + h5p_set_file_image(fapl_id, image::Vector{UInt8}) + +Set the file image from a `Vector{UInt8}`. +""" +function h5p_set_file_image(fapl_id, image::Vector{UInt8}) + h5p_set_file_image(fapl_id, image, length(image)) +end + # Note: The following function(s) implement direct ccalls because the binding generator # cannot (yet) do the string wrapping and memory freeing. diff --git a/src/drivers/ros3.jl b/src/drivers/ros3.jl index 85f762a26..33090d97b 100644 --- a/src/drivers/ros3.jl +++ b/src/drivers/ros3.jl @@ -54,7 +54,7 @@ function Base.convert(::Type{API.H5FD_ros3_fapl_t}, driver::ROS3) end function get_driver(fapl::Properties, ::Type{ROS3}) - r_fa = Ref{H5FD_ros3_fapl_t}() + r_fa = Ref{API.H5FD_ros3_fapl_t}() API.h5p_get_fapl_ros3(fapl, r_fa) return ROS3(r_fa[]) end diff --git a/src/file.jl b/src/file.jl index d8c151b53..6d1d9875a 100644 --- a/src/file.jl +++ b/src/file.jl @@ -41,6 +41,9 @@ function h5open( if cr && (tr || !isfile(filename)) flag = swmr ? API.H5F_ACC_TRUNC | API.H5F_ACC_SWMR_WRITE : API.H5F_ACC_TRUNC fid = API.h5f_create(filename, flag, fcpl, fapl) + elseif fapl.driver isa Drivers.Core && fapl.driver.backing_store == 0 + flag = wr ? API.H5F_ACC_RDWR : API.H5F_ACC_RDONLY + fid = API.h5f_open(filename, flag, fapl) else occursin(r"(s3a?|https?)://", filename) || ishdf5(filename) || @@ -77,7 +80,7 @@ function h5open( end """ - function h5open(f::Function, args...; pv...) + h5open(f::Function, args...; pv...) Apply the function f to the result of `h5open(args...; kwargs...)` and close the resulting `HDF5.File` upon completion. @@ -103,6 +106,46 @@ function h5open(f::Function, args...; context=copy(CONTEXT), pv...) end end +""" + h5open(file_image::Vector{UInt8}, mode::AbstractString="r"; + name=nothing, + fapl=FileAccessProperties(), + increment=8192, + backing_store=false, + write_tracking=false, + page_size=524288, + pv... + ) + +Open a file image contained in a `Vector{UInt8}`. See [`API.h5p_set_file_image`](@ref). + +Unlike [`Drivers.Core`](@ref) the default here is not to use a backing store. +""" +function h5open( + file_image::Vector{UInt8}, + mode::AbstractString="r"; + name=nothing, + fapl=FileAccessProperties(), + increment=8192, + backing_store=false, + write_tracking=false, + page_size=524288, + pv... +) + fapl.driver = Drivers.Core(; increment, backing_store, write_tracking, page_size) + fapl.file_image = file_image + if isnothing(name) + if fapl.driver.backing_store != 0 + # The temporary file will be used as a backing store + name = tempname() + else + # Provide it with a unique name based on the objectid + name = ": " * repr(objectid(file_image)) + end + end + h5open(name, mode; fapl, pv...) +end + function h5rewrite(f::Function, filename::AbstractString, args...) tmppath, tmpio = mktemp(dirname(filename)) close(tmpio) @@ -169,3 +212,15 @@ start_swmr_write(h5::File) = API.h5f_start_swmr_write(h5) # Flush buffers Base.flush(f::Union{Object,Attribute,Datatype,File}, scope=API.H5F_SCOPE_GLOBAL) = API.h5f_flush(checkvalid(f), scope) + +# File image conversion + +function Vector{UInt8}(h5f::File) + flush(h5f) + API.h5f_get_file_image(h5f) +end +function File(file_image::Vector{UInt8}, name=nothing) + h5open(file_image; name) +end +Base.convert(::Type{Vector{UInt8}}, h5f::File) = Vector{UInt8}(h5f) +Base.convert(::Type{File}, file_image::Vector{UInt8}) = File(file_image) diff --git a/src/properties.jl b/src/properties.jl index 48a57d14a..01df0d457 100644 --- a/src/properties.jl +++ b/src/properties.jl @@ -743,6 +743,7 @@ class_propertynames(::Type{FileAccessProperties}) = ( :file_locking, :libver_bounds, :meta_block_size, + :file_image, ) function class_getproperty(::Type{FileAccessProperties}, p::Properties, name::Symbol) @@ -753,6 +754,7 @@ function class_getproperty(::Type{FileAccessProperties}, p::Properties, name::Sy name === :file_locking ? API.h5p_get_file_locking(p) : name === :libver_bounds ? get_libver_bounds(p) : name === :meta_block_size ? API.h5p_get_meta_block_size(p) : + name === :file_image ? API.h5p_get_file_image(p) : # deprecated name === :fapl_mpio ? (depwarn("The `fapl_mpio` property is deprecated, use `driver=HDF5.Drivers.MPIO(...)` instead.", :fapl_mpio); drv = get_driver(p, MPIO); (drv.comm, drv.info)) : class_getproperty(superclass(FileAccessProperties), p, name) @@ -764,6 +766,7 @@ function class_setproperty!(::Type{FileAccessProperties}, p::Properties, name::S name === :file_locking ? API.h5p_set_file_locking(p, val...) : name === :libver_bounds ? set_libver_bounds!(p, val) : name === :meta_block_size ? API.h5p_set_meta_block_size(p, val) : + name === :file_image ? API.h5p_set_file_image(p, val) : # deprecated name === :fapl_mpio ? (depwarn("The `fapl_mpio` property is deprecated, use `driver=HDF5.Drivers.MPIO(...)` instead.", :fapl_mpio); Drivers.set_driver!(p, Drivers.MPIO(val...))) : class_setproperty!(superclass(FileAccessProperties), p, name, val) diff --git a/src/show.jl b/src/show.jl index 833c42652..51f7b49c1 100644 --- a/src/show.jl +++ b/src/show.jl @@ -36,7 +36,20 @@ function Base.show(io::IO, prop::Properties) # or always well-defined (e.g. chunk if layout != :chunked, dxpl_mpio if no MPI) try val = getproperty(prop, name) - print(io, "\n ", rpad(name, 15), " = ", repr(val), ",") + if name == :file_image + print( + io, + "\n ", + rpad(name, 15), + " = ", + repr(typeof(val)), + ", length = ", + length(val), + "," + ) + else + print(io, "\n ", rpad(name, 15), " = ", repr(val), ",") + end catch e end end diff --git a/test/drivers.jl b/test/drivers.jl index 709871b60..16ad93a15 100644 --- a/test/drivers.jl +++ b/test/drivers.jl @@ -14,8 +14,18 @@ using Test end fn = tempname() - h5open(fn, "w"; driver=Drivers.Core(; backing_store=false)) do f + file_image = h5open(fn, "w"; driver=Drivers.Core(; backing_store=false)) do f ds = write_dataset(f, "core_dataset", A) + Vector{UInt8}(f) end @test !isfile(fn) + + h5open(file_image) do f + @test f["core_dataset"][] == A + end + + fn = tempname() + h5open(fn, "r"; driver=Drivers.Core(; backing_store=false), file_image) do f + @test f["core_dataset"][] == A + end end diff --git a/test/properties.jl b/test/properties.jl index ac0ffb32c..81353ff5f 100644 --- a/test/properties.jl +++ b/test/properties.jl @@ -121,5 +121,20 @@ using Test nothing end + file_image = read(fn) + + rm(fn; force=true) + + fapl = HDF5.FileAccessProperties() + fapl.driver = HDF5.Drivers.Core(; backing_store=false) + fapl.file_image = copy(file_image) + @test fapl.file_image == file_image + file_image2 = h5open(fn, "r"; fapl) do f + @test haskey(f, "group") + convert(Vector{UInt8}, f) + end + # file_image2 does not include user block + @test file_image2 == @view(file_image[1025:end]) + rm(fn; force=true) end