Skip to content

Commit

Permalink
Merge pull request #587 from JuliaIO/dev
Browse files Browse the repository at this point in the history
v0.4.53
  • Loading branch information
JonasIsensee authored Sep 1, 2024
2 parents 99782ad + a8a60e4 commit b9fdf3e
Show file tree
Hide file tree
Showing 17 changed files with 303 additions and 311 deletions.
1 change: 1 addition & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ updates:
directory: "/" # Location of package manifests
schedule:
interval: "weekly"
target-branch: "dev"
21 changes: 21 additions & 0 deletions .github/workflows/Docs.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
name: Documenter
on:
push:
branches: [main, master, dev]
tags: [v*]
pull_request:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
Documenter:
permissions:
contents: write
statuses: write
name: Documentation
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: julia-actions/julia-docdeploy@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
15 changes: 8 additions & 7 deletions .github/workflows/Downgrade.yml
Original file line number Diff line number Diff line change
@@ -1,15 +1,16 @@
name: Downgrade
on:
pull_request:
branches:
- master
branches: [main, master, dev]
paths-ignore:
- 'docs/**'
push:
branches:
- master
branches: [main, master, dev]
paths-ignore:
- 'docs/**'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
test:
runs-on: ubuntu-latest
Expand All @@ -18,11 +19,11 @@ jobs:
version: ['1']
steps:
- uses: actions/checkout@v4
- uses: julia-actions/setup-julia@v1
- uses: julia-actions/setup-julia@v2
with:
version: ${{ matrix.version }}
- uses: cjdoris/julia-downgrade-compat-action@v1
- uses: julia-actions/julia-downgrade-compat@v1
with:
skip: Pkg,TOML,Mmap
skip: Pkg, TOML, Mmap
- uses: julia-actions/julia-buildpkg@v1
- uses: julia-actions/julia-runtest@v1
4 changes: 1 addition & 3 deletions .github/workflows/Invalidations.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ name: Invalidations

on:
pull_request:
branches: [main, master, dev]

concurrency:
# Skip intermediate builds: always.
Expand All @@ -11,9 +12,6 @@ concurrency:

jobs:
evaluate:
# Only run on PRs to the default branch.
# In the PR trigger above branches can be specified only explicitly whereas this check should work for master, main, or any other default branch
if: github.base_ref == github.event.repository.default_branch
runs-on: ubuntu-latest
steps:
- uses: julia-actions/setup-julia@v2
Expand Down
56 changes: 8 additions & 48 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -1,40 +1,28 @@
name: CI
on:
pull_request:
branches:
- master
- dev
branches: [main, master, dev]
push:
branches:
- master
- dev
branches: [main, master, dev]
tags: '*'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
test:
name: Tests, Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
version:
- '1.6'
- '1'
- 'nightly'
os: [ubuntu-latest, windows-latest, macos-latest] # adjust according to need, e.g. os: [ubuntu-latest] if testing only on linux
arch:
- x64
version: ['min', '1', 'nightly']
os: [ubuntu-latest, windows-latest, macos-latest]
arch: [x64]
include:
- os: ubuntu-latest
version: '1'
arch: x86
steps:
# Cancel ongoing CI test runs if pushing to branch again before the previous tests
# have finished
- name: Cancel ongoing test runs for previous commits
uses: styfle/[email protected]
with:
access_token: ${{ github.token }}

# Do tests
- uses: actions/checkout@v4
- uses: julia-actions/setup-julia@v2
Expand All @@ -49,31 +37,3 @@ jobs:
with:
token: ${{ secrets.CODECOV_TOKEN }}
file: lcov.info
docs:
name: Documentation
runs-on: ubuntu-latest
steps:
# Cancel ongoing documentation build if pushing to branch again before the previous
# build is finished.
- name: Cancel ongoing documentation builds for previous commits
uses: styfle/[email protected]
with:
access_token: ${{ github.token }}

# Build docs
- uses: actions/checkout@v4
- uses: julia-actions/setup-julia@v2
with:
version: '1'
- name: Instantiate and install dependencies
run: |
julia --project=docs -e '
using Pkg
Pkg.develop(PackageSpec(path=pwd()))
Pkg.instantiate()'
- name: Generate documentation and deploy
env: # needed for pushing to gh-pages branch
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # For authentication with SSH deploy key
run:
julia --project=docs docs/make.jl
71 changes: 71 additions & 0 deletions .github/workflows/stale_preview_removal.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
name: Doc Preview Cleanup

on:
schedule:
- cron: "0 0 * * *"

jobs:
doc-preview-cleanup:
runs-on: ubuntu-latest
steps:
- name: Checkout gh-pages branch
uses: actions/checkout@v4
with:
ref: gh-pages
- uses: julia-actions/setup-julia@v2
- name: Check for stale PR previews
shell: julia {0}
run: |
using Pkg
pkg"activate --temp"
pkg"add HTTP JSON3"
using HTTP
using JSON3
using Dates
repo = ENV["GITHUB_REPOSITORY"]
retention_days = 14
pr_previews = map(filter(startswith("PR"), readdir("previews"))) do dir
parse(Int, match(r"PR(\d*)", dir)[1])
end
function all_prs()
query_prs(page) = JSON3.read(HTTP.get("https://api.github.com/repos/$repo/pulls?per_page=100;page=$(page)").body)
prs = []
page = 1
while true
page_prs = query_prs(page)
isempty(page_prs) && break
append!(prs, page_prs)
page += 1
end
return prs
end
prs = all_prs()
open_within_threshold = map(x -> x.number, filter(prs) do pr
time = DateTime(pr.updated_at[1:19], ISODateTimeFormat)
return pr.state == "open" && Dates.days(now() - time) <= retention_days
end)
stale_previews = setdiff(pr_previews, open_within_threshold)
@info "Found $(length(stale_previews)) stale previews"
if isempty(stale_previews)
@info "No stale previews"
exit(1)
end
for pr in stale_previews
path = joinpath("previews", "PR$pr")
@info "Removing $path"
run(`git rm -rf $path`)
end
- name: Push changes
run: |
git config user.name "Documenter.jl"
git config user.email "[email protected]"
git commit -m "delete preview"
git branch gh-pages-new $(echo "delete history" | git commit-tree HEAD^{tree})
git push --force origin gh-pages-new:gh-pages
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
## 0.4.53
- Experimental: Slicing and inplace updating of array datasets
- updated CI workflows
- improve pretty printing of attribute header message
- fix storing of datatype info for h5 compat

## 0.4.52
- fix attribute loading
- new features: `readmmap` `ismmappable` and `allocate_early` (api experimental)
Expand Down
2 changes: 1 addition & 1 deletion Project.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
name = "JLD2"
uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
version = "0.4.52"
version = "0.4.53"

[deps]
FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
Expand Down
12 changes: 6 additions & 6 deletions src/data/number_types.jl
Original file line number Diff line number Diff line change
Expand Up @@ -67,18 +67,18 @@ h5fieldtype(::JLDFile, ::Type{Bool}, ::Type{Bool}, ::Initialized) =BitFieldDatat
jltype(::JLDFile, ::BitFieldDatatype) = ReadRepresentation{Bool, Bool}()

h5fieldtype(::JLDFile, ::Type{Float16}, ::Type{Float16}, ::Initialized) =
FloatingPointDatatype(DT_FLOATING_POINT, 0x20, 0x0f, 0x00, 2, 0, 16, 10, 5, 0, 10, 0x0000000f)
FloatingPointDatatype(UInt8(DT_FLOATING_POINT) + 0x3<<4, 0x20, 0x0f, 0x00, 2, 0, 16, 10, 5, 0, 10, 0x0000000f)
h5fieldtype(::JLDFile, ::Type{Float32}, ::Type{Float32}, ::Initialized) =
FloatingPointDatatype(DT_FLOATING_POINT, 0x20, 0x1f, 0x00, 4, 0, 32, 23, 8, 0, 23, 0x0000007f)
FloatingPointDatatype(UInt8(DT_FLOATING_POINT) + 0x3<<4, 0x20, 0x1f, 0x00, 4, 0, 32, 23, 8, 0, 23, 0x0000007f)
h5fieldtype(::JLDFile, ::Type{Float64}, ::Type{Float64}, ::Initialized) =
FloatingPointDatatype(DT_FLOATING_POINT, 0x20, 0x3f, 0x00, 8, 0, 64, 52, 11, 0, 52, 0x000003ff)
FloatingPointDatatype(UInt8(DT_FLOATING_POINT) + 0x3<<4, 0x20, 0x3f, 0x00, 8, 0, 64, 52, 11, 0, 52, 0x000003ff)

h5fieldtype(::JLDFile, ::Type{BENumber{Float16}}, ::Type{Float16}, ::Initialized) =
FloatingPointDatatype(DT_FLOATING_POINT, 0x21, 0x0f, 0x00, 2, 0, 16, 10, 5, 0, 10, 0x0000000f)
FloatingPointDatatype(UInt8(DT_FLOATING_POINT) + 0x3<<4, 0x21, 0x0f, 0x00, 2, 0, 16, 10, 5, 0, 10, 0x0000000f)
h5fieldtype(::JLDFile, ::Type{BENumber{Float32}}, ::Type{Float32}, ::Initialized) =
FloatingPointDatatype(DT_FLOATING_POINT, 0x21, 0x1f, 0x00, 4, 0, 32, 23, 8, 0, 23, 0x0000007f)
FloatingPointDatatype(UInt8(DT_FLOATING_POINT) + 0x3<<4, 0x21, 0x1f, 0x00, 4, 0, 32, 23, 8, 0, 23, 0x0000007f)
h5fieldtype(::JLDFile, ::Type{BENumber{Float64}}, ::Type{Float64}, ::Initialized) =
FloatingPointDatatype(DT_FLOATING_POINT, 0x21, 0x3f, 0x00, 8, 0, 64, 52, 11, 0, 52, 0x000003ff)
FloatingPointDatatype(UInt8(DT_FLOATING_POINT) + 0x3<<4, 0x21, 0x3f, 0x00, 8, 0, 64, 52, 11, 0, 52, 0x000003ff)

function jltype(f::JLDFile, dt::FloatingPointDatatype)
if dt == h5fieldtype(f, Float64, Float64, Val{true})
Expand Down
55 changes: 18 additions & 37 deletions src/data/reconstructing_datatypes.jl
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,10 @@ end

# jltype is the inverse of h5type, providing a ReadRepresentation for an
# H5Datatype. We handle committed datatypes here, and other datatypes below.
function jltype(f::JLDFile, cdt::CommittedDatatype)
function jltype(f::JLDFile, sdt::Union{SharedDatatype,CommittedDatatype})
cdt = get(f.datatype_locations, sdt.header_offset, sdt)
haskey(f.h5jltype, cdt) && return f.h5jltype[cdt]::ReadRepresentation

dt, attrs = read_shared_datatype(f, cdt)

julia_type_attr = nothing
Expand All @@ -75,51 +77,40 @@ function jltype(f::JLDFile, cdt::CommittedDatatype)
written_type_attr = attr
end
end
isnothing(julia_type_attr) && return f.h5jltype[cdt] = jltype(f, dt)

if isa(julia_type_attr, Nothing)
throw(InvalidDataException())
end
julia_type_attr = julia_type_attr::ReadAttribute

# If type of datatype is this datatype, then this is the committed
# datatype that describes a datatype
if julia_type_attr.datatype isa SharedDatatype &&
julia_type_attr.datatype.header_offset == cdt.header_offset
# Verify that the datatype matches our expectations
# Bootstrap: the datatype of datatype is a datatype
if julia_type_attr.datatype == SharedDatatype(cdt.header_offset)
if dt != H5TYPE_DATATYPE
error("""The HDF5 datatype representing a Julia datatype does not match
the expectations of this version of JLD.
You may need to update JLD to read this file.""")
throw(InternalError("""The HDF5 datatype representing a Julia datatype does not match
the expectations of this version of JLD2.
You may need to update JLD2 to read this file."""))
end
f.jlh5type[DataType] = cdt
f.datatypes[cdt.index] = dt
return (f.h5jltype[cdt] = ReadRepresentation{DataType, DataTypeODR()}())
end

f.plain && return f.h5jltype[cdt] = jltype(f, dt)

datatype = read_attr_data(f, julia_type_attr)
if f.plain && !(datatype isa Upgrade) && !(datatype <: Tuple)
rr = jltype(f, dt)
return f.h5jltype[cdt] = rr
end

if written_type_attr !== nothing
if !isnothing(written_type_attr)
# Custom serialization
custom_datatype = read_attr_data(f, written_type_attr)
read_as = _readas(custom_datatype, datatype)
if read_as <: UnknownType
@warn("custom serialization of $(typestring(read_as))" *
" encountered, but the type does not exist in the workspace; the data will be read unconverted")
rr = (constructrr(f, custom_datatype, dt, attrs)::Tuple{ReadRepresentation,Bool})[1]
rr, _ = constructrr(f, custom_datatype, dt, attrs)
canonical = false
else
rr, canonical = constructrr(f, custom_datatype, dt, attrs)::Tuple{ReadRepresentation,Bool}
rrty = typeof(rr)
rr = ReadRepresentation{read_as, CustomSerialization{rrty.parameters[1], rrty.parameters[2]}}()
canonical = canonical && writeas(read_as) === custom_datatype
rr, canonical = constructrr(f, custom_datatype, dt, attrs)
rr = ReadRepresentation{read_as, CustomSerialization{typeof(rr).parameters...}}()
canonical &= writeas(read_as) === custom_datatype
end
else
rr, canonical = constructrr(f, datatype, dt, attrs)::Tuple{ReadRepresentation,Bool}
rr, canonical = constructrr(f, datatype, dt, attrs)
end

canonical && (f.jlh5type[datatype] = cdt)
Expand All @@ -128,16 +119,6 @@ function jltype(f::JLDFile, cdt::CommittedDatatype)
end


# jltype is the inverse of h5type, providing a ReadRepresentation for an
# H5Datatype. We handle shared datatypes here: ones that were not "committed" by JLD2.
function jltype(f::JLDFile, sdt::SharedDatatype)
haskey(f.h5jltype, sdt) && return f.h5jltype[sdt]::ReadRepresentation
dt, attrs = read_shared_datatype(f, sdt)
rr = jltype(f, dt)
f.h5jltype[sdt] = rr
end



# Constructs a ReadRepresentation for a given opaque (bitstype) type
function constructrr(::JLDFile, T::DataType, dt::BasicDatatype, attrs::Vector{ReadAttribute})
Expand Down Expand Up @@ -381,7 +362,7 @@ function types_from_refs(f::JLDFile, ptr::Ptr)
# If the reference is to a committed datatype, read the datatype
nulldt = CommittedDatatype(UNDEFINED_ADDRESS, 0)
cdt = get(f.datatype_locations, ref, nulldt)
res = cdt !== nulldt ? (typeof(jltype(f, cdt)::ReadRepresentation)::DataType).parameters[1] : load_dataset(f, ref)
res = cdt !== nulldt ? eltype(jltype(f, cdt)) : load_dataset(f, ref)
unknown_params = unknown_params || isunknowntype(res) || isreconstructed(res)
res
end for ref in refs]
Expand Down
Loading

2 comments on commit b9fdf3e

@JonasIsensee
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@JuliaRegistrator
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Registration pull request created: JuliaRegistries/General/114297

Tip: Release Notes

Did you know you can add release notes too? Just add markdown formatted text underneath the comment after the text
"Release notes:" and it will be added to the registry PR, and if TagBot is installed it will also be added to the
release that TagBot creates. i.e.

@JuliaRegistrator register

Release notes:

## Breaking changes

- blah

To add them here just re-invoke and the PR will be updated.

Tagging

After the above pull request is merged, it is recommended that a tag is created on this repository for the registered package version.

This will be done automatically if the Julia TagBot GitHub Action is installed, or can be done manually through the github interface, or via:

git tag -a v0.4.53 -m "<description of version>" b9fdf3e03a947135583d56d49e1cc43ca561fa2f
git push origin v0.4.53

Please sign in to comment.