From c7df0bc9e01fd94c38b29f77fc6e988a7a13a82d Mon Sep 17 00:00:00 2001 From: rafaqz Date: Mon, 12 Aug 2024 00:43:26 +0200 Subject: [PATCH] bugfix GRIB --- .../gribdatasets_source.jl | 11 +++++---- ext/RastersNCDatasetsExt/ncdatasets_source.jl | 24 +++++++++++-------- src/methods/rasterize.jl | 8 ++++--- src/sources/commondatamodel.jl | 20 ++++++++++------ src/stack.jl | 3 ++- src/utils.jl | 13 +++++----- test/sources/gribdatasets.jl | 7 ++++-- 7 files changed, 53 insertions(+), 33 deletions(-) diff --git a/ext/RastersGRIBDatasetsExt/gribdatasets_source.jl b/ext/RastersGRIBDatasetsExt/gribdatasets_source.jl index 25df0317..f3824cbd 100644 --- a/ext/RastersGRIBDatasetsExt/gribdatasets_source.jl +++ b/ext/RastersGRIBDatasetsExt/gribdatasets_source.jl @@ -1,4 +1,5 @@ const GDS = GRIBDatasets +const CDM = CommonDataModel function RA.OpenStack(fs::RA.FileStack{GRIBsource,K}) where K RA.OpenStack{GRIBsource,K}(GDS.GRIBDataset(RA.filename(fs))) @@ -13,9 +14,11 @@ function RA._open(f, ::GRIBsource, filename::AbstractString; write=false, kw...) RA._open(f, GRIBsource(), ds; kw...) end -# Hack to get the inner DiskArrays chunks as they are not exposed at the top level -RA._get_eachchunk(var::GDS.Variable) = DiskArrays.eachchunk(var.values) -RA._get_haschunks(var::GDS.Variable) = DiskArrays.haschunks(var.values) - RA._sourcetrait(::GDS.Variable) = GRIBsource() RA._sourcetrait(::GDS.GRIBDataset) = GRIBsource() + +function RA.missingval(var::GDS.Variable{T}, args...) where T + mv = GDS.missing_value(var) + T1 = promote_type(typeof(mv), T) + return T1(mv) +end diff --git a/ext/RastersNCDatasetsExt/ncdatasets_source.jl b/ext/RastersNCDatasetsExt/ncdatasets_source.jl index 434de506..00f86229 100644 --- a/ext/RastersNCDatasetsExt/ncdatasets_source.jl +++ b/ext/RastersNCDatasetsExt/ncdatasets_source.jl @@ -49,10 +49,11 @@ function Base.write(filename::AbstractString, ::NCDsource, s::AbstractRasterStac return filename end +Base.close(os::RA.OpenStack{NCDsource}) = NCD.close(RA.dataset(os)) + function RA.OpenStack(fs::RA.FileStack{NCDsource,K}) where K RA.OpenStack{NCDsource,K}(NCD.Dataset(RA.filename(fs))) end -Base.close(os::RA.OpenStack{NCDsource}) = NCD.close(RA.dataset(os)) function RA._open(f, ::NCDsource, filename::AbstractString; write=false, kw...) isfile(filename) || RA._isurl(filename) || RA._filenotfound_error(filename) @@ -62,6 +63,18 @@ function RA._open(f, ::NCDsource, filename::AbstractString; write=false, kw...) end end +RA._sourcetrait(::NCD.Dataset) = NCDsource() +RA._sourcetrait(::NCD.Variable) = NCDsource() + +@inline function RA.get_scale(metadata::Metadata{NCDsource}, scaled::Bool) + scale = scaled ? get(metadata, "scale_factor", nothing) : nothing + offset = scaled ? get(metadata, "add_offset", nothing) : nothing + return scale, offset +end + +RA.missingval(var::NCD.Variable, args...) = missingval(CDM.attribs(var), "_FillValue", nothing) +RA.missingval(var::NCD.Variable, md::Metadata{<:NCDsource}) = get(md, "_FillValue", nothing) + # Add a var array to a dataset before writing it. function _writevar!(ds::AbstractDataset, A::AbstractRaster{T,N}; verbose=true, @@ -150,15 +163,6 @@ function _def_dim_var!(ds::AbstractDataset, dim::Dimension) return nothing end -RA._sourcetrait(::NCD.Dataset) = NCDsource() -RA._sourcetrait(::NCD.Variable) = NCDsource() - -@inline function RA.get_scale(metadata::Metadata{NCDsource}, scaled::Bool) - scale = scaled ? get(metadata, "scale_factor", nothing) : nothing - offset = scaled ? get(metadata, "add_offset", nothing) : nothing - return scale, offset -end - # precompilation # const _NCDVar = NCDatasets.CFVariable{Union{Missing, Float32}, 3, NCDatasets.Variable{Float32, 3, NCDatasets.NCDataset}, NCDatasets.Attributes{NCDatasets.NCDataset{Nothing}}, NamedTuple{(:fillvalue, :scale_factor, :add_offset, :calendar, :time_origin, :time_factor), Tuple{Float32, Nothing, Nothing, Nothing, Nothing, Nothing}}} diff --git a/src/methods/rasterize.jl b/src/methods/rasterize.jl index c391018d..3cebe1b3 100644 --- a/src/methods/rasterize.jl +++ b/src/methods/rasterize.jl @@ -30,7 +30,7 @@ _reduce_init(::typeof(sum), ::Type{T}, missingval) where T = zero(nonmissingtype _reduce_init(::typeof(prod), ::Type{T}, missingval) where T = oneunit(nonmissingtype(T)) _reduce_init(::typeof(minimum), ::Type{T}, missingval) where T = typemax(nonmissingtype(T)) _reduce_init(::typeof(maximum), ::Type{T}, missingval) where T = typemin(nonmissingtype(T)) -_reduce_init(::typeof(last), ::Type{T}, missingval) where T = _maybe_nothing_to_missing(missingval) +_reduce_init(::typeof(last), ::Type{T}, missingval) where T = _maybe_to_missing(missingval) struct FillChooser{F,I,M} fill::F @@ -73,10 +73,12 @@ RasterCreator(to, data; kw...) = RasterCreator(_extent(to); kw...) function RasterCreator(to::Extents.Extent; res::Union{Nothing,Real,NTuple{<:Any,<:Real}}=nothing, size::Union{Nothing,Int,NTuple{<:Any,Int}}=nothing, + crs=nokw, + mappedcrs=nokw, kw... ) - to_as_dims = _extent2dims(to; size, res) - return RasterCreator(to_as_dims; kw...) + to_as_dims = _extent2dims(to; size, res, crs, mappedcrs) + return RasterCreator(to_as_dims; crs, mappedcrs, kw...) end diff --git a/src/sources/commondatamodel.jl b/src/sources/commondatamodel.jl index 802e570b..f627a1cb 100644 --- a/src/sources/commondatamodel.jl +++ b/src/sources/commondatamodel.jl @@ -53,10 +53,18 @@ function FileStack{source}(ds::AbstractDataset, filename::AbstractString; end function _open(f, ::CDMsource, ds::AbstractDataset; - name=nokw, group=nothing, mod=NoMod(), kw... + name=nokw, + group=nothing, + mod=NoMod(), + kw... ) g = _getgroup(ds, group) - x = isnokw(name) ? g : _maybe_modify(CDM.variable(g, _firstname(g, name)), mod) + x = if isnokw(name) + g + else + v = CDM.variable(g, string(_name_or_firstname(g, name))) + _maybe_modify(v, mod) + end return cleanreturn(f(x)) end _open(f, ::CDMsource, var::AbstractArray; mod=NoMod(), kw...) = @@ -68,9 +76,7 @@ _getgroup(ds, group::Union{Symbol,AbstractString}) = ds.group[String(group)] _getgroup(ds, group::Pair) = _getgroup(ds.group[String(group[1])], group[2]) filekey(ds::AbstractDataset, name::Union{String,Symbol}) = Symbol(name) -filekey(ds::AbstractDataset, name) = _firstname(ds, name) -missingval(var::AbstractVariable, args...) = get(CDM.attribs(var), "_FillValue", nothing) -missingval(var::AbstractVariable, md::Metadata{<:CDMsource}) = get(md, "_FillValue", nothing) +filekey(ds::AbstractDataset, name) = _name_or_firstname(ds, name) cleanreturn(A::AbstractVariable) = Array(A) haslayers(::CDMsource) = true defaultcrs(::CDMsource) = EPSG(4326) @@ -166,8 +172,8 @@ end # Utils ######################################################################## # TODO don't load all keys here with _layers -_firstname(ds::AbstractDataset, name) = Symbol(name) -function _firstname(ds::AbstractDataset, name::Union{Nothing,NoKW}=nokw) +_name_or_firstname(ds::AbstractDataset, name) = Symbol(name) +function _name_or_firstname(ds::AbstractDataset, name::Union{Nothing,NoKW}=nokw) names = _nondimnames(ds) if length(names) > 0 return Symbol(first(names)) diff --git a/src/stack.jl b/src/stack.jl index 269a0193..6a6f103c 100644 --- a/src/stack.jl +++ b/src/stack.jl @@ -544,8 +544,9 @@ function _layer_stack(filename; else missingval end + maskingval1 = isnokw(maskingval) && !isnothing(missingval1) ? missing : maskingval eltypes = map(eltype, layers.vars) - mods = _stack_mods(eltypes, layermetadata1, missingval1, maskingval; scaled, coerce) + mods = _stack_mods(eltypes, layermetadata1, missingval1, maskingval1; scaled, coerce) name = Tuple(map(Symbol, layers.names)) NT = NamedTuple{name} data = if lazy diff --git a/src/utils.jl b/src/utils.jl index 904c7dd3..b5047861 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -113,10 +113,10 @@ end # Extents function _extent2dims(to::Extents.Extent; - size=nokw, res=nokw, crs=nokw, sampling=nokw, + size=nokw, res=nokw, crs=nokw, mappedcrs=nokw, sampling=nokw, ) sampling = _match_to_extent(to, isnokw(sampling) ? Intervals(Start()) : sampling) - _extent2dims(to, size, res; crs, sampling) + _extent2dims(to, size, res; crs, mappedcrs, sampling) end function _extent2dims(to::Extents.Extent, size::Union{Nothing,NoKW}, res::Union{Nothing,NoKW}; kw...) throw(ArgumentError("Pass either `size` or `res` keywords or a `Tuple` of `Dimension`s for `to`.")) @@ -149,7 +149,7 @@ end _extent2dims(to::Extents.Extent, size, res::Union{Nothing,NoKW}; kw...) = _extent2dims(to, _match_to_extent(to, size), res; kw...) function _extent2dims(to::Extents.Extent, size::Tuple, res::Union{Nothing,NoKW}; - sampling::Tuple, crs + sampling::Tuple, crs, mappedcrs ) ranges = map(values(to), size, sampling) do (start, stop), length, s if s isa Points @@ -158,16 +158,17 @@ function _extent2dims(to::Extents.Extent, size::Tuple, res::Union{Nothing,NoKW}; range(; start, stop, length=length+1)[1:end-1] end end - return _extent2dims(to, ranges; sampling, crs) + return _extent2dims(to, ranges; sampling, crs, mappedcrs) end -function _extent2dims(::Extents.Extent{K}, ranges; crs, sampling::Tuple) where K +function _extent2dims(::Extents.Extent{K}, ranges; crs, mappedcrs, sampling::Tuple) where K crs = isnokw(crs) ? nothing : crs + mappedcrs = isnokw(mappedcrs) ? nothing : mappedcrs emptydims = map(name2dim, K) lookups = map(emptydims, ranges, sampling) do d, range, s order = Lookups.orderof(range) span = Regular(step(range)) if d isa SpatialDim && !isnothing(crs) - Projected(range; sampling=s, order, span, crs) + Projected(range; sampling=s, order, span, crs, mappedcrs) else Sampled(range; sampling=s, order, span) end diff --git a/test/sources/gribdatasets.jl b/test/sources/gribdatasets.jl index 3779e68d..255984b3 100644 --- a/test/sources/gribdatasets.jl +++ b/test/sources/gribdatasets.jl @@ -25,6 +25,9 @@ gribexamples_dir = abspath(joinpath(dirname(pathof(GRIBDatasets)), "..", "test", era5 = joinpath(gribexamples_dir, "era5-levels-members.grib") +ds = GRIBDatasets.GRIBDataset(era5) +v = ds[:z] + @testset "Raster" begin @time gribarray = Raster(era5) @time lazyarray = Raster(era5; lazy=true) @@ -87,8 +90,8 @@ era5 = joinpath(gribexamples_dir, "era5-levels-members.grib") end @testset "cf attributes" begin - z = lazystack[:z] - @test metadata(z)["standard_name"] == "geopotential" + z = lazystack.z + @test metadata(z)["cfName"] == "geopotential" @test metadata(lazystack)["Conventions"] == "CF-1.7" x = dims(lazystack, :X)