bugfix GRIB

ext/RastersGRIBDatasetsExt/gribdatasets_source.jl

@@ -1,4 +1,5 @@
 const GDS = GRIBDatasets
+const CDM = CommonDataModel
 
 function RA.OpenStack(fs::RA.FileStack{GRIBsource,K}) where K
     RA.OpenStack{GRIBsource,K}(GDS.GRIBDataset(RA.filename(fs)))
@@ -13,9 +14,11 @@ function RA._open(f, ::GRIBsource, filename::AbstractString; write=false, kw...)
     RA._open(f, GRIBsource(), ds; kw...)
 end
 
-# Hack to get the inner DiskArrays chunks as they are not exposed at the top level
-RA._get_eachchunk(var::GDS.Variable) = DiskArrays.eachchunk(var.values)
-RA._get_haschunks(var::GDS.Variable) = DiskArrays.haschunks(var.values)
-
 RA._sourcetrait(::GDS.Variable) = GRIBsource()
 RA._sourcetrait(::GDS.GRIBDataset) = GRIBsource()
+
+function RA.missingval(var::GDS.Variable{T}, args...) where T
+    mv = GDS.missing_value(var)
+    T1 = promote_type(typeof(mv), T)
+    return T1(mv)
+end

ext/RastersNCDatasetsExt/ncdatasets_source.jl

@@ -49,10 +49,11 @@ function Base.write(filename::AbstractString, ::NCDsource, s::AbstractRasterStac
     return filename
 end
 
+Base.close(os::RA.OpenStack{NCDsource}) = NCD.close(RA.dataset(os))
+
 function RA.OpenStack(fs::RA.FileStack{NCDsource,K}) where K
     RA.OpenStack{NCDsource,K}(NCD.Dataset(RA.filename(fs)))
 end
-Base.close(os::RA.OpenStack{NCDsource}) = NCD.close(RA.dataset(os))
 
 function RA._open(f, ::NCDsource, filename::AbstractString; write=false, kw...)
     isfile(filename) || RA._isurl(filename) || RA._filenotfound_error(filename)
@@ -62,6 +63,18 @@ function RA._open(f, ::NCDsource, filename::AbstractString; write=false, kw...)
     end
 end
 
+RA._sourcetrait(::NCD.Dataset) = NCDsource()
+RA._sourcetrait(::NCD.Variable) = NCDsource()
+
+@inline function RA.get_scale(metadata::Metadata{NCDsource}, scaled::Bool)
+    scale = scaled ? get(metadata, "scale_factor", nothing) : nothing
+    offset = scaled ? get(metadata, "add_offset", nothing) : nothing
+    return scale, offset
+end
+
+RA.missingval(var::NCD.Variable, args...) = missingval(CDM.attribs(var), "_FillValue", nothing)
+RA.missingval(var::NCD.Variable, md::Metadata{<:NCDsource}) = get(md, "_FillValue", nothing)
+
 # Add a var array to a dataset before writing it.
 function _writevar!(ds::AbstractDataset, A::AbstractRaster{T,N};
     verbose=true,
@@ -150,15 +163,6 @@ function _def_dim_var!(ds::AbstractDataset, dim::Dimension)
     return nothing
 end
 
-RA._sourcetrait(::NCD.Dataset) = NCDsource()
-RA._sourcetrait(::NCD.Variable) = NCDsource()
-
-@inline function RA.get_scale(metadata::Metadata{NCDsource}, scaled::Bool)
-    scale = scaled ? get(metadata, "scale_factor", nothing) : nothing
-    offset = scaled ? get(metadata, "add_offset", nothing) : nothing
-    return scale, offset
-end
-
 # precompilation
 
 # const _NCDVar = NCDatasets.CFVariable{Union{Missing, Float32}, 3, NCDatasets.Variable{Float32, 3, NCDatasets.NCDataset}, NCDatasets.Attributes{NCDatasets.NCDataset{Nothing}}, NamedTuple{(:fillvalue, :scale_factor, :add_offset, :calendar, :time_origin, :time_factor), Tuple{Float32, Nothing, Nothing, Nothing, Nothing, Nothing}}}

src/methods/rasterize.jl

@@ -30,7 +30,7 @@ _reduce_init(::typeof(sum), ::Type{T}, missingval) where T = zero(nonmissingtype
 _reduce_init(::typeof(prod), ::Type{T}, missingval) where T = oneunit(nonmissingtype(T))
 _reduce_init(::typeof(minimum), ::Type{T}, missingval) where T = typemax(nonmissingtype(T))
 _reduce_init(::typeof(maximum), ::Type{T}, missingval) where T = typemin(nonmissingtype(T))
-_reduce_init(::typeof(last), ::Type{T}, missingval) where T = _maybe_nothing_to_missing(missingval)
+_reduce_init(::typeof(last), ::Type{T}, missingval) where T = _maybe_to_missing(missingval)
 
 struct FillChooser{F,I,M}
     fill::F
@@ -73,10 +73,12 @@ RasterCreator(to, data; kw...) = RasterCreator(_extent(to); kw...)
 function RasterCreator(to::Extents.Extent;
     res::Union{Nothing,Real,NTuple{<:Any,<:Real}}=nothing,
     size::Union{Nothing,Int,NTuple{<:Any,Int}}=nothing, 
+    crs=nokw,
+    mappedcrs=nokw,
     kw...
 )
-    to_as_dims = _extent2dims(to; size, res)
-    return RasterCreator(to_as_dims; kw...)
+    to_as_dims = _extent2dims(to; size, res, crs, mappedcrs)
+    return RasterCreator(to_as_dims; crs, mappedcrs, kw...)
 end
 
 

src/sources/commondatamodel.jl

@@ -53,10 +53,18 @@ function FileStack{source}(ds::AbstractDataset, filename::AbstractString;
 end
 
 function _open(f, ::CDMsource, ds::AbstractDataset; 
-    name=nokw, group=nothing, mod=NoMod(), kw...
+    name=nokw, 
+    group=nothing, 
+    mod=NoMod(), 
+    kw...
 )
     g = _getgroup(ds, group)
-    x = isnokw(name) ? g : _maybe_modify(CDM.variable(g, _firstname(g, name)), mod)
+    x = if isnokw(name) 
+        g 
+    else
+        v = CDM.variable(g, string(_name_or_firstname(g, name)))
+        _maybe_modify(v, mod)
+    end
     return cleanreturn(f(x)) 
 end
 _open(f, ::CDMsource, var::AbstractArray; mod=NoMod(), kw...) = 
@@ -68,9 +76,7 @@ _getgroup(ds, group::Union{Symbol,AbstractString}) = ds.group[String(group)]
 _getgroup(ds, group::Pair) = _getgroup(ds.group[String(group[1])], group[2])
 
 filekey(ds::AbstractDataset, name::Union{String,Symbol}) = Symbol(name)
-filekey(ds::AbstractDataset, name) = _firstname(ds, name)
-missingval(var::AbstractVariable, args...) = get(CDM.attribs(var), "_FillValue", nothing)
-missingval(var::AbstractVariable, md::Metadata{<:CDMsource}) = get(md, "_FillValue", nothing)
+filekey(ds::AbstractDataset, name) = _name_or_firstname(ds, name)
 cleanreturn(A::AbstractVariable) = Array(A)
 haslayers(::CDMsource) = true
 defaultcrs(::CDMsource) = EPSG(4326)
@@ -166,8 +172,8 @@ end
 # Utils ########################################################################
 
 # TODO don't load all keys here with _layers
-_firstname(ds::AbstractDataset, name) = Symbol(name)
-function _firstname(ds::AbstractDataset, name::Union{Nothing,NoKW}=nokw)
+_name_or_firstname(ds::AbstractDataset, name) = Symbol(name)
+function _name_or_firstname(ds::AbstractDataset, name::Union{Nothing,NoKW}=nokw)
     names = _nondimnames(ds)
     if length(names) > 0
         return Symbol(first(names))

src/stack.jl

@@ -544,8 +544,9 @@ function _layer_stack(filename;
         else
             missingval
         end
+        maskingval1 = isnokw(maskingval) && !isnothing(missingval1) ? missing : maskingval
         eltypes = map(eltype, layers.vars)
-        mods = _stack_mods(eltypes, layermetadata1, missingval1, maskingval; scaled, coerce)
+        mods = _stack_mods(eltypes, layermetadata1, missingval1, maskingval1; scaled, coerce)
         name = Tuple(map(Symbol, layers.names))
         NT = NamedTuple{name}
         data = if lazy

src/utils.jl

@@ -113,10 +113,10 @@ end
 # Extents
 
 function _extent2dims(to::Extents.Extent; 
-    size=nokw, res=nokw, crs=nokw, sampling=nokw,
+    size=nokw, res=nokw, crs=nokw, mappedcrs=nokw, sampling=nokw,
 )
     sampling = _match_to_extent(to, isnokw(sampling) ? Intervals(Start()) : sampling)
-    _extent2dims(to, size, res; crs, sampling)
+    _extent2dims(to, size, res; crs, mappedcrs, sampling)
 end
 function _extent2dims(to::Extents.Extent, size::Union{Nothing,NoKW}, res::Union{Nothing,NoKW}; kw...)
     throw(ArgumentError("Pass either `size` or `res` keywords or a `Tuple` of `Dimension`s for `to`."))
@@ -149,7 +149,7 @@ end
 _extent2dims(to::Extents.Extent, size, res::Union{Nothing,NoKW}; kw...) =
     _extent2dims(to, _match_to_extent(to, size), res; kw...)
 function _extent2dims(to::Extents.Extent, size::Tuple, res::Union{Nothing,NoKW};
-    sampling::Tuple, crs
+    sampling::Tuple, crs, mappedcrs
 )
     ranges = map(values(to), size, sampling) do (start, stop), length, s
         if s isa Points
@@ -158,16 +158,17 @@ function _extent2dims(to::Extents.Extent, size::Tuple, res::Union{Nothing,NoKW};
             range(; start, stop, length=length+1)[1:end-1]
         end
     end
-    return _extent2dims(to, ranges; sampling, crs)
+    return _extent2dims(to, ranges; sampling, crs, mappedcrs)
 end
-function _extent2dims(::Extents.Extent{K}, ranges; crs, sampling::Tuple) where K
+function _extent2dims(::Extents.Extent{K}, ranges; crs, mappedcrs, sampling::Tuple) where K
     crs = isnokw(crs) ? nothing : crs 
+    mappedcrs = isnokw(mappedcrs) ? nothing : mappedcrs 
     emptydims = map(name2dim, K)
     lookups = map(emptydims, ranges, sampling) do d, range, s
         order = Lookups.orderof(range)
         span = Regular(step(range))
         if d isa SpatialDim && !isnothing(crs)
-            Projected(range; sampling=s, order, span, crs)
+            Projected(range; sampling=s, order, span, crs, mappedcrs)
         else
             Sampled(range; sampling=s, order, span)
         end

test/sources/gribdatasets.jl

@@ -25,6 +25,9 @@ gribexamples_dir = abspath(joinpath(dirname(pathof(GRIBDatasets)), "..", "test",
 
 era5 = joinpath(gribexamples_dir, "era5-levels-members.grib")
 
+ds = GRIBDatasets.GRIBDataset(era5)
+v = ds[:z]
+
 @testset "Raster" begin
     @time gribarray = Raster(era5)
     @time lazyarray = Raster(era5; lazy=true)
@@ -87,8 +90,8 @@ era5 = joinpath(gribexamples_dir, "era5-levels-members.grib")
     end
 
     @testset "cf attributes" begin
-        z = lazystack[:z]
-        @test metadata(z)["standard_name"] == "geopotential"
+        z = lazystack.z
+        @test metadata(z)["cfName"] == "geopotential"
 
         @test metadata(lazystack)["Conventions"] == "CF-1.7"
         x = dims(lazystack, :X)