diff --git a/docs/src/projective/gallery.md b/docs/src/projective/gallery.md index 2e726ab..b85d8de 100644 --- a/docs/src/projective/gallery.md +++ b/docs/src/projective/gallery.md @@ -26,6 +26,45 @@ tfm = CenterCrop((196, 196)) image = Image(imagedata) apply(tfm, image) |> itemdata ``` +Customization of how pixel values are interpolated and extrapolated during +transformations is done with the Item types ([`Image`](@ref), +[`MaskBinary`](@ref), [`MaskMulti`](@ref)). For example, if we scale the image +we can see how the interpolation affects how values of projected pixels are +calculated. +```@example deps +using Interpolations: BSpline, Constant, Linear +tfm = ScaleFixed((2000, 2000)) |> CenterCrop((200, 200)) +showgrid( + [ + # Default is linear interpolation for Image + apply(tfm, Image(imagedata)), + # Nearest neighbor interpolation + apply(tfm, Image(imagedata; interpolate=BSpline(Constant()))), + # Linear interpolation + apply(tfm, Image(imagedata; interpolate=BSpline(Linear()))), + ]; + ncol=3, + npad=8, +) +``` +Similarly, if we crop to a larger region than the image, we can see how +extrapolation affects how pixel values are calculated in the regions outside +the original image bounds. +```@example deps +import Interpolations +tfm = CenterCrop((400, 400)) +showgrid( + [ + apply(tfm, Image(imagedata)), + apply(tfm, Image(imagedata; extrapolate=1)), + apply(tfm, Image(imagedata; extrapolate=Interpolations.Flat())), + apply(tfm, Image(imagedata; extrapolate=Interpolations.Periodic())), + apply(tfm, Image(imagedata; extrapolate=Interpolations.Reflect())), + ]; + ncol=5, + npad=8, +) +``` Now let's say we want to train a light house detector and have a bounding box for the light house. We can use the [`BoundingBox`](@ref) item to represent it. It takes the two corners of the bounding rectangle as the first argument. As the second argument we have to pass the size of the corresponding image. diff --git a/src/items/image.jl b/src/items/image.jl index 7aea4dc..cf971ac 100644 --- a/src/items/image.jl +++ b/src/items/image.jl @@ -4,9 +4,21 @@ # correspond to the image axes. """ - Image(image[, bounds]) - -Item representing an N-dimensional image with element type T. + Image(image[, bounds]; interpolate=BSpline(Linear()), extrapolate=zero(T)) + +Item representing an N-dimensional image with element type T. Optionally, the +interpolation and extrapolation method can be provided. Interpolation here +refers to how the values of projected pixels that fall into the transformed +content bounds are calculated. Extrapolation refers to how to assign values +that fall outside the projected content bounds. The default is linear +interpolation and to fill new regions with zero. + +!!! info + The `Interpolations` package provides numerous methods for use with + the `interpolate` and `extrapolate` keyword arguments. For instance, + `BSpline(Linear())` and `BSpline(Constant())` provide linear and nearest + neighbor interpolation, respectively. In addition `Flat()`, `Reflect()` and + `Periodic()` boundary conditions are available for extrapolation. ## Examples @@ -30,14 +42,24 @@ showitems(item) struct Image{N,T} <: AbstractArrayItem{N,T} data::AbstractArray{T,N} bounds::Bounds{N} + interpolate::Interpolations.InterpolationType + extrapolate::ImageTransformations.FillType end -Image(data) = Image(data, Bounds(axes(data))) - -function Image(data::AbstractArray{T,N}, sz::NTuple{N,Int}) where {T,N} - return Image(data, Bounds(sz)) +function Image( + data::AbstractArray{T,N}, + bounds::Bounds{N}; + interpolate::Interpolations.InterpolationType=BSpline(Linear()), + extrapolate::ImageTransformations.FillType=zero(T), +) where {T,N} + return Image(data, bounds, interpolate, extrapolate) end +Image(data; kwargs...) = Image(data, Bounds(axes(data)); kwargs...) + +function Image(data::AbstractArray{T,N}, sz::NTuple{N,Int}; kwargs...) where {T,N} + return Image(data, Bounds(sz); kwargs...) +end Base.show(io::IO, item::Image{N,T}) where {N,T} = print(io, "Image{$N, $T}() with bounds $(item.bounds)") @@ -68,7 +90,8 @@ function project(P, image::Image{N, T}, bounds::Bounds) where {N, T} itemdata(image), inv(P), bounds.rs; - fillvalue = zero(T)) + method=image.interpolate, + fillvalue=image.extrapolate) return Image(data_, bounds) end @@ -79,7 +102,7 @@ function project!(bufimage::Image, P, image::Image{N, T}, bounds::Bounds{N}) whe a = OffsetArray(parent(itemdata(bufimage)), bounds.rs) res = warp!( a, - box_extrapolation(itemdata(image); fillvalue=zero(T)), + box_extrapolation(itemdata(image); method=image.interpolate, fillvalue=image.extrapolate), inv(P), ) return Image(res, bounds) diff --git a/src/items/mask.jl b/src/items/mask.jl index 7f1caa9..d02cdb1 100644 --- a/src/items/mask.jl +++ b/src/items/mask.jl @@ -1,17 +1,27 @@ """ - MaskMulti(a, [classes]) - -An `N`-dimensional multilabel mask with labels `classes`. + MaskMulti(a, [classes]; interpolate=BSpline(Constant()), extrapolate=Flat()) + +An `N`-dimensional multilabel mask with labels `classes`. Optionally, the +interpolation and extrapolation method can be provided. Interpolation here +refers to how the values of projected pixels that fall into the transformed +content bounds are calculated. Extrapolation refers to how to assign values +that fall outside the projected content bounds. The default is nearest neighbor +interpolation and flat extrapolation of the edges into new regions. + +!!! info + The `Interpolations` package provides numerous methods for use with + the `interpolate` and `extrapolate` keyword arguments. For instance, + `BSpline(Linear())` and `BSpline(Constant())` provide linear and nearest + neighbor interpolation, respectively. In addition `Flat()`, `Reflect()` and + `Periodic()` boundary conditions are available for extrapolation. ## Examples -{cell=MaskMulti} ```julia using DataAugmentation mask = MaskMulti(rand(1:3, 100, 100)) ``` -{cell=MaskMulti} ```julia showitems(mask) ``` @@ -20,19 +30,30 @@ struct MaskMulti{N, T<:Integer, U} <: AbstractArrayItem{N, T} data::AbstractArray{T, N} classes::AbstractVector{U} bounds::Bounds{N} + interpolate::Interpolations.InterpolationType + extrapolate::ImageTransformations.FillType end +function MaskMulti( + data::AbstractArray{T,N}, + classes::AbstractVector{U}, + bounds::Bounds{N}; + interpolate::Interpolations.InterpolationType=BSpline(Constant()), + extrapolate::ImageTransformations.FillType=Flat(), +) where {N, T<:Integer, U} + return MaskMulti(data, classes, bounds, interpolate, extrapolate) +end -function MaskMulti(a::AbstractArray, classes = unique(a)) +function MaskMulti(a::AbstractArray, classes = unique(a); kwargs...) bounds = Bounds(size(a)) minimum(a) >= 1 || error("Class values must start at 1") - return MaskMulti(a, classes, bounds) + return MaskMulti(a, classes, bounds; kwargs...) end -MaskMulti(a::AbstractArray{<:Gray{T}}, args...) where T = MaskMulti(reinterpret(T, a), args...) -MaskMulti(a::AbstractArray{<:Normed{T}}, args...) where T = MaskMulti(reinterpret(T, a), args...) -MaskMulti(a::IndirectArray, classes = a.values, bounds = Bounds(size(a))) = - MaskMulti(a.index, classes, bounds) +MaskMulti(a::AbstractArray{<:Gray{T}}, args...; kwargs...) where T = MaskMulti(reinterpret(T, a), args...; kwargs...) +MaskMulti(a::AbstractArray{<:Normed{T}}, args...; kwargs...) where T = MaskMulti(reinterpret(T, a), args...; kwargs...) +MaskMulti(a::IndirectArray, classes = a.values, bounds = Bounds(size(a)); kwargs...) = + MaskMulti(a.index, classes, bounds; kwargs...) Base.show(io::IO, mask::MaskMulti{N, T}) where {N, T} = print(io, "MaskMulti{$N, $T}() with size $(size(itemdata(mask))) and $(length(mask.classes)) classes") @@ -41,12 +62,15 @@ Base.show(io::IO, mask::MaskMulti{N, T}) where {N, T} = getbounds(mask::MaskMulti) = mask.bounds -function project(P, mask::MaskMulti, bounds::Bounds) - a = itemdata(mask) - etp = mask_extrapolation(a) - res = warp(etp, inv(P), bounds.rs) +function project(P, mask::MaskMulti{N, T, U}, bounds::Bounds{N}) where {N, T, U} + res = warp( + itemdata(mask), + inv(P), + bounds.rs; + method=mask.interpolate, + fillvalue=mask.extrapolate) return MaskMulti( - res, + convert.(T, res), mask.classes, bounds ) @@ -57,7 +81,7 @@ function project!(bufmask::MaskMulti, P, mask::MaskMulti, bounds) a = OffsetArray(parent(itemdata(bufmask)), bounds.rs) warp!( a, - mask_extrapolation(itemdata(mask)), + box_extrapolation(itemdata(mask); method=mask.interpolate, fillvalue=mask.extrapolate), inv(P), ) return MaskMulti( @@ -80,19 +104,29 @@ end # ## Binary masks """ - MaskBinary(a) - -An `N`-dimensional binary mask. + MaskBinary(a; interpolate=BSpline(Constant()), extrapolate=Flat()) + +An `N`-dimensional binary mask. Optionally, the interpolation and extrapolation +method can be provided. Interpolation here refers to how the values of +projected pixels that fall into the transformed content bounds are calculated. +Extrapolation refers to how to assign values that fall outside the projected +content bounds. The default is nearest neighbor interpolation and flat +extrapolation of the edges into new regions. + +!!! info + The `Interpolations` package provides numerous methods for use with + the `interpolate` and `extrapolate` keyword arguments. For instance, + `BSpline(Linear())` and `BSpline(Constant())` provide linear and nearest + neighbor interpolation, respectively. In addition `Flat()`, `Reflect()` and + `Periodic()` boundary conditions are available for extrapolation. ## Examples -{cell=MaskMulti} ```julia using DataAugmentation mask = MaskBinary(rand(Bool, 100, 100)) ``` -{cell=MaskMulti} ```julia showitems(mask) ``` @@ -100,10 +134,17 @@ showitems(mask) struct MaskBinary{N} <: AbstractArrayItem{N, Bool} data::AbstractArray{Bool, N} bounds::Bounds{N} + interpolate::Interpolations.InterpolationType + extrapolate::ImageTransformations.FillType end -function MaskBinary(a::AbstractArray{Bool, N}, bounds = Bounds(size(a))) where N - return MaskBinary(a, bounds) +function MaskBinary( + a::AbstractArray, + bounds = Bounds(size(a)); + interpolate::Interpolations.InterpolationType=BSpline(Constant()), + extrapolate::ImageTransformations.FillType=Flat(), +) + return MaskBinary(a, bounds, interpolate, extrapolate) end Base.show(io::IO, mask::MaskBinary{N}) where {N} = @@ -112,19 +153,21 @@ Base.show(io::IO, mask::MaskBinary{N}) where {N} = getbounds(mask::MaskBinary) = mask.bounds function project(P, mask::MaskBinary, bounds::Bounds) - etp = mask_extrapolation(itemdata(mask)) - return MaskBinary( - warp(etp, inv(P), bounds.rs), - bounds, - ) + res = warp( + itemdata(mask), + inv(P), + bounds.rs; + method=mask.interpolate, + fillvalue=mask.extrapolate) + return MaskBinary(convert.(Bool, res), bounds) end function project!(bufmask::MaskBinary, P, mask::MaskBinary, bounds) a = OffsetArray(parent(itemdata(bufmask)), bounds.rs) - res = warp!( + warp!( a, - mask_extrapolation(itemdata(mask)), + box_extrapolation(itemdata(mask); method=mask.interpolate, fillvalue=mask.extrapolate), inv(P), ) return MaskBinary( @@ -136,15 +179,3 @@ end function showitem!(img, mask::MaskBinary) showimage!(img, colorview(Gray, itemdata(mask))) end -# ## Helpers - - -function mask_extrapolation( - mask::AbstractArray{T}; - t = T, - degree = Constant(), - boundary = Flat()) where T - itp = interpolate(t, t, mask, BSpline(degree)) - etp = extrapolate(itp, Flat()) - return etp -end diff --git a/src/testing.jl b/src/testing.jl index 606dddd..06d49b5 100644 --- a/src/testing.jl +++ b/src/testing.jl @@ -103,17 +103,19 @@ function testitem end testitem(::Type{ArrayItem}) = testitem(ArrayItem{2, Float32}) testitem(::Type{ArrayItem{N, T}}) where {N, T} = ArrayItem(rand(T, ntuple(i -> 16, N))) -testitem(::Type{Image}) = testitem(Image{2, RGB{N0f8}}) -testitem(::Type{Image{N, T}}) where {N, T} = Image(rand(T, ntuple(i -> 16, N))) +testitem(::Type{Image}; kwargs...) = testitem(Image{2, RGB{N0f8}}; kwargs...) +testitem(::Type{Image{N}}; kwargs...) where {N} = testitem(Image{N, RGB{N0f8}}; kwargs...) +testitem(::Type{Image{N, T}}; kwargs...) where {N, T} = Image(rand(T, ntuple(i -> 16, N)); kwargs...) -testitem(::Type{MaskBinary}) = testitem(MaskBinary{2}) -testitem(::Type{MaskBinary{N}}) where {N} = MaskBinary(rand(Bool, ntuple(i -> 16, N))) +testitem(::Type{MaskBinary}; kwargs...) = testitem(MaskBinary{2}; kwargs...) +testitem(::Type{MaskBinary{N}}; kwargs...) where {N} = MaskBinary(rand(Bool, ntuple(i -> 16, N)); kwargs...) -testitem(::Type{MaskMulti}) = testitem(MaskMulti{2, UInt8}) -function testitem(::Type{MaskMulti{N, T}}) where {N, T} +testitem(::Type{MaskMulti}; kwargs...) = testitem(MaskMulti{2, UInt8}; kwargs...) +testitem(::Type{MaskMulti{N}}; kwargs...) where {N} = testitem(MaskMulti{N, UInt8}; kwargs...) +function testitem(::Type{MaskMulti{N, T}}; kwargs...) where {N, T} n = rand(2:10) data = T.(rand(1:n, ntuple(i -> 16, N))) - MaskMulti(data, 1:n) + MaskMulti(data, 1:n; kwargs...) end diff --git a/test/Project.toml b/test/Project.toml index 27c79da..b39687e 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -8,8 +8,11 @@ CoordinateTransformations = "150eb455-5306-5404-9cee-2592286d6298" DataAugmentation = "88a5189c-e7ff-4f85-ac6b-e6158070f02e" FixedPointNumbers = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" ImageShow = "4e3cecfd-b093-5904-9786-8bbb286a6a31" +Interpolations = "a98d9a8b-a2ab-59e6-89dd-64a1c18fca59" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" -StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" +OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" Rotations = "6038ab10-8711-5258-84ad-4b1120ba62dc" +StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TestSetExtensions = "98d24dd4-01ad-11ea-1b02-c9a08f80db04" diff --git a/test/imports.jl b/test/imports.jl index 7561244..0945f2d 100644 --- a/test/imports.jl +++ b/test/imports.jl @@ -7,6 +7,9 @@ using Colors using FixedPointNumbers: N0f8 using LinearAlgebra using Rotations +using Statistics +using OffsetArrays +import Interpolations using DataAugmentation: Item, Transform, getrandstate, itemdata, setdata, ComposedProjectiveTransform, diff --git a/test/testing.jl b/test/testing.jl index bf82552..7aaa904 100644 --- a/test/testing.jl +++ b/test/testing.jl @@ -8,3 +8,94 @@ const ITEMS = (ArrayItem, Image, MaskBinary, MaskMulti, Keypoints, Polygon, Boun @test testitem(I) isa I end end + +@testset ExtendedTestSet "$(N)D $(I) extrapolate (constant)" for N in (2, 3), I in (Image, MaskBinary, MaskMulti) + item = testitem(I{N}; extrapolate=1) + data = item |> itemdata + + bounds = getbounds(item) + sizes = length.(bounds.rs) + + tfm = CenterCrop(ntuple(i -> 3*sizes[i], N)) + transformed = apply(tfm, item) |> itemdata + + @test data ≈ transformed[bounds.rs...] + + expected_value = convert(eltype(data), 1) + for d in 1:N + extrapolated = selectdim(transformed, d, (1-sizes[d]):0) + @test all(v -> v ≈ expected_value, extrapolated) + extrapolated = selectdim(transformed, d, (sizes[d]+1):2*sizes[d]) + @test all(v -> v ≈ expected_value, extrapolated) + end +end + +@testset ExtendedTestSet "$(N)D $(I) extrapolate (flat)" for N in (2, 3), I in (Image, MaskBinary, MaskMulti) + item = testitem(I{N}; extrapolate=Interpolations.Flat()) + data = item |> itemdata + + bounds = getbounds(item) + sizes = length.(bounds.rs) + + tfm = CenterCrop(ntuple(i -> 2*sizes[i], N)) + transformed = apply(tfm, item) |> itemdata + + expected = similar(transformed) + for index in CartesianIndices(expected) + clamped = clamp.(Tuple(index), bounds.rs) + expected[index] = data[clamped...] + end + @test transformed ≈ expected +end + +@testset ExtendedTestSet "$(N)D $(I) extrapolate (reflect)" for N in (2, 3), I in (Image, MaskBinary, MaskMulti) + item = testitem(I{N}; extrapolate=Interpolations.Reflect()) + data = item |> itemdata + + bounds = getbounds(item) + sizes = length.(bounds.rs) + + tfm = Crop(ntuple(i -> i==1 ? 2 * sizes[i] - 1 : sizes[i], N)) + transformed = apply(tfm, item) |> itemdata + + @test transformed[ntuple(i -> bounds.rs[i], N)...] ≈ data + @test transformed[ntuple(i -> i==1 ? (2*sizes[i]-1:-1:sizes[i]) : bounds.rs[i], N)...] ≈ data +end + +@testset ExtendedTestSet "$(N)D $(I) interpolate (nearest)" for N in (2, 3), I in (Image, MaskBinary, MaskMulti) + item = testitem(I{N}; interpolate=Interpolations.BSpline(Interpolations.Constant())) + data = item |> itemdata + + tfm = Project(LinearMap(UniformScaling(2))) + transformed = apply(tfm, item) |> itemdata + + expected = similar(transformed) + for index in CartesianIndices(expected) + original = map(i -> ceil(Int, i/2), Tuple(index)) + expected[index] = data[original...] + end + @test transformed ≈ expected +end + +@testset ExtendedTestSet "$(N)D $(I) interpolate (linear)" for N in (2, 3), I in (MaskBinary, MaskMulti) + tfm = Project(LinearMap(UniformScaling(2))) + item = testitem(I{N}; interpolate=Interpolations.BSpline(Interpolations.Linear())) + @test_throws InexactError apply(tfm, item) +end + +@testset ExtendedTestSet "$(N)D $(I) interpolate (linear)" for N in (2,3), I in (Image,) + item = testitem(I{N}; interpolate=Interpolations.BSpline(Interpolations.Linear())) + data = item |> itemdata + + tfm = Project(LinearMap(UniformScaling(2))) + transformed = apply(tfm, item) |> itemdata + + expected = similar(transformed) + for index in CartesianIndices(expected) + neighborhood = map(Tuple(index)) do i + (floor(Int, i/2) : ceil(Int, i/2)) + end + expected[index] = mean(data[neighborhood...]) + end + @test all(t ≈ e for (t, e) in zip(transformed, expected)) +end