Skip to content
This repository was archived by the owner on May 4, 2019. It is now read-only.

Commit 3432db6

Browse files
nalimilannalimilan
committed
Rewrite broadcast() based on lift()
Remove the custom implementation of broadcast(), and just call the base method on the lift()ed method. This implements the blacklist approach: methods with a custom lifting behavior like isnull() should override lift() to get passed Nullable values directly; if they do not return a Nullable, the result is a standard Array rather than a NullableArray. Performance will probably be worse than before, but at least the semantics will be correct. We can always re-implement the custom and faster versions later when broadcast() has stabilized in Base and Nullable support has settled.
1 parent f0d2e3d commit 3432db6

File tree

4 files changed

+165
-198
lines changed

4 files changed

+165
-198
lines changed

‎src/NullableArrays.jl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ include("primitives.jl")
2727
include("indexing.jl")
2828
include("map.jl")
2929
include("nullablevector.jl")
30+
include("lift.jl")
3031
include("operators.jl")
3132
include("broadcast.jl")
3233
include("reduce.jl")

‎src/broadcast.jl

Lines changed: 19 additions & 175 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
1-
using Base: promote_eltype
2-
using Base.Cartesian
31
if VERSION >= v"0.6.0-dev.693"
42
using Base.Broadcast: check_broadcast_indices, broadcast_indices
3+
using Base.Broadcast: check_broadcast_indices, broadcast_indices,
4+
_default_eltype, ftype, ziptype
55
else
66
using Base.Broadcast: check_broadcast_shape, broadcast_shape
7+
using Base.Broadcast: check_broadcast_shape, broadcast_shape,
8+
_default_eltype, ftype, ziptype
79
const check_broadcast_indices = check_broadcast_shape
810
const broadcast_indices = broadcast_shape
911
end
@@ -15,177 +17,9 @@ else
1517
_to_shape(x) = x
1618
end
1719

18-
if VERSION < v"0.5.0-dev+5434"
19-
function gen_nullcheck(narrays::Int, nd::Int)
20-
e_nullcheck = macroexpand(:( @nref $nd isnull_1 d->j_d_1 ))
21-
for k = 2:narrays
22-
isnull = Symbol("isnull_$k")
23-
j_d_k = Symbol("j_d_$k")
24-
e_isnull_k = macroexpand(:( @nref $nd $(isnull) d->$(j_d_k) ))
25-
e_nullcheck = Expr(:||, e_nullcheck, e_isnull_k)
26-
end
27-
return e_nullcheck
28-
end
29-
30-
function gen_broadcast_body(nd::Int, narrays::Int, f, lift::Bool)
31-
F = Expr(:quote, f)
32-
e_nullcheck = gen_nullcheck(narrays, nd)
33-
if lift
34-
return quote
35-
# set up aliases to facilitate subsequent Base.Cartesian magic
36-
B_isnull = B.isnull
37-
@nexprs $narrays k->(values_k = A_k.values)
38-
@nexprs $narrays k->(isnull_k = A_k.isnull)
39-
# check size
40-
@assert ndims(B) == $nd
41-
@ncall $narrays check_broadcast_shape size(B) k->A_k
42-
# main loops
43-
@nloops($nd, i, B,
44-
d->(@nexprs $narrays k->(j_d_k = size(A_k, d) == 1 ? 1 : i_d)), # pre
45-
begin # body
46-
if $e_nullcheck
47-
@inbounds (@nref $nd B_isnull i) = true
48-
else
49-
@nexprs $narrays k->(@inbounds v_k = @nref $nd values_k d->j_d_k)
50-
@inbounds (@nref $nd B i) = (@ncall $narrays $F v)
51-
end
52-
end
53-
)
54-
end
55-
else
56-
return Base.Broadcast.gen_broadcast_body_cartesian(nd, narrays, f)
57-
end
58-
end
59-
60-
function gen_broadcast_function(nd::Int, narrays::Int, f, lift::Bool)
61-
As = [Symbol("A_"*string(i)) for i = 1:narrays]
62-
body = gen_broadcast_body(nd, narrays, f, lift)
63-
@eval let
64-
local _F_
65-
function _F_(B, $(As...))
66-
$body
67-
end
68-
_F_
69-
end
70-
end
71-
72-
function Base.broadcast!(f, X::NullableArray; lift::Bool=false)
73-
broadcast!(f, X, X; lift=lift)
74-
end
75-
76-
@eval let cache = Dict{Any, Dict{Bool, Dict{Int, Dict{Int, Any}}}}()
77-
@doc """
78-
`broadcast!(f, B::NullableArray, As::NullableArray...; lift::Bool=false)`
79-
This method implements the same behavior as that of `broadcast!` when called on
80-
regular `Array` arguments. It also includes the `lift` keyword argument, which
81-
when set to true will lift `f` over the entries of the `As`.
82-
83-
Lifting is disabled by default. Note that this method's signature specifies
84-
the destination `B` array as well as the source `As` arrays as all
85-
`NullableArray`s. Thus, calling `broadcast!` on a arguments consisting
86-
of both `Array`s and `NullableArray`s will fall back to the implementation
87-
of `broadcast!` in `base/broadcast.jl`.
88-
""" ->
89-
function Base.broadcast!(f, B::NullableArray, As::NullableArray...; lift::Bool=false)
90-
nd = ndims(B)
91-
narrays = length(As)
92-
93-
cache_f = Base.@get! cache f Dict{Bool, Dict{Int, Dict{Int, Any}}}()
94-
cache_lift = Base.@get! cache_f lift Dict{Int, Dict{Int, Any}}()
95-
cache_f_na = Base.@get! cache_lift narrays Dict{Int, Any}()
96-
func = Base.@get! cache_f_na nd gen_broadcast_function(nd, narrays, f, lift)
97-
98-
func(B, As...)
99-
return B
100-
end
101-
end # let cache
102-
else
103-
using Base.Broadcast: newindexer, map_newindexer, newindex
104-
105-
function _nullcheck(nargs)
106-
nullcheck = :(isnull_1[I_1])
107-
for i in 2:nargs
108-
sym_isnull = Symbol("isnull_$i")
109-
sym_idx = Symbol("I_$i")
110-
nullcheck = Expr(:||, :($sym_isnull[$sym_idx]), nullcheck)
111-
end
112-
# if 0 argument arrays, treat nullcheck as though it returns false
113-
nargs >= 1 ? nullcheck : :(false)
114-
end
115-
116-
@generated function Base.Broadcast._broadcast!{K,ID,XT,nargs}(f,
117-
Z::NullableArray, keeps::K, Idefaults::ID, Xs::XT, ::Type{Val{nargs}}; lift=false)
118-
nullcheck = _nullcheck(nargs)
119-
quote
120-
T = eltype(Z)
121-
$(Expr(:meta, :noinline))
122-
# destructure keeps and Xs tuples (common to both lifted and non-lifted broadcast)
123-
@nexprs $nargs i->(keep_i = keeps[i])
124-
@nexprs $nargs i->(Idefault_i = Idefaults[i])
125-
if !lift
126-
# destructure the keeps and As tuples
127-
@nexprs $nargs i->(X_i = Xs[i])
128-
@simd for I in CartesianRange(indices(Z))
129-
# reverse-broadcast the indices
130-
@nexprs $nargs i->(I_i = newindex(I, keep_i, Idefault_i))
131-
# extract array values
132-
@nexprs $nargs i->(@inbounds val_i = X_i[I_i])
133-
# call the function and store the result
134-
@inbounds Z[I] = @ncall $nargs f val
135-
end
136-
else
137-
# destructure the indexmaps and Xs tuples
138-
@nexprs $nargs i->(values_i = Xs[i].values)
139-
@nexprs $nargs i->(isnull_i = Xs[i].isnull)
140-
@simd for I in CartesianRange(indices(Z))
141-
# reverse-broadcast the indices
142-
@nexprs $nargs i->(I_i = newindex(I, keep_i, Idefault_i))
143-
if $nullcheck
144-
# if any args are null, store null
145-
@inbounds Z.isnull[I] = true
146-
else
147-
# extract array values
148-
@nexprs $nargs i->(@inbounds val_i = values_i[I_i])
149-
# call the function and store the result
150-
@inbounds Z[I] = @ncall $nargs f val
151-
end
152-
end
153-
end
154-
end
155-
end
156-
157-
@doc """
158-
`broadcast!(f, B::NullableArray, As::NullableArray...; lift::Bool=false)`
159-
160-
This method implements the same behavior as that of `broadcast!` when called
161-
on regular `Array` arguments. It also includes the `lift` keyword argument,
162-
which when set to true will lift `f` over the entries of the `As`.
163-
164-
Lifting is disabled by default. Note that this method's signature specifies
165-
the destination `B` array as well as the source `As` arrays as all
166-
`NullableArray`s. Thus, calling `broadcast!` on a arguments consisting of
167-
both `Array`s and `NullableArray`s will fall back to the implementation of
168-
`broadcast!` in `base/broadcast.jl`.
169-
""" ->
170-
# Required to solve dispatch ambiguity between
171-
# broadcast!(f, X::AbstractArray, x::Number...)
172-
# broadcast!(f, Z::NullableArrays.NullableArray, Xs::NullableArrays.NullableArray...)
173-
@inline Base.broadcast!(f, Z::NullableArray; lift=false) =
174-
broadcast!(f, Z, Z; lift=lift)
175-
176-
@inline function Base.broadcast!(f, Z::NullableArray, Xs::NullableArray...;
177-
lift=false)
178-
nargs = length(Xs)
179-
shape = indices(Z)
180-
check_broadcast_indices(shape, Xs...)
181-
keeps, Idefaults = map_newindexer(shape, Xs)
182-
Base.Broadcast._broadcast!(f, Z, keeps, Idefaults, Xs, Val{nargs}; lift=lift)
183-
return Z
184-
end
185-
end
18620

18721
@doc """
188-
`broadcast(f, As::NullableArray...;lift::Bool=false)`
22+
`broadcast(f, As::NullableArray...)`
18923
19024
This method implements the same behavior as that of `broadcast` when called on
19125
regular `Array` arguments. It also includes the `lift` keyword argument, which
@@ -196,10 +30,20 @@ source `As` arrays as all `NullableArray`s. Thus, calling `broadcast!` on
19630
arguments consisting of both `Array`s and `NullableArray`s will fall back to the
19731
implementation of `broadcast` in `base/broadcast.jl`.
19832
""" ->
199-
@inline function Base.broadcast(f, Xs::NullableArray...;lift::Bool=false)
200-
return broadcast!(f, NullableArray(eltype(promote_eltype(Xs...)),
201-
_to_shape(broadcast_indices(Xs...))),
202-
Xs...; lift=lift)
33+
function Base.broadcast{N}(f, As::Vararg{NullableArray, N})
34+
f2(x...) = lift(f, x...)
35+
T = _default_eltype(Base.Generator{ziptype(As...), ftype(f2, As...)})
36+
if isleaftype(T) && !(T <: Nullable)
37+
return invoke(broadcast, Tuple{Function, Vararg{AbstractArray, N}}, f2, As...)
38+
else
39+
dest = NullableArray(eltype(T), _to_shape(broadcast_indices(As...)))
40+
return invoke(broadcast!, Tuple{Function, AbstractArray, Vararg{AbstractArray, N}}, f2, dest, As...)
41+
end
42+
end
43+
44+
function Base.broadcast!{N}(f, dest::AbstractArray, As::Vararg{NullableArray, N})
45+
f2(x...) = lift(f, x...)
46+
invoke(broadcast!, Tuple{Function, AbstractArray, Vararg{AbstractArray, N}}, f2, dest, As...)
20347
end
20448

20549
# broadcasted ops

‎src/lift.jl

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
import Base: null_safe_op
2+
3+
##############################################################################
4+
##
5+
## Standard lifting semantics
6+
##
7+
## For a function call f(xs...), return null if any x in xs is null;
8+
## otherwise, return f applied to values of xs.
9+
##
10+
##############################################################################
11+
12+
@inline function lift(f, x)
13+
if null_safe_op(f, typeof(x))
14+
return @compat Nullable(f(x.value), !isnull(x))
15+
else
16+
U = Core.Inference.return_type(f, Tuple{eltype(typeof(x))})
17+
if isnull(x)
18+
return Nullable{U}()
19+
else
20+
return Nullable(f(unsafe_get(x)))
21+
end
22+
end
23+
end
24+
25+
@inline function lift(f, x1, x2)
26+
if null_safe_op(f, typeof(x1), typeof(x2))
27+
return @compat Nullable(
28+
f(x1.value, x2.value), !(isnull(x1) | isnull(x2))
29+
)
30+
else
31+
U = Core.Inference.return_type(
32+
f, Tuple{eltype(typeof(x1)), eltype(typeof(x2))}
33+
)
34+
if isnull(x1) | isnull(x2)
35+
return Nullable{U}()
36+
else
37+
return Nullable(f(unsafe_get(x1), unsafe_get(x2)))
38+
end
39+
end
40+
end
41+
42+
hasnulls(xs...) = any(isnull.(xs))
43+
44+
@inline function lift(f, xs...)
45+
if null_safe_op(f, map(typeof, xs)...)
46+
return @compat Nullable(
47+
f(map(unsafe_get, xs)...), !(mapreduce(isnull, |, xs))
48+
)
49+
else
50+
U = Core.Inference.return_type(
51+
f, Tuple{map(x->eltype(typeof(x)), xs)...}
52+
)
53+
if hasnulls(xs...)
54+
return Nullable{U}()
55+
else
56+
return Nullable(f(map(unsafe_get, xs)...))
57+
end
58+
end
59+
end
60+
61+
##############################################################################
62+
##
63+
## Non-standard lifting semantics
64+
##
65+
##############################################################################
66+
67+
# three-valued logic implementation
68+
@inline function lift(::typeof(&), x, y)::Nullable{Bool}
69+
return ifelse( isnull(x),
70+
ifelse( isnull(y),
71+
Nullable{Bool}(), # x, y null
72+
ifelse( unsafe_get(y),
73+
Nullable{Bool}(), # x null, y == true
74+
Nullable(false) # x null, y == false
75+
)
76+
),
77+
ifelse( isnull(y),
78+
ifelse( unsafe_get(x),
79+
Nullable{Bool}(), # x == true, y null
80+
Nullable(false) # x == false, y null
81+
),
82+
Nullable(unsafe_get(x) & unsafe_get(y)) # x, y not null
83+
)
84+
)
85+
end
86+
87+
# three-valued logic implementation
88+
@inline function lift(::typeof(|), x, y)::Nullable{Bool}
89+
return ifelse( isnull(x),
90+
ifelse( isnull(y),
91+
Nullable{Bool}(), # x, y null
92+
ifelse( unsafe_get(y),
93+
Nullable(true), # x null, y == true
94+
Nullable{Bool}() # x null, y == false
95+
)
96+
),
97+
ifelse( isnull(y),
98+
ifelse( unsafe_get(x),
99+
Nullable(true), # x == true, y null
100+
Nullable{Bool}() # x == false, y null
101+
),
102+
Nullable(unsafe_get(x) | unsafe_get(y)) # x, y not null
103+
)
104+
)
105+
end
106+
107+
# TODO: Decide on semantics for isequal and uncomment the following
108+
# @inline function lift(::typeof(isequal), x, y)
109+
# return ifelse( isnull(x),
110+
# ifelse( isnull(y),
111+
# true, # x, y null
112+
# false # x null, y not null
113+
# ),
114+
# ifelse( isnull(y),
115+
# false, # x not null, y null
116+
# isequal(unsafe_get(x), unsafe_get(y)) # x, y not null
117+
# )
118+
# )
119+
# end
120+
121+
@inline function lift(::typeof(isless), x, y)::Bool
122+
if null_safe_op(isless, typeof(x), typeof(y))
123+
return ifelse( isnull(x),
124+
false, # x null
125+
ifelse( isnull(y),
126+
true, # x not null, y null
127+
isless(unsafe_get(x), unsafe_get(y)) # x, y not null
128+
)
129+
)
130+
else
131+
return isnull(x) ? false :
132+
isnull(y) ? true : isless(unsafe_get(x), unsafe_get(y))
133+
end
134+
end
135+
136+
@inline lift(::typeof(isnull), x) = isnull(x)
137+
@inline lift(::typeof(get), x::Nullable) = get(x)
138+
@inline lift(::typeof(get), x::Nullable, y) = get(x, y)
139+

0 commit comments

Comments
 (0)