return_type_tfunc was too pessimistic

make Union{} a leaftype (and also Tuples of Unions)
and ensure return_type_tfunc will infer as much as possible

base/broadcast.jl

@@ -287,7 +287,7 @@ _broadcast_eltype(f, A, Bs...) = Base._return_type(f, eltypestuple(A, Bs...))
     T = _broadcast_eltype(f, A, Bs...)
     shape = broadcast_indices(A, Bs...)
     iter = CartesianRange(shape)
-    if isleaftype(T)
+    if isleaftype(T) && T !== Union{}
         return broadcast_t(f, T, shape, iter, A, Bs...)
     end
     if isempty(iter)
@@ -303,7 +303,7 @@ end
 @inline function broadcast_c(f, ::Type{Nullable}, a...)
     nonnull = all(hasvalue, a)
     S = _broadcast_eltype(f, a...)
-    if isleaftype(S) && null_safe_eltype_op(f, a...)
+    if isleaftype(S) && S !== Union{} && null_safe_eltype_op(f, a...)
         Nullable{S}(f(map(unsafe_get, a)...), nonnull)
     else
         if nonnull

base/complex.jl

@@ -865,7 +865,7 @@ big{T<:Integer}(z::Complex{T}) = Complex{BigInt}(z)
 complex{T<:Complex}(A::AbstractArray{T}) = A
 
 function complex{T}(A::AbstractArray{T})
-    if !isleaftype(T)
+    if !isleaftype(T) || T === Union{}
         error("`complex` not defined on abstractly-typed arrays; please convert to a more specific type")
     end
     convert(AbstractArray{typeof(complex(zero(T)))}, A)

base/dict.jl

@@ -162,7 +162,7 @@ associative_with_eltype(DT_apply, ::Type) = DT_apply(Any, Any)()
 associative_with_eltype{F}(DT_apply::F, kv, t) = grow_to!(associative_with_eltype(DT_apply, _default_eltype(typeof(kv))), kv)
 function associative_with_eltype{F}(DT_apply::F, kv::Generator, t)
     T = _default_eltype(typeof(kv))
-    if T <: Union{Pair, Tuple{Any, Any}} && isleaftype(T)
+    if T <: Union{Pair, Tuple{Any, Any}} && isleaftype(T) && T !== Union{}
         return associative_with_eltype(DT_apply, kv, T)
     end
     return grow_to!(associative_with_eltype(DT_apply, T), kv)

base/float.jl

@@ -757,7 +757,7 @@ fpinttype(::Type{Float16}) = UInt16
 float{T<:AbstractFloat}(A::AbstractArray{T}) = A
 
 function float{T}(A::AbstractArray{T})
-    if !isleaftype(T)
+    if !isleaftype(T) || T == Union{}
         error("`float` not defined on abstractly-typed arrays; please convert to a more specific type")
     end
     convert(AbstractArray{typeof(float(zero(T)))}, A)

base/inference.jl

@@ -448,7 +448,7 @@ function typeof_tfunc(t::ANY)
         if !isleaftype(tp)
             return DataType # typeof(Kind::Type)::DataType
         else
-            return Const(typeof(tp))
+            return Const(typeof(tp)) # XXX: this is not necessarily true
         end
     elseif isa(t, DataType)
         if isleaftype(t) || isvarargtype(t)
@@ -1188,35 +1188,44 @@ function abstract_apply(af::ANY, fargs, aargtypes::Vector{Any}, vtypes::VarTable
     return abstract_call(af, (), Any[Const(af), Vararg{Any}], vtypes, sv)
 end
 
-function pure_eval_call(f::ANY, argtypes::ANY, atype::ANY, vtypes::VarTable, sv::InferenceState)
-    if f === return_type && length(argtypes) == 3
+function return_type_tfunc(argtypes::ANY, vtypes::VarTable, sv::InferenceState)
+    if length(argtypes) == 3
         tt = argtypes[3]
-        if isa(tt, Const) || isconstType(tt)
-            af = argtypes[2]
-            af_isconst = isa(af, Const) || isconstType(af)
-            if (af_isconst || (isleaftype(af) &&
-                               !(af <: Builtin) && !(af <: IntrinsicFunction)))
+        if isa(tt, Const) || (isType(tt) && !has_free_typevars(tt))
+            aft = argtypes[2]
+            if isa(aft, Const) || (isType(aft) && !has_free_typevars(aft)) ||
+                   (isleaftype(aft) && !(aft <: Builtin) && !(aft <: IntrinsicFunction))
                 af_argtype = isa(tt, Const) ? tt.val : tt.parameters[1]
-                if af_argtype <: Tuple && isa(af_argtype, DataType)
-                    argtypes_vec = Any[af, af_argtype.parameters...]
-                    if af_isconst
-                        rt = abstract_call(isa(af,Const) ? af.val : af.parameters[1],
-                                           (), argtypes_vec, vtypes, sv)
+                if isa(af_argtype, DataType) && af_argtype <: Tuple
+                    argtypes_vec = Any[aft, af_argtype.parameters...]
+                    if isa(aft, Const)
+                        rt = abstract_call(aft.val, (), argtypes_vec, vtypes, sv)
+                    elseif isconstType(aft)
+                        rt = abstract_call(aft.parameters[1], (), argtypes_vec, vtypes, sv)
                     else
                         rt = abstract_call_gf_by_type(nothing, argtypes_to_type(argtypes_vec), sv)
                     end
-                    if isa(rt,Const)
-                        return Type{widenconst(rt)}
-                    elseif isleaftype(rt) || isleaftype(af_argtype) || rt === Bottom
-                        return Type{rt}
+                    if isa(rt, Const)
+                        # output was computed to be constant
+                        return Const(typeof(rt.val))
+                    elseif isleaftype(rt)
+                        # output type was known for certain
+                        return Const(rt)
+                    elseif (isa(tt, Const) || isconstType(tt)) &&
+                           (isa(aft, Const) || isconstType(aft))
+                        # input arguments were known for certain
+                        return Const(rt)
                     else
-                        return Type{R} where R<:rt
+                        return Type{R} where R <: rt
                     end
                 end
             end
         end
     end
+    return NF
+end
 
+function pure_eval_call(f::ANY, argtypes::ANY, atype::ANY, vtypes::VarTable, sv::InferenceState)
     for i = 2:length(argtypes)
         a = argtypes[i]
         if !(isa(a,Const) || isconstType(a))
@@ -1309,6 +1318,11 @@ function abstract_call(f::ANY, fargs, argtypes::Vector{Any}, vtypes::VarTable, s
             end
         end
         return Any
+    elseif f === return_type
+        rt_rt = return_type_tfunc(argtypes, vtypes, sv)
+        if rt_rt !== NF
+            return rt_rt
+        end
     end
 
     tm = _topmod(sv)
@@ -1810,7 +1824,7 @@ function code_for_method(method::Method, atypes::ANY, sparams::SimpleVector, wor
         # don't call staged functions on abstract types.
         # (see issues #8504, #10230)
         # we can't guarantee that their type behavior is monotonic.
-        # XXX: this test is wrong if Types (such as DataType) are present
+        # XXX: this test is wrong if Types (such as DataType or Bottom) are present
         return nothing
     end
     if preexisting
@@ -4293,9 +4307,8 @@ function is_allocation(e::ANY, sv::InferenceState)
         return (length(e.args)-1,())
     elseif e.head === :new
         typ = widenconst(exprtype(e, sv.src, sv.mod))
-        if isleaftype(typ)
-            @assert(isa(typ,DataType))
-            nf = length(e.args)-1
+        if isa(typ, DataType) && isleaftype(typ)
+            nf = length(e.args) - 1
             names = fieldnames(typ)
             @assert(nf <= nfields(typ))
             if nf < nfields(typ)

base/reflection.jl

@@ -230,7 +230,7 @@ isbits(x) = (@_pure_meta; isbits(typeof(x)))
 Determine whether `T` is a concrete type that can have instances, meaning its only subtypes
 are itself and `Union{}` (but `T` itself is not `Union{}`).
 """
-isleaftype(t::ANY) = (@_pure_meta; isa(t, DataType) && t.isleaftype)
+isleaftype(t::ANY) = (@_pure_meta; (isa(t, DataType) && t.isleaftype) || t === Union{})
 
 """
     typeintersect(T, S)

base/set.jl

@@ -10,7 +10,7 @@ Set() = Set{Any}()
 Set(itr) = Set{eltype(itr)}(itr)
 function Set(g::Generator)
     T = _default_eltype(typeof(g))
-    (isleaftype(T) || T === Union{}) || return grow_to!(Set{T}(), g)
+    isleaftype(T) || return grow_to!(Set{T}(), g)
     return Set{T}(g)
 end
 

base/show.jl

@@ -541,7 +541,7 @@ function show_expr_type(io::IO, ty, emph)
     elseif ty === Core.IntrinsicFunction
         print(io, "::I")
     else
-        if emph && (!isleaftype(ty) || ty == Core.Box)
+        if emph && (!isleaftype(ty) || ty == Core.Box || ty == Union{})
             emphasize(io, "::$ty")
         else
             print(io, "::$ty")

src/jltypes.c

@@ -204,7 +204,7 @@ JL_DLLEXPORT int (jl_is_leaf_type)(jl_value_t *v)
         return 1;
 #endif
     }
-    return 0;
+    return v == jl_bottom_type;
 }
 
 // Return true for any type (Integer or Unsigned) that can fit in a

src/julia.h

@@ -991,7 +991,7 @@ jl_value_t *jl_rewrap_unionall(jl_value_t *t, jl_value_t *u);
 #ifdef NDEBUG
 STATIC_INLINE int jl_is_leaf_type_(jl_value_t *v)
 {
-    return jl_is_datatype(v) && ((jl_datatype_t*)v)->isleaftype;
+    return (jl_is_datatype(v) && ((jl_datatype_t*)v)->isleaftype) || v == jl_bottom_type;
 }
 #define jl_is_leaf_type(v) jl_is_leaf_type_(v)
 #endif