Skip to content

Commit fee3f0d

Browse files
timholytimholy
committed
Add a generic code-inferrability test
Closes #36393
1 parent 489808a commit fee3f0d

File tree

5 files changed

+500
-1
lines changed

5 files changed

+500
-1
lines changed

‎base/reflection.jl

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -753,6 +753,13 @@ function signature_type(@nospecialize(f), @nospecialize(args))
753753
end
754754
end
755755

756+
function call_type(@nospecialize(tt))
757+
ft = tt.parameters[1]
758+
argt = Tuple{tt.parameters[2:end]...}
759+
name = Symbol(String(ft.name.name)[2:end]) # strip off leading '#'
760+
return (getfield(ft.name.module, name), argt)
761+
end
762+
756763
"""
757764
code_lowered(f, types; generated=true, debuginfo=:default)
758765
@@ -929,6 +936,14 @@ function visit(f, d::Core.TypeMapEntry)
929936
end
930937
nothing
931938
end
939+
function visit(f, d::SimpleVector)
940+
for i = 1:length(d)
941+
if isassigned(d, i)
942+
f(d[i])
943+
end
944+
end
945+
nothing
946+
end
932947

933948
function length(mt::Core.MethodTable)
934949
n = 0

‎test/choosetests.jl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ function choosetests(choices = [])
5555
"boundscheck", "error", "ambiguous", "cartesian", "osutils",
5656
"channels", "iostream", "secretbuffer", "specificity",
5757
"reinterpretarray", "syntax", "corelogging", "missing", "asyncmap",
58-
"smallarrayshrink"
58+
"smallarrayshrink", "inference_qa"
5959
]
6060

6161
tests = []

‎test/inference_qa.jl

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
# This file is a part of Julia. License is MIT: https://julialang.org/license
2+
3+
# The aim of these tests is to enforce a minimum quality of inferrability of Julia's
4+
# Base and specifically its precompiled methods. Passing these tests does not
5+
# indicate that Julia has no inference problems, but they are designed to catch some
6+
# inadvertent problems. While `@inferred` only tests the return type, the tests
7+
# in this file are designed to check the overall inference quality of method *internals*.
8+
9+
# If you fail tests here, you can usually fix problems using `@code_warntype` or Cthulhu.jl's `@descend`.
10+
11+
using Test
12+
13+
isdefined(@__MODULE__, :is_atrisk_type) || include("testhelpers/methodanalysis.jl")
14+
15+
## Test the testing tools
16+
17+
@testset "is_atrisk_type" begin
18+
@test is_atrisk_type(Tuple{typeof(==),Any,Any})
19+
@test is_atrisk_type(Tuple{typeof(==),Symbol,Any})
20+
@test is_atrisk_type(Tuple{typeof(==),Any,Symbol})
21+
@test !is_atrisk_type(Tuple{typeof(==),Symbol,Symbol})
22+
@test !is_atrisk_type(Tuple{typeof(convert),Type{Any},Any})
23+
@test !is_atrisk_type(Tuple{typeof(convert),Type{AbstractString},AbstractString})
24+
@test !is_atrisk_type(Tuple{typeof(convert),Type{AbstractString},String})
25+
@test is_atrisk_type(Tuple{typeof(convert),Type{String},AbstractString})
26+
@test !is_atrisk_type(Tuple{typeof(convert),Type{Union{Int,Float32}},Int})
27+
@test !is_atrisk_type(Tuple{typeof(convert),Type{Union{Int,Float32}},Int32})
28+
@test is_atrisk_type(Tuple{typeof(convert),Type{Union{Int,Float32}},Integer})
29+
@test !is_atrisk_type(Tuple{typeof(convert),Type{T} where T<:Union{Int,Float32},Int})
30+
@test !is_atrisk_type(Tuple{typeof(map),Function,Vector{Any}})
31+
@test !is_atrisk_type(Tuple{typeof(getindex),Dict{Union{String,Int},Any},Union{String,Int}})
32+
@test is_atrisk_type(Tuple{typeof(getindex),Dict{Union{String,Int},Any},Any})
33+
@test !is_atrisk_type(Tuple{Type{BoundsError},Any,Any})
34+
@test is_atrisk_type(Tuple{typeof(sin),Any})
35+
@test !is_atrisk_type(Tuple{typeof(length),Tuple{Any,Any}})
36+
@test is_atrisk_type(Tuple{typeof(setindex!),Vector{Int},Any,Int})
37+
@test !is_atrisk_type(Tuple{typeof(setindex!),Vector{Any},Any,Int})
38+
@test is_atrisk_type(Tuple{typeof(push!),Vector{Int},Any})
39+
@test !is_atrisk_type(Tuple{typeof(push!),Vector{Any},Any})
40+
@test !is_atrisk_type(Tuple{typeof(push!),Vector{T} where T,Any})
41+
end
42+
43+
## Prepwork
44+
45+
# Count # of backedges for MethodInstances with abstract types
46+
47+
function get_atrisk_methodinstances()
48+
mivulnerabilities = Pair{MethodInstance,Int}[]
49+
for mi in methodinstances()
50+
if !fromcc(mi.def.module)
51+
if is_atrisk_type(mi.specTypes)
52+
push!(mivulnerabilities, mi => length(all_backedges(mi)))
53+
end
54+
end
55+
end
56+
return sort!(mivulnerabilities; by=last)
57+
end
58+
mivulnerabilities = get_atrisk_methodinstances()
59+
60+
# Get all the things that depends on these at-risk methodinstances
61+
const atrisk_methodinstances = Set{MethodInstance}()
62+
for (mi, c) in mivulnerabilities
63+
isdefined(mi, :backedges) && all_backedges!(atrisk_methodinstances, mi)
64+
push!(atrisk_methodinstances, mi)
65+
end
66+
67+
# Split into exported & private functions
68+
mivulnerabilities_exported = Pair{MethodInstance,Int}[]
69+
mivulnerabilities_private = similar(mivulnerabilities_exported)
70+
for (mi, c) in mivulnerabilities
71+
if isexported(mi)
72+
push!(mivulnerabilities_exported, mi=>c)
73+
else
74+
push!(mivulnerabilities_private, mi=>c)
75+
end
76+
end
77+
78+
## Tests
79+
80+
@testset "Base.require vulnerabilities" begin
81+
# Invalidating the code that loads packages leads to major slowdowns, especially if it happens repeatedly
82+
# in a dependent chain of package loads. Ideally, we'd make this code-path "bulletproof".
83+
for m in methods(Base.require)
84+
@test isempty(remove_unlikely_methodinstances(atrisk_triggers(m, first.(mivulnerabilities_exported))))
85+
86+
# It's far less important to protect against invalidation of private functions,
87+
# since generally packages should not extend these functions. Nevertheless it wouldn't
88+
# be a bad thing.
89+
@test_broken isempty(remove_unlikely_methodinstances(atrisk_triggers(m, first.(mivulnerabilities_private))))
90+
end
91+
end
92+
93+
# If you fail these tests, it may or may not be your fault---you may just be the
94+
# one that pushed one of these tests over the edge. Check `badcounts` and `meancounts`
95+
# before and after your changes; if the change is quite small, it's unlikely that your changes
96+
# are the root cause. In such cases, failures here should be investigated or reported,
97+
# but if inference problems can be fixed elsewhere you may not have to change your PR.
98+
# Conversely, if your changes *do* increase these numbers substantially, your changes have likely
99+
# triggered widespread inference problems---you should fix them before merging your PR.
100+
#
101+
# Never increase the thresholds here without public discussion. Indeed, the goal is to
102+
# shrink them as much as possible.
103+
@testset "Aggregate at-risk methodinstances" begin
104+
# Test overall number of atrisk MethodInstances and their average number of backedges
105+
badexp = Set(remove_unlikely_methodinstances(first.(mivulnerabilities_exported)))
106+
badcounts = filter(pr->pr.first ∈ badexp, mivulnerabilities_exported)
107+
@test length(badcounts) < 1250 # 1000
108+
if length(badcounts) < 800
109+
@info "There are now only $(length(badcounts)) at-risk specializations of exported methods, consider dropping the threshold"
110+
end
111+
meancounts = sum(last.(badcounts))/length(badcounts)
112+
@test meancounts < 33 # 32
113+
if meancounts < 24
114+
@info "The mean number of at-risk backedges is now only $meancounts, consider dropping the threshold"
115+
end
116+
end
117+
118+
@testset "Specific return types" begin
119+
# All the is* functions
120+
# Not all of the broken cases have been checked carefully; it's possible some of these should return
121+
# `Union{Bool,Missing}` or something.
122+
@test function_returns(isabspath, Bool)
123+
@test function_returns(isabstracttype, Bool)
124+
@test_broken function_returns(isapprox, Bool)
125+
@test_broken function_returns(isascii, Bool)
126+
# @test function_returns(isassigned, Bool)
127+
@test function_returns(isbits, Bool)
128+
@test function_returns(isbitstype, Bool)
129+
@test function_returns(isblockdev, Bool)
130+
@test function_returns(ischardev, Bool)
131+
@test function_returns(iscntrl, Bool)
132+
@test function_returns(isconcretetype, Bool)
133+
@test function_returns(isconst, Bool)
134+
@test function_returns(isdefined, Bool)
135+
@test function_returns(isdigit, Bool)
136+
@test function_returns(isdir, Bool)
137+
@test function_returns(isdirpath, Bool)
138+
@test_broken function_returns(isdisjoint, Bool)
139+
@test function_returns(isdispatchtuple, Bool)
140+
@test_broken function_returns(isempty, Bool)
141+
@test_broken function_returns(isequal, Bool; minargs=2)
142+
@test_broken function_returns(iseven, Bool)
143+
@test function_returns(isexported, Bool)
144+
@test function_returns(isfifo, Bool)
145+
@test function_returns(isfile, Bool)
146+
@test_broken function_returns(isfinite, Bool)
147+
@test_broken function_returns(isinf, Bool)
148+
@test_broken function_returns(isinteger, Bool)
149+
@test function_returns(isinteractive, Bool)
150+
@test_broken function_returns(isless, Bool)
151+
@test function_returns(isletter, Bool)
152+
@test function_returns(islink, Bool)
153+
@test function_returns(islocked, Bool)
154+
@test function_returns(islowercase, Bool)
155+
@test_broken function_returns(ismarked, Bool)
156+
@test function_returns(ismissing, Bool)
157+
@test function_returns(ismount, Bool)
158+
@test function_returns(ismutable, Bool)
159+
@test function_returns(isnan, Bool)
160+
@test function_returns(isnothing, Bool)
161+
@test function_returns(isnumeric, Bool)
162+
@test_broken function_returns(isodd, Bool)
163+
@test_broken function_returns(isone, Bool)
164+
@test_broken function_returns(isopen, Bool)
165+
@test function_returns(ispath, Bool)
166+
@test_broken function_returns(isperm, Bool)
167+
@test_broken function_returns(ispow2, Bool)
168+
@test function_returns(isprimitivetype, Bool)
169+
@test function_returns(isprint, Bool)
170+
@test function_returns(ispunct, Bool)
171+
@test_broken function_returns(isreadable, Bool)
172+
@test_broken function_returns(isreadonly, Bool)
173+
@test_broken function_returns(isready, Bool)
174+
@test_broken function_returns(isreal, Bool)
175+
@test function_returns(issetequal, Bool)
176+
@test function_returns(issetgid, Bool)
177+
@test function_returns(issetuid, Bool)
178+
@test function_returns(issocket, Bool)
179+
@test_broken function_returns(issorted, Bool)
180+
@test function_returns(isspace, Bool)
181+
@test function_returns(issticky, Bool)
182+
@test function_returns(isstructtype, Bool)
183+
@test_broken function_returns(issubnormal, Bool)
184+
@test_broken function_returns(issubset, Bool)
185+
@test function_returns(istaskdone, Bool)
186+
@test function_returns(istaskfailed, Bool)
187+
@test function_returns(istaskstarted, Bool)
188+
@test_broken function_returns(istextmime, Bool)
189+
@test function_returns(isuppercase, Bool)
190+
@test_broken function_returns(isvalid, Bool)
191+
@test_broken function_returns(iswritable, Bool)
192+
@test function_returns(isxdigit, Bool)
193+
@test_broken function_returns(iszero, Bool)
194+
195+
@test function_returns(eof, Bool)
196+
end
197+
198+
@testset "Never inferred" begin
199+
# Check that we never infer certain methodinstances,
200+
# It would be great to broaden this beyond Real, but this is a good start.
201+
# If you fail these tests, it means an internal operation forced
202+
# the compiler to infer one of these methods for a problematic combination of types.
203+
function subtype_real(@nospecialize T)
204+
while isa(T, TypeVar)
205+
T = T.ub
206+
end
207+
return T<:Real
208+
end
209+
for f in (==, isequal, <, <=)
210+
for mi in methodinstances(f)
211+
if any(subtype_real, Base.unwrap_unionall(mi.specTypes).parameters)
212+
@test !is_atrisk_type(mi.specTypes)
213+
end
214+
end
215+
end
216+
end

‎test/runtests.jl

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,14 @@ linalg_tests = tests[linalg_test_ids]
7777
deleteat!(tests, linalg_test_ids)
7878
prepend!(tests, linalg_tests)
7979

80+
# do inference_qa at the beginning (in a fresh session) to avoid trouble from specializations
81+
# introduced by running other tests
82+
idx = findfirst(isequal("inference_qa"), tests)
83+
if idx !== nothing
84+
deleteat!(tests, idx)
85+
pushfirst!(tests, "inference_qa")
86+
end
87+
8088
import LinearAlgebra
8189
cd(@__DIR__) do
8290
n = 1

0 commit comments

Comments
 (0)