From f8c6d1c106648f53874c9b0ebee1ae7382766dd0 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 31 Oct 2024 13:21:46 -0400 Subject: [PATCH 01/10] Fix some missing write barriers and add some helpful comments (#56396) I was trying some performance optimization which didn't end up working out, but in the process I found two missing write barriers and added some helpful comments for future readers, so that part is probably still useful. --- base/runtime_internals.jl | 2 +- src/gc-interface.h | 5 +++++ src/julia.h | 6 +++--- src/module.c | 10 +++++++--- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index ab867f8fcae6d..dd526a24d6494 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -248,7 +248,7 @@ binding_kind(m::Module, s::Symbol) = binding_kind(lookup_binding_partition(tls_w delete_binding(mod::Module, sym::Symbol) Force the binding `mod.sym` to be undefined again, allowing it be redefined. -Note that this operation is very expensive, requirinig a full scan of all code in the system, +Note that this operation is very expensive, requiring a full scan of all code in the system, as well as potential recompilation of any methods that (may) have used binding information. diff --git a/src/gc-interface.h b/src/gc-interface.h index 0b5df17a3b8c5..eb6687d52d9ab 100644 --- a/src/gc-interface.h +++ b/src/gc-interface.h @@ -192,6 +192,11 @@ JL_DLLEXPORT void *jl_gc_perm_alloc(size_t sz, int zero, unsigned align, // object header must be included in the object size. This object is allocated in an // immortal region that is never swept. The second parameter specifies the type of the // object being allocated and will be used to set the object header. +// +// !!! warning: Because permanently allocated objects are not swept, the GC will not +// necessarily mark any objects that would have ordinarily been rooted by +// the allocated object. All objects stored in fields of this object +// must be either permanently allocated or have other roots. struct _jl_value_t *jl_gc_permobj(size_t sz, void *ty) JL_NOTSAFEPOINT; // ========================================================================= // diff --git a/src/julia.h b/src/julia.h index ffd669cd828a4..a710192d5756c 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1138,15 +1138,15 @@ JL_DLLEXPORT jl_weakref_t *jl_gc_new_weakref(jl_value_t *value); STATIC_INLINE void jl_gc_wb(const void *parent, const void *ptr) JL_NOTSAFEPOINT { // parent and ptr isa jl_value_t* - if (__unlikely(jl_astaggedvalue(parent)->bits.gc == 3 && // parent is old and not in remset - (jl_astaggedvalue(ptr)->bits.gc & 1) == 0)) // ptr is young + if (__unlikely(jl_astaggedvalue(parent)->bits.gc == 3 /* GC_OLD_MARKED */ && // parent is old and not in remset + (jl_astaggedvalue(ptr)->bits.gc & 1 /* GC_MARKED */) == 0)) // ptr is young jl_gc_queue_root((jl_value_t*)parent); } STATIC_INLINE void jl_gc_wb_back(const void *ptr) JL_NOTSAFEPOINT // ptr isa jl_value_t* { // if ptr is old - if (__unlikely(jl_astaggedvalue(ptr)->bits.gc == 3)) { + if (__unlikely(jl_astaggedvalue(ptr)->bits.gc == 3 /* GC_OLD_MARKED */)) { jl_gc_queue_root((jl_value_t*)ptr); } } diff --git a/src/module.c b/src/module.c index 9b4d26cc7b000..bdacd487e978d 100644 --- a/src/module.c +++ b/src/module.c @@ -37,6 +37,7 @@ jl_binding_partition_t *jl_get_binding_partition(jl_binding_t *b, size_t world) _Atomic(jl_binding_partition_t *)*insert = &b->partitions; jl_binding_partition_t *bpart = jl_atomic_load_relaxed(insert); size_t max_world = (size_t)-1; + jl_binding_partition_t *new_bpart = NULL; while (1) { while (bpart && world < bpart->min_world) { insert = &bpart->next; @@ -46,11 +47,11 @@ jl_binding_partition_t *jl_get_binding_partition(jl_binding_t *b, size_t world) } if (bpart && world <= jl_atomic_load_relaxed(&bpart->max_world)) return bpart; - jl_binding_partition_t *new_bpart = new_binding_partition(); + if (!new_bpart) + new_bpart = new_binding_partition(); jl_atomic_store_relaxed(&new_bpart->next, bpart); jl_gc_wb_fresh(new_bpart, bpart); - if (bpart) - new_bpart->min_world = jl_atomic_load_relaxed(&bpart->max_world) + 1; + new_bpart->min_world = bpart ? jl_atomic_load_relaxed(&bpart->max_world) + 1 : 0; jl_atomic_store_relaxed(&new_bpart->max_world, max_world); if (jl_atomic_cmpswap(insert, &bpart, new_bpart)) { jl_gc_wb(parent, new_bpart); @@ -548,6 +549,7 @@ static jl_binding_t *jl_resolve_owner(jl_binding_t *b/*optional*/, jl_module_t * // changing, for example if this var is assigned to later. if (!jl_atomic_cmpswap(&bpart->restriction, &pku, encode_restriction((jl_value_t*)b2, BINDING_KIND_IMPLICIT))) goto retry; + jl_gc_wb(bpart, b2); if (b2->deprecated) { b->deprecated = 1; // we will warn about this below, but we might want to warn at the use sites too if (m != jl_main_module && m != jl_base_module && @@ -740,6 +742,7 @@ static void module_import_(jl_module_t *to, jl_module_t *from, jl_sym_t *asname, jl_ptr_kind_union_t new_pku = encode_restriction((jl_value_t*)b, (explici != 0) ? BINDING_KIND_IMPORTED : BINDING_KIND_EXPLICIT); if (!jl_atomic_cmpswap(&btopart->restriction, &bto_pku, new_pku)) goto retry; + jl_gc_wb(btopart, b); bto->deprecated |= b->deprecated; // we already warned about this above, but we might want to warn at the use sites too } else { @@ -749,6 +752,7 @@ static void module_import_(jl_module_t *to, jl_module_t *from, jl_sym_t *asname, jl_ptr_kind_union_t new_pku = encode_restriction(decode_restriction_value(bto_pku), (explici != 0) ? BINDING_KIND_IMPORTED : BINDING_KIND_EXPLICIT); if (!jl_atomic_cmpswap(&btopart->restriction, &bto_pku, new_pku)) goto retry; + // No wb, because the value is unchanged } } else if (jl_bkind_is_some_import(decode_restriction_kind(bto_pku))) { From da74ef1933b12410b217748e0f7fbcbe52e10d29 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 31 Oct 2024 14:15:39 -0400 Subject: [PATCH 02/10] compiler: fix specialization mistake introduced by #40985 (#56404) Hopefully there aren't any others like this hiding around? Not useful to make a new closure for every method that we inline, since we just called `===` inside it --- base/compiler/ssair/inlining.jl | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl index 5017b619469ff..dfdd317f74d87 100644 --- a/base/compiler/ssair/inlining.jl +++ b/base/compiler/ssair/inlining.jl @@ -939,7 +939,6 @@ function analyze_method!(match::MethodMatch, argtypes::Vector{Any}, allow_typevars::Bool, invokesig::Union{Nothing,Vector{Any}}=nothing, volatile_inf_result::Union{Nothing,VolatileInferenceResult}=nothing) method = match.method - spec_types = match.spec_types # Check that we have the correct number of arguments na = Int(method.nargs) @@ -954,6 +953,7 @@ function analyze_method!(match::MethodMatch, argtypes::Vector{Any}, if !match.fully_covers # type-intersection was not able to give us a simple list of types, so # ir_inline_unionsplit won't be able to deal with inlining this + spec_types = match.spec_types if !(spec_types isa DataType && length(spec_types.parameters) == npassedargs && !isvarargtype(spec_types.parameters[end])) return nothing @@ -1428,14 +1428,13 @@ function handle_match!(cases::Vector{InliningCase}, match::MethodMatch, argtypes::Vector{Any}, @nospecialize(info::CallInfo), flag::UInt32, state::InliningState; allow_typevars::Bool, volatile_inf_result::Union{Nothing,VolatileInferenceResult}) - spec_types = match.spec_types # We may see duplicated dispatch signatures here when a signature gets widened # during abstract interpretation: for the purpose of inlining, we can just skip # processing this dispatch candidate (unless unmatched type parameters are present) - !allow_typevars && any(case::InliningCase->case.sig === spec_types, cases) && return true + !allow_typevars && any(case::InliningCase->case.sig === match.spec_types, cases) && return true item = analyze_method!(match, argtypes, info, flag, state; allow_typevars, volatile_inf_result) item === nothing && return false - push!(cases, InliningCase(spec_types, item)) + push!(cases, InliningCase(match.spec_types, item)) return true end @@ -1443,13 +1442,12 @@ function handle_const_prop_result!(cases::Vector{InliningCase}, result::ConstPro match::MethodMatch, @nospecialize(info::CallInfo), flag::UInt32, state::InliningState; allow_typevars::Bool) mi = result.result.linfo - spec_types = match.spec_types if !validate_sparams(mi.sparam_vals) (allow_typevars && !may_have_fcalls(mi.def::Method)) || return false end item = resolve_todo(mi, result.result, info, flag, state) item === nothing && return false - push!(cases, InliningCase(spec_types, item)) + push!(cases, InliningCase(match.spec_types, item)) return true end @@ -1479,11 +1477,10 @@ end function handle_semi_concrete_result!(cases::Vector{InliningCase}, result::SemiConcreteResult, match::MethodMatch, @nospecialize(info::CallInfo), flag::UInt32, state::InliningState) mi = result.mi - spec_types = match.spec_types validate_sparams(mi.sparam_vals) || return false item = semiconcrete_result_item(result, info, flag, state) item === nothing && return false - push!(cases, InliningCase(spec_types, item)) + push!(cases, InliningCase(match.spec_types, item)) return true end From 7715cf287a9920ba86cf7405f636b18b85eede47 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 31 Oct 2024 19:07:10 -0400 Subject: [PATCH 03/10] Avoid racy double-load of binding restriction in `import_module` (#56395) Fixes #56333 --- src/toplevel.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/toplevel.c b/src/toplevel.c index c2fbc38d067eb..6dcab3095e320 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -665,11 +665,16 @@ static void import_module(jl_module_t *JL_NONNULL m, jl_module_t *import, jl_sym jl_sym_t *name = asname ? asname : import->name; // TODO: this is a bit race-y with what error message we might print jl_binding_t *b = jl_get_module_binding(m, name, 1); - if (jl_get_binding_value_if_const(b) == (jl_value_t*)import) - return; jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); jl_ptr_kind_union_t pku = jl_atomic_load_relaxed(&bpart->restriction); if (decode_restriction_kind(pku) != BINDING_KIND_GUARD && decode_restriction_kind(pku) != BINDING_KIND_FAILED) { + // Unlike regular constant declaration, we allow this as long as we eventually end up at a constant. + pku = jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); + if (decode_restriction_kind(pku) == BINDING_KIND_CONST || decode_restriction_kind(pku) == BINDING_KIND_CONST_IMPORT) { + // Already declared (e.g. on another thread) or imported. + if (decode_restriction_value(pku) == (jl_value_t*)import) + return; + } jl_errorf("importing %s into %s conflicts with an existing global", jl_symbol_name(name), jl_symbol_name(m->name)); } From 040174c446ba63b89a13b6de267fced9ad270848 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Fri, 1 Nov 2024 14:56:36 +0900 Subject: [PATCH 04/10] define `InteractiveUtils.@infer_[return|exception]_type` (#56398) Also simplifies the definitions of `@code_typed` and the other similar macros. --- stdlib/InteractiveUtils/src/macros.jl | 40 ++++++++---------------- stdlib/InteractiveUtils/test/runtests.jl | 2 ++ 2 files changed, 15 insertions(+), 27 deletions(-) diff --git a/stdlib/InteractiveUtils/src/macros.jl b/stdlib/InteractiveUtils/src/macros.jl index 0c272940b82d9..a21bf30dbcd6c 100644 --- a/stdlib/InteractiveUtils/src/macros.jl +++ b/stdlib/InteractiveUtils/src/macros.jl @@ -2,7 +2,8 @@ # macro wrappers for various reflection functions -import Base: typesof, insert!, replace_ref_begin_end!, infer_effects, code_ircode +using Base: typesof, insert!, replace_ref_begin_end!, + infer_return_type, infer_exception_type, infer_effects, code_ircode # defined in Base so it's possible to time all imports, including InteractiveUtils and its deps # via. `Base.@time_imports` etc. @@ -225,35 +226,20 @@ macro which(ex0::Symbol) return :(which($__module__, $ex0)) end -for fname in [:code_warntype, :code_llvm, :code_native, :infer_effects] - @eval begin - macro ($fname)(ex0...) - gen_call_with_extracted_types_and_kwargs(__module__, $(Expr(:quote, fname)), ex0) - end - end -end - -macro code_typed(ex0...) - thecall = gen_call_with_extracted_types_and_kwargs(__module__, :code_typed, ex0) - quote - local results = $thecall - length(results) == 1 ? results[1] : results +for fname in [:code_warntype, :code_llvm, :code_native, + :infer_return_type, :infer_effects, :infer_exception_type] + @eval macro ($fname)(ex0...) + gen_call_with_extracted_types_and_kwargs(__module__, $(QuoteNode(fname)), ex0) end end -macro code_lowered(ex0...) - thecall = gen_call_with_extracted_types_and_kwargs(__module__, :code_lowered, ex0) - quote - local results = $thecall - length(results) == 1 ? results[1] : results - end -end - -macro code_ircode(ex0...) - thecall = gen_call_with_extracted_types_and_kwargs(__module__, :code_ircode, ex0) - quote - local results = $thecall - length(results) == 1 ? results[1] : results +for fname in [:code_typed, :code_lowered, :code_ircode] + @eval macro ($fname)(ex0...) + thecall = gen_call_with_extracted_types_and_kwargs(__module__, $(QuoteNode(fname)), ex0) + quote + local results = $thecall + length(results) == 1 ? results[1] : results + end end end diff --git a/stdlib/InteractiveUtils/test/runtests.jl b/stdlib/InteractiveUtils/test/runtests.jl index bb64153818c1d..0de67fea69dea 100644 --- a/stdlib/InteractiveUtils/test/runtests.jl +++ b/stdlib/InteractiveUtils/test/runtests.jl @@ -819,6 +819,8 @@ end end @test Base.infer_effects(sin, (Int,)) == InteractiveUtils.@infer_effects sin(42) +@test Base.infer_return_type(sin, (Int,)) == InteractiveUtils.@infer_return_type sin(42) +@test Base.infer_exception_type(sin, (Int,)) == InteractiveUtils.@infer_exception_type sin(42) @test first(InteractiveUtils.@code_ircode sin(42)) isa Core.Compiler.IRCode @test first(InteractiveUtils.@code_ircode optimize_until="Inlining" sin(42)) isa Core.Compiler.IRCode From 671e1d87595d706ec2fa1659d990c507596f8fbc Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Fri, 1 Nov 2024 15:02:11 +0900 Subject: [PATCH 05/10] irinterp: set `IR_FLAG_REFINED` for narrowed `PhiNode`s (#56391) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `adce_pass!` can transform a `Union`-type `PhiNode` into a narrower `PhiNode`, but in such cases, the `IR_FLAG_REFINED` flag isn’t set on that `PhiNode` statement. By setting this flag, irinterp can perform statement reprocessing using the narrowed `PhiNode`, enabling type stability in cases like JuliaLang/julia#56387. - fixes JuliaLang/julia#56387 --- base/compiler/ssair/passes.jl | 12 +++++++----- test/compiler/inference.jl | 12 ++++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index e3f294c4e91fe..4ad5dcfb2a3c8 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -2114,18 +2114,19 @@ function adce_pass!(ir::IRCode, inlining::Union{Nothing,InliningState}=nothing) unionphi = unionphis[i] phi = unionphi[1] t = unionphi[2] + inst = compact.result[phi] if t === Union{} - stmt = compact[SSAValue(phi)][:stmt]::PhiNode + stmt = inst[:stmt]::PhiNode kill_phi!(compact, phi_uses, 1:length(stmt.values), SSAValue(phi), stmt, true) made_changes = true continue elseif t === Any continue - elseif βŠ‘(𝕃ₒ, compact.result[phi][:type], t) - continue end + ⊏ = strictpartialorder(𝕃ₒ) + t ⊏ inst[:type] || continue to_drop = Int[] - stmt = compact[SSAValue(phi)][:stmt] + stmt = inst[:stmt] stmt === nothing && continue stmt = stmt::PhiNode for i = 1:length(stmt.values) @@ -2137,7 +2138,8 @@ function adce_pass!(ir::IRCode, inlining::Union{Nothing,InliningState}=nothing) push!(to_drop, i) end end - compact.result[phi][:type] = t + inst[:type] = t + add_flag!(inst, IR_FLAG_REFINED) # t ⊏ inst[:type] kill_phi!(compact, phi_uses, to_drop, SSAValue(phi), stmt, false) made_changes = true end diff --git a/test/compiler/inference.jl b/test/compiler/inference.jl index dab8e57aa2309..2fc7e917186f4 100644 --- a/test/compiler/inference.jl +++ b/test/compiler/inference.jl @@ -6078,3 +6078,15 @@ let src = code_typed((Union{Nothing,AtomicModifySafety},)) do x end |> only |> first @test any(@nospecialize(x)->Meta.isexpr(x, :invoke_modify), src.code) end + +function issue56387(nt::NamedTuple, field::Symbol=:a) + NT = typeof(nt) + names = fieldnames(NT) + types = fieldtypes(NT) + index = findfirst(==(field), names) + if index === nothing + throw(ArgumentError("Field $field not found")) + end + types[index] +end +@test Base.infer_return_type(issue56387, (typeof((;a=1)),)) == Type{Int} From dc57caf2470fcdc21a9bb11c0f97924577d7c5c8 Mon Sep 17 00:00:00 2001 From: Micah Rufsvold <86363075+mrufsvold@users.noreply.github.com> Date: Fri, 1 Nov 2024 02:03:42 -0400 Subject: [PATCH 06/10] document isopen(::Channel) (#56376) This PR has two purposes -- 1) Add some documentation for public API 2) Add a small note about a footgun I've hit a few times: `!isopen(ch)` does not mean that you are "done" with the channel because buffered channels can still have items left in them that need to be taken. --------- Co-authored-by: CY Han --- base/channels.jl | 50 ++++++++++++++++++++++++++++++++++++---- doc/src/base/parallel.md | 1 + 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/base/channels.jl b/base/channels.jl index 3acbf37246a58..8882171095e7a 100644 --- a/base/channels.jl +++ b/base/channels.jl @@ -212,11 +212,51 @@ function close(c::Channel, @nospecialize(excp::Exception)) nothing end -# Use acquire here to pair with release store in `close`, so that subsequent `isready` calls -# are forced to see `isready == true` if they see `isopen == false`. This means users must -# call `isopen` before `isready` if you are using the race-y APIs (or call `iterate`, which -# does this right for you). -isopen(c::Channel) = ((@atomic :acquire c.state) === :open) +""" + isopen(c::Channel) +Determines whether a [`Channel`](@ref) is open for new [`put!`](@ref) operations. +Notice that a `Channel`` can be closed and still have +buffered elements which can be consumed with [`take!`](@ref). + +# Examples + +Buffered channel with task: +```jldoctest +julia> c = Channel(ch -> put!(ch, 1), 1); + +julia> isopen(c) # The channel is closed to new `put!`s +false + +julia> isready(c) # The channel is closed but still contains elements +true + +julia> take!(c) +1 + +julia> isready(c) +false +``` + +Unbuffered channel: +```jldoctest +julia> c = Channel{Int}(); + +julia> isopen(c) +true + +julia> close(c) + +julia> isopen(c) +false +``` +""" +function isopen(c::Channel) + # Use acquire here to pair with release store in `close`, so that subsequent `isready` calls + # are forced to see `isready == true` if they see `isopen == false`. This means users must + # call `isopen` before `isready` if you are using the race-y APIs (or call `iterate`, which + # does this right for you). + return ((@atomic :acquire c.state) === :open) +end """ empty!(c::Channel) diff --git a/doc/src/base/parallel.md b/doc/src/base/parallel.md index 9f24db176b538..cd5c95f17994a 100644 --- a/doc/src/base/parallel.md +++ b/doc/src/base/parallel.md @@ -67,6 +67,7 @@ Base.put!(::Channel, ::Any) Base.take!(::Channel) Base.isfull(::Channel) Base.isready(::Channel) +Base.isopen(::Channel) Base.fetch(::Channel) Base.close(::Channel) Base.bind(c::Channel, task::Task) From 706a4f6c5d159366bed25e8217ce80748e3963fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= <765740+giordano@users.noreply.github.com> Date: Fri, 1 Nov 2024 06:13:06 +0000 Subject: [PATCH 07/10] Make build system respect `FORCE_COLOR` and `NO_COLOR` settings (#56346) Follow up to #53742, but for the build system. CC: @omus. --- Make.inc | 63 ++++++++++++++++++++++++++++++++++++++++++-------------- Makefile | 2 +- NEWS.md | 4 ++-- 3 files changed, 50 insertions(+), 19 deletions(-) diff --git a/Make.inc b/Make.inc index cb79e3ca1b5a9..9f6535ae05885 100644 --- a/Make.inc +++ b/Make.inc @@ -118,6 +118,51 @@ SPACE:=$(eval) $(eval) export LC_ALL=C export LANG=C +# Respect `FORCE_COLOR` environment variable: . +ifndef FORCE_COLOR +FORCE_COLOR := "" +endif + +# Respect `NO_COLOR` environment variable: . +ifndef NO_COLOR +NO_COLOR := "" +endif + +# When both `FORCE_COLOR` and `NO_COLOR` are defined, the former has precedence. +ifneq ($(FORCE_COLOR), "") +NO_COLOR = "" +endif + +WARNCOLOR:="\033[33;1m" +ENDCOLOR:="\033[0m" + +CCCOLOR:="\033[34m" +LINKCOLOR:="\033[34;1m" +PERLCOLOR:="\033[35m" +FLISPCOLOR:="\033[32m" +JULIACOLOR:="\033[32;1m" +DTRACECOLOR:="\033[32;1m" + +SRCCOLOR:="\033[33m" +BINCOLOR:="\033[37;1m" +JULCOLOR:="\033[34;1m" + +ifneq ($(NO_COLOR), "") +WARNCOLOR:="" +ENDCOLOR:="" + +CCCOLOR:="" +LINKCOLOR:="" +PERLCOLOR:="" +FLISPCOLOR:="" +JULIACOLOR:="" +DTRACECOLOR:="" + +SRCCOLOR:="" +BINCOLOR:="" +JULCOLOR:="" +endif + # We need python for things like BB triplet recognition and relative path computation. # We don't really care about version, generally, so just find something that works: PYTHON := "$(shell which python 2>/dev/null || which python3 2>/dev/null || which python2 2>/dev/null || echo "{python|python3|python2} not found")" @@ -140,7 +185,7 @@ ifeq ($(BUILDROOT),) ifeq ("$(origin O)", "command line") BUILDROOT := $(abspath $O) BUILDDIR := $(abspath $(BUILDROOT)/$(call rel_path,$(JULIAHOME),$(SRCDIR))) - $(info $(shell printf '\033[32;1mBuilding into $(BUILDROOT)\033[0m')) # use printf to expand the escape sequences + $(info $(shell printf '$(JULIACOLOR)Building into $(BUILDROOT)$(ENDCOLOR)')) # use printf to expand the escape sequences else BUILDROOT:=$(JULIAHOME) endif @@ -1759,24 +1804,10 @@ ifndef VERBOSE VERBOSE := 0 endif -WARNCOLOR:="\033[33;1m" -ENDCOLOR:="\033[0m" - ifeq ($(VERBOSE), 0) QUIET_MAKE = -s -CCCOLOR:="\033[34m" -LINKCOLOR:="\033[34;1m" -PERLCOLOR:="\033[35m" -FLISPCOLOR:="\033[32m" -JULIACOLOR:="\033[32;1m" -DTRACECOLOR:="\033[32;1m" - -SRCCOLOR:="\033[33m" -BINCOLOR:="\033[37;1m" -JULCOLOR:="\033[34;1m" - GOAL=$(subst ','\'',$(subst $(abspath $(JULIAHOME))/,,$(abspath $@))) PRINT_CC = printf ' %b %b\n' $(CCCOLOR)CC$(ENDCOLOR) $(SRCCOLOR)$(GOAL)$(ENDCOLOR); $(1) @@ -1797,7 +1828,7 @@ PRINT_FLISP = echo '$(subst ','\'',$(1))'; $(1) PRINT_JULIA = echo '$(subst ','\'',$(1))'; $(1) PRINT_DTRACE = echo '$(subst ','\'',$(1))'; $(1) -endif +endif # VERBOSE # Makefile debugging trick: # call print-VARIABLE to see the runtime value of any variable diff --git a/Makefile b/Makefile index 4fd8b878c5d1f..d1e5b31f85b1c 100644 --- a/Makefile +++ b/Makefile @@ -650,7 +650,7 @@ testall1: check-whitespace $(JULIA_BUILD_MODE) test-%: check-whitespace $(JULIA_BUILD_MODE) .FORCE @([ $$(( $$(date +%s) - $$(date -r $(build_private_libdir)/sys.$(SHLIB_EXT) +%s) )) -le 100 ] && \ - printf '\033[93m HINT The system image was recently rebuilt. Are you aware of the test-revise-* targets? See CONTRIBUTING.md. \033[0m\n') || true + printf '$(WARNCOLOR) HINT The system image was recently rebuilt. Are you aware of the test-revise-* targets? See CONTRIBUTING.md. $(ENDCOLOR)\n') || true @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test $* JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) test-revise-%: .FORCE diff --git a/NEWS.md b/NEWS.md index 5e066ffd9cdcf..ba9ca1c521c55 100644 --- a/NEWS.md +++ b/NEWS.md @@ -57,8 +57,8 @@ Command-line option changes * The `-m/--module` flag can be passed to run the `main` function inside a package with a set of arguments. This `main` function should be declared using `@main` to indicate that it is an entry point. * Enabling or disabling color text in Julia can now be controlled with the -[`NO_COLOR`](https://no-color.org/) or [`FORCE_COLOR`](https://force-color.org/) environment -variables. ([#53742]). + [`NO_COLOR`](https://no-color.org/) or [`FORCE_COLOR`](https://force-color.org/) environment + variables. These variables are also honored by Julia's build system ([#53742], [#56346]). * `--project=@temp` starts Julia with a temporary environment. * New `--trace-compile-timing` option to report how long each method reported by `--trace-compile` took to compile, in ms. ([#54662]) From 76351909e2daf7cb07cffe9fa76d3e8e4d38137f Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 1 Nov 2024 06:15:16 -0400 Subject: [PATCH 08/10] Add `edges` vector to CodeInstance/CodeInfo to keep backedges as edges (#54894) Appears to add about 11MB (128MB to 139MB) to the system image, and to decrease the stdlib size by 55 MB (325MB to 270MB), so seems overall favorable right now. The edges are computed following the encoding to correctly reflect the backedges. Co-authored-by: Shuhei Kadowaki --- base/Base.jl | 1 - base/boot.jl | 6 +- base/compiler/abstractinterpretation.jl | 215 +++--- base/compiler/inferencestate.jl | 62 +- base/compiler/optimize.jl | 4 +- base/compiler/ssair/inlining.jl | 92 ++- base/compiler/ssair/irinterp.jl | 6 - base/compiler/ssair/passes.jl | 4 +- base/compiler/stmtinfo.jl | 220 ++++++- base/compiler/tfuncs.jl | 29 +- base/compiler/typeinfer.jl | 268 ++++---- base/compiler/types.jl | 53 +- base/compiler/utilities.jl | 37 +- base/expr.jl | 2 +- base/show.jl | 6 +- src/common_symbols1.inc | 2 - src/common_symbols2.inc | 4 +- src/gf.c | 126 ++-- src/ircode.c | 31 +- src/jltypes.c | 14 +- src/julia.h | 5 +- src/julia_internal.h | 15 +- src/method.c | 70 +- src/opaque_closure.c | 12 +- src/serialize.h | 90 +-- src/staticdata.c | 75 +-- src/staticdata_utils.c | 827 +++++++++--------------- src/toplevel.c | 2 +- stdlib/REPL/src/REPLCompletions.jl | 6 +- test/compiler/AbstractInterpreter.jl | 11 +- test/compiler/EscapeAnalysis/EAUtils.jl | 5 +- test/compiler/contextual.jl | 4 +- test/compiler/invalidation.jl | 11 +- test/core.jl | 2 +- test/precompile.jl | 32 +- test/precompile_absint1.jl | 10 +- test/precompile_absint2.jl | 17 +- test/stacktraces.jl | 5 +- 38 files changed, 1159 insertions(+), 1222 deletions(-) diff --git a/base/Base.jl b/base/Base.jl index c5e318ffe5e38..3b56dca166cee 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -204,7 +204,6 @@ function Core._hasmethod(@nospecialize(f), @nospecialize(t)) # this function has return Core._hasmethod(tt) end - # core operations & types include("promotion.jl") include("tuple.jl") diff --git a/base/boot.jl b/base/boot.jl index ed3e22391f215..5d40191ecab21 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -535,11 +535,11 @@ function CodeInstance( mi::MethodInstance, owner, @nospecialize(rettype), @nospecialize(exctype), @nospecialize(inferred_const), @nospecialize(inferred), const_flags::Int32, min_world::UInt, max_world::UInt, effects::UInt32, @nospecialize(analysis_results), - relocatability::UInt8, edges::Union{DebugInfo,Nothing}) + relocatability::UInt8, di::Union{DebugInfo,Nothing}, edges::SimpleVector) return ccall(:jl_new_codeinst, Ref{CodeInstance}, - (Any, Any, Any, Any, Any, Any, Int32, UInt, UInt, UInt32, Any, UInt8, Any), + (Any, Any, Any, Any, Any, Any, Int32, UInt, UInt, UInt32, Any, UInt8, Any, Any), mi, owner, rettype, exctype, inferred_const, inferred, const_flags, min_world, max_world, - effects, analysis_results, relocatability, edges) + effects, analysis_results, relocatability, di, edges) end GlobalRef(m::Module, s::Symbol) = ccall(:jl_module_globalref, Ref{GlobalRef}, (Any, Any), m, s) Module(name::Symbol=:anonymous, std_imports::Bool=true, default_names::Bool=true) = ccall(:jl_f_new_module, Ref{Module}, (Any, Bool, Bool), name, std_imports, default_names) diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index 777240adf581b..e20b74454bb22 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -57,7 +57,6 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), gfresult = Future{CallMeta}() # intermediate work for computing gfresult rettype = exctype = Bottom - edges = MethodInstance[] conditionals = nothing # keeps refinement information of call argument types when the return type is boolean seenall = true const_results = nothing # or const_results::Vector{Union{Nothing,ConstResult}} if any const results are available @@ -73,7 +72,7 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), napplicable = length(applicable) multiple_matches = napplicable > 1 while i <= napplicable - match = applicable[i]::MethodMatch + (; match, edges, edge_idx) = applicable[i] method = match.method sig = match.spec_types if bail_out_toplevel_call(interp, InferenceLoopState(sig, rettype, all_effects), sv) @@ -95,7 +94,7 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), #end mresult = abstract_call_method(interp, method, sig, match.sparams, multiple_matches, si, sv)::Future function handle1(interp, sv) - local (; rt, exct, edge, effects, volatile_inf_result) = mresult[] + local (; rt, exct, effects, edge, volatile_inf_result) = mresult[] this_conditional = ignorelimited(rt) this_rt = widenwrappedconditional(rt) this_exct = exct @@ -109,6 +108,7 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), if const_call_result !== nothing this_const_conditional = ignorelimited(const_call_result.rt) this_const_rt = widenwrappedconditional(const_call_result.rt) + const_edge = nothing if this_const_rt βŠ‘β‚š this_rt # As long as the const-prop result we have is not *worse* than # what we found out on types, we'd like to use it. Even if the @@ -119,9 +119,9 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), # e.g. in cases when there are cycles but cached result is still accurate this_conditional = this_const_conditional this_rt = this_const_rt - (; effects, const_result, edge) = const_call_result + (; effects, const_result, const_edge) = const_call_result elseif is_better_effects(const_call_result.effects, effects) - (; effects, const_result, edge) = const_call_result + (; effects, const_result, const_edge) = const_call_result else add_remark!(interp, sv, "[constprop] Discarded because the result was wider than inference") end @@ -129,10 +129,13 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), # because consistent-cy does not apply to exceptions. if const_call_result.exct β‹€ this_exct this_exct = const_call_result.exct - (; const_result, edge) = const_call_result + (; const_result, const_edge) = const_call_result else add_remark!(interp, sv, "[constprop] Discarded exception type because result was wider than inference") end + if const_edge !== nothing + edge = const_edge + end end all_effects = merge_effects(all_effects, effects) @@ -142,7 +145,6 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), end const_results[i] = const_result end - edge === nothing || push!(edges, edge) @assert !(this_conditional isa Conditional || this_rt isa MustAlias) "invalid lattice element returned from inter-procedural context" if can_propagate_conditional(this_conditional, argtypes) # The only case where we need to keep this in rt is where @@ -165,6 +167,7 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), conditionals[2][i] = conditionals[2][i] βŠ”α΅’ cnd.elsetype end end + edges[edge_idx] = edge if i < napplicable && bail_out_call(interp, InferenceLoopState(sig, rettype, all_effects), sv) add_remark!(interp, sv, "Call inference reached maximally imprecise information. Bailing on.") seenall = false @@ -172,7 +175,7 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), end i += 1 return true - end + end # function handle1 if isready(mresult) && handle1(interp, sv) continue else @@ -208,7 +211,7 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), if (isa(sv, InferenceState) && infer_compilation_signature(interp) && (seenall && 1 == napplicable) && rettype !== Any && rettype !== Bottom && !is_removable_if_unused(all_effects)) - match = applicable[1]::MethodMatch + (; match) = applicable[1] method = match.method sig = match.spec_types mi = specialize_method(match; preexisting=true) @@ -230,8 +233,6 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), # and avoid keeping track of a more complex result type. rettype = Any end - any_slot_refined = slotrefinements !== nothing - add_call_backedges!(interp, rettype, all_effects, any_slot_refined, edges, matches, atype.contents, sv) if isa(sv, InferenceState) # TODO (#48913) implement a proper recursion handling for irinterp: # This works just because currently the `:terminate` condition guarantees that @@ -247,7 +248,7 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), gfresult[] = CallMeta(rettype, exctype, all_effects, info, slotrefinements) return true - end # infercalls + end # function infercalls # start making progress on the first call infercalls(interp, sv) || push!(sv.tasks, infercalls) return gfresult @@ -257,8 +258,14 @@ struct FailedMethodMatch reason::String end +struct MethodMatchTarget + match::MethodMatch + edges::Vector{Union{Nothing,CodeInstance}} + edge_idx::Int +end + struct MethodMatches - applicable::Vector{Any} + applicable::Vector{MethodMatchTarget} info::MethodMatchInfo valid_worlds::WorldRange end @@ -267,15 +274,9 @@ any_ambig(info::MethodMatchInfo) = any_ambig(info.results) any_ambig(m::MethodMatches) = any_ambig(m.info) fully_covering(info::MethodMatchInfo) = info.fullmatch fully_covering(m::MethodMatches) = fully_covering(m.info) -function add_uncovered_edges!(sv::AbsIntState, info::MethodMatchInfo, @nospecialize(atype)) - fully_covering(info) || add_mt_backedge!(sv, info.mt, atype) - nothing -end -add_uncovered_edges!(sv::AbsIntState, matches::MethodMatches, @nospecialize(atype)) = - add_uncovered_edges!(sv, matches.info, atype) struct UnionSplitMethodMatches - applicable::Vector{Any} + applicable::Vector{MethodMatchTarget} applicable_argtypes::Vector{Vector{Any}} info::UnionSplitInfo valid_worlds::WorldRange @@ -284,23 +285,14 @@ any_ambig(info::UnionSplitInfo) = any(any_ambig, info.split) any_ambig(m::UnionSplitMethodMatches) = any_ambig(m.info) fully_covering(info::UnionSplitInfo) = all(fully_covering, info.split) fully_covering(m::UnionSplitMethodMatches) = fully_covering(m.info) -function add_uncovered_edges!(sv::AbsIntState, info::UnionSplitInfo, @nospecialize(atype)) - all(fully_covering, info.split) && return nothing - # add mt backedges with removing duplications - for mt in uncovered_method_tables(info) - add_mt_backedge!(sv, mt, atype) - end -end -add_uncovered_edges!(sv::AbsIntState, matches::UnionSplitMethodMatches, @nospecialize(atype)) = - add_uncovered_edges!(sv, matches.info, atype) -function uncovered_method_tables(info::UnionSplitInfo) - mts = MethodTable[] + +nmatches(info::MethodMatchInfo) = length(info.results) +function nmatches(info::UnionSplitInfo) + n = 0 for mminfo in info.split - fully_covering(mminfo) && continue - any(mtβ€²::MethodTable->mtβ€²===mminfo.mt, mts) && continue - push!(mts, mminfo.mt) + n += nmatches(mminfo) end - return mts + return n end function find_method_matches(interp::AbstractInterpreter, argtypes::Vector{Any}, @nospecialize(atype); @@ -320,7 +312,7 @@ function find_union_split_method_matches(interp::AbstractInterpreter, argtypes:: @nospecialize(atype), max_methods::Int) split_argtypes = switchtupleunion(typeinf_lattice(interp), argtypes) infos = MethodMatchInfo[] - applicable = Any[] + applicable = MethodMatchTarget[] applicable_argtypes = Vector{Any}[] # arrays like `argtypes`, including constants, for each match valid_worlds = WorldRange() for i in 1:length(split_argtypes) @@ -333,14 +325,14 @@ function find_union_split_method_matches(interp::AbstractInterpreter, argtypes:: if thismatches === nothing return FailedMethodMatch("For one of the union split cases, too many methods matched") end - for m in thismatches - push!(applicable, m) - push!(applicable_argtypes, arg_n) - end valid_worlds = intersect(valid_worlds, thismatches.valid_worlds) thisfullmatch = any(match::MethodMatch->match.fully_covers, thismatches) - thisinfo = MethodMatchInfo(thismatches, mt, thisfullmatch) + thisinfo = MethodMatchInfo(thismatches, mt, sig_n, thisfullmatch) push!(infos, thisinfo) + for idx = 1:length(thismatches) + push!(applicable, MethodMatchTarget(thismatches[idx], thisinfo.edges, idx)) + push!(applicable_argtypes, arg_n) + end end info = UnionSplitInfo(infos) return UnionSplitMethodMatches( @@ -360,8 +352,9 @@ function find_simple_method_matches(interp::AbstractInterpreter, @nospecialize(a return FailedMethodMatch("Too many methods matched") end fullmatch = any(match::MethodMatch->match.fully_covers, matches) - info = MethodMatchInfo(matches, mt, fullmatch) - return MethodMatches(matches.matches, info, matches.valid_worlds) + info = MethodMatchInfo(matches, mt, atype, fullmatch) + applicable = MethodMatchTarget[MethodMatchTarget(matches[idx], info.edges, idx) for idx = 1:length(matches)] + return MethodMatches(applicable, info, matches.valid_worlds) end """ @@ -532,7 +525,7 @@ function conditional_argtype(𝕃ᡒ::AbstractLattice, @nospecialize(rt), @nospe end end -function collect_slot_refinements(𝕃ᡒ::AbstractLattice, applicable::Vector{Any}, +function collect_slot_refinements(𝕃ᡒ::AbstractLattice, applicable::Vector{MethodMatchTarget}, argtypes::Vector{Any}, fargs::Vector{Any}, sv::InferenceState) ⊏, βŠ” = strictpartialorder(𝕃ᡒ), join(𝕃ᡒ) slotrefinements = nothing @@ -546,7 +539,7 @@ function collect_slot_refinements(𝕃ᡒ::AbstractLattice, applicable::Vector{A end sigt = Bottom for j = 1:length(applicable) - match = applicable[j]::MethodMatch + (;match) = applicable[j] valid_as_lattice(match.spec_types, true) || continue sigt = sigt βŠ” fieldtype(match.spec_types, i) end @@ -561,31 +554,6 @@ function collect_slot_refinements(𝕃ᡒ::AbstractLattice, applicable::Vector{A return slotrefinements end -function add_call_backedges!(interp::AbstractInterpreter, @nospecialize(rettype), - all_effects::Effects, any_slot_refined::Bool, edges::Vector{MethodInstance}, - matches::Union{MethodMatches,UnionSplitMethodMatches}, @nospecialize(atype), - sv::AbsIntState) - # don't bother to add backedges when both type and effects information are already - # maximized to the top since a new method couldn't refine or widen them anyway - if rettype === Any - # ignore the `:nonoverlayed` property if `interp` doesn't use overlayed method table - # since it will never be tainted anyway - if !isoverlayed(method_table(interp)) - all_effects = Effects(all_effects; nonoverlayed=ALWAYS_FALSE) - end - if all_effects === Effects() && !any_slot_refined - return nothing - end - end - for edge in edges - add_backedge!(sv, edge) - end - # also need an edge to the method table in case something gets - # added that did not intersect with any existing method - add_uncovered_edges!(sv, matches, atype) - return nothing -end - const RECURSION_UNUSED_MSG = "Bounded recursion detected with unused result. Annotated return type may be wider than true result." const RECURSION_MSG = "Bounded recursion detected. Call was widened to force convergence." const RECURSION_MSG_HARDLIMIT = "Bounded recursion detected under hardlimit. Call was widened to force convergence." @@ -595,9 +563,9 @@ function abstract_call_method(interp::AbstractInterpreter, hardlimit::Bool, si::StmtInfo, sv::AbsIntState) sigtuple = unwrap_unionall(sig) sigtuple isa DataType || - return Future(MethodCallResult(Any, Any, false, false, nothing, Effects())) + return Future(MethodCallResult(Any, Any, Effects(), nothing, false, false)) all(@nospecialize(x) -> valid_as_lattice(unwrapva(x), true), sigtuple.parameters) || - return Future(MethodCallResult(Union{}, Any, false, false, nothing, EFFECTS_THROWS)) # catch bad type intersections early + return Future(MethodCallResult(Union{}, Any, EFFECTS_THROWS, nothing, false, false)) # catch bad type intersections early if is_nospecializeinfer(method) sig = get_nospecializeinfer_sig(method, sig, sparams) @@ -622,7 +590,7 @@ function abstract_call_method(interp::AbstractInterpreter, # we have a self-cycle in the call-graph, but not in the inference graph (typically): # break this edge now (before we record it) by returning early # (non-typically, this means that we lose the ability to detect a guaranteed StackOverflow in some cases) - return Future(MethodCallResult(Any, Any, true, true, nothing, Effects())) + return Future(MethodCallResult(Any, Any, Effects(), nothing, true, true)) end topmost = nothing edgecycle = true @@ -677,7 +645,7 @@ function abstract_call_method(interp::AbstractInterpreter, # since it's very unlikely that we'll try to inline this, # or want make an invoke edge to its calling convention return type. # (non-typically, this means that we lose the ability to detect a guaranteed StackOverflow in some cases) - return Future(MethodCallResult(Any, Any, true, true, nothing, Effects())) + return Future(MethodCallResult(Any, Any, Effects(), nothing, true, true)) end add_remark!(interp, sv, washardlimit ? RECURSION_MSG_HARDLIMIT : RECURSION_MSG) # TODO (#48913) implement a proper recursion handling for irinterp: @@ -803,9 +771,9 @@ function matches_sv(parent::AbsIntState, sv::AbsIntState) method_for_inference_limit_heuristics(sv) === method_for_inference_limit_heuristics(parent)) end -function is_edge_recursed(edge::MethodInstance, caller::AbsIntState) +function is_edge_recursed(edge::CodeInstance, caller::AbsIntState) return any(AbsIntStackUnwind(caller)) do sv::AbsIntState - return edge === frame_instance(sv) + return edge.def === frame_instance(sv) end end @@ -832,18 +800,15 @@ end struct MethodCallResult rt exct + effects::Effects + edge::Union{Nothing,CodeInstance} edgecycle::Bool edgelimited::Bool - edge::Union{Nothing,MethodInstance} - effects::Effects volatile_inf_result::Union{Nothing,VolatileInferenceResult} - function MethodCallResult(@nospecialize(rt), @nospecialize(exct), - edgecycle::Bool, - edgelimited::Bool, - edge::Union{Nothing,MethodInstance}, - effects::Effects, + function MethodCallResult(@nospecialize(rt), @nospecialize(exct), effects::Effects, + edge::Union{Nothing,CodeInstance}, edgecycle::Bool, edgelimited::Bool, volatile_inf_result::Union{Nothing,VolatileInferenceResult}=nothing) - return new(rt, exct, edgecycle, edgelimited, edge, effects, volatile_inf_result) + return new(rt, exct, effects, edge, edgecycle, edgelimited, volatile_inf_result) end end @@ -853,18 +818,17 @@ struct InvokeCall InvokeCall(@nospecialize(types), @nospecialize(lookupsig)) = new(types, lookupsig) end -struct ConstCallResults +struct ConstCallResult rt::Any exct::Any const_result::ConstResult effects::Effects - edge::MethodInstance - function ConstCallResults( + const_edge::Union{Nothing,CodeInstance} + function ConstCallResult( @nospecialize(rt), @nospecialize(exct), - const_result::ConstResult, - effects::Effects, - edge::MethodInstance) - return new(rt, exct, const_result, effects, edge) + const_result::ConstResult, effects::Effects, + const_edge::Union{Nothing,CodeInstance}) + return new(rt, exct, const_result, effects, const_edge) end end @@ -947,8 +911,7 @@ function concrete_eval_eligible(interp::AbstractInterpreter, return :none end end - mi = result.edge - if mi !== nothing && is_foldable(effects, #=check_rtcall=#true) + if result.edge !== nothing && is_foldable(effects, #=check_rtcall=#true) if f !== nothing && is_all_const_arg(arginfo, #=start=#2) if (is_nonoverlayed(interp) || is_nonoverlayed(effects) || # Even if overlay methods are involved, when `:consistent_overlay` is @@ -1010,15 +973,17 @@ function concrete_eval_call(interp::AbstractInterpreter, f = invoke end world = get_inference_world(interp) - edge = result.edge::MethodInstance + edge = result.edge::CodeInstance value = try Core._call_in_world_total(world, f, args...) catch e # The evaluation threw. By :consistent-cy, we're guaranteed this would have happened at runtime. # Howevever, at present, :consistency does not mandate the type of the exception - return ConstCallResults(Bottom, Any, ConcreteResult(edge, result.effects), result.effects, edge) + concrete_result = ConcreteResult(edge, result.effects) + return ConstCallResult(Bottom, Any, concrete_result, result.effects, #=const_edge=#nothing) end - return ConstCallResults(Const(value), Union{}, ConcreteResult(edge, EFFECTS_TOTAL, value), EFFECTS_TOTAL, edge) + concrete_result = ConcreteResult(edge, EFFECTS_TOTAL, value) + return ConstCallResult(Const(value), Bottom, concrete_result, EFFECTS_TOTAL, #=const_edge=#nothing) end # check if there is a cycle and duplicated inference of `mi` @@ -1262,9 +1227,9 @@ function semi_concrete_eval_call(interp::AbstractInterpreter, mi::MethodInstance, result::MethodCallResult, arginfo::ArgInfo, sv::AbsIntState) world = frame_world(sv) mi_cache = WorldView(code_cache(interp), world) - code = get(mi_cache, mi, nothing) - if code !== nothing - irsv = IRInterpretationState(interp, code, mi, arginfo.argtypes, world) + codeinst = get(mi_cache, mi, nothing) + if codeinst !== nothing + irsv = IRInterpretationState(interp, codeinst, mi, arginfo.argtypes, world) if irsv !== nothing assign_parentchild!(irsv, sv) rt, (nothrow, noub) = ir_abstract_constant_propagation(interp, irsv) @@ -1283,16 +1248,21 @@ function semi_concrete_eval_call(interp::AbstractInterpreter, effects = Effects(effects; noub=ALWAYS_TRUE) end exct = refine_exception_type(result.exct, effects) - return ConstCallResults(rt, exct, SemiConcreteResult(mi, ir, effects, spec_info(irsv)), effects, mi) + semi_concrete_result = SemiConcreteResult(codeinst, ir, effects, spec_info(irsv)) + const_edge = nothing # TODO use the edges from irsv? + return ConstCallResult(rt, exct, semi_concrete_result, effects, const_edge) end end end return nothing end -const_prop_result(inf_result::InferenceResult) = - ConstCallResults(inf_result.result, inf_result.exc_result, ConstPropResult(inf_result), - inf_result.ipo_effects, inf_result.linfo) +function const_prop_result(inf_result::InferenceResult) + @assert isdefined(inf_result, :ci_as_edge) "InferenceResult without ci_as_edge" + const_prop_result = ConstPropResult(inf_result) + return ConstCallResult(inf_result.result, inf_result.exc_result, const_prop_result, + inf_result.ipo_effects, inf_result.ci_as_edge) +end # return cached result of constant analysis return_localcache_result(::AbstractInterpreter, inf_result::InferenceResult, ::AbsIntState) = @@ -1305,7 +1275,7 @@ end function const_prop_call(interp::AbstractInterpreter, mi::MethodInstance, result::MethodCallResult, arginfo::ArgInfo, sv::AbsIntState, - concrete_eval_result::Union{Nothing, ConstCallResults}=nothing) + concrete_eval_result::Union{Nothing,ConstCallResult}=nothing) inf_cache = get_inference_cache(interp) 𝕃ᡒ = typeinf_lattice(interp) forwarded_argtypes = compute_forwarded_argtypes(interp, arginfo, sv) @@ -1353,6 +1323,7 @@ function const_prop_call(interp::AbstractInterpreter, pop!(callstack) return nothing end + inf_result.ci_as_edge = codeinst_as_edge(interp, frame) @assert frame.frameid != 0 && frame.cycleid == frame.frameid @assert frame.parentid == sv.frameid @assert inf_result.result !== nothing @@ -1691,7 +1662,7 @@ function abstract_iteration(interp::AbstractInterpreter, @nospecialize(itft), @n end iterateresult[] = AbstractIterationResult(ret, AbstractIterationInfo(calls, false)) return true - end # inferiterate_2arg + end # function inferiterate_2arg # continue making progress as much as possible, on iterate(arg, state) inferiterate_2arg(interp, sv) || push!(sv.tasks, inferiterate_2arg) return true @@ -1861,7 +1832,7 @@ function abstract_apply(interp::AbstractInterpreter, argtypes::Vector{Any}, si:: # For now, only propagate info if we don't also union-split the iteration applyresult[] = CallMeta(res, exctype, all_effects, retinfo) return true - end + end # function infercalls # start making progress on the first call infercalls(interp, sv) || push!(sv.tasks, infercalls) return applyresult @@ -2230,7 +2201,7 @@ function abstract_invoke(interp::AbstractInterpreter, arginfo::ArgInfo, si::Stmt mresult = abstract_call_method(interp, method, ti, env, false, si, sv)::Future match = MethodMatch(ti, env, method, argtype <: method.sig) return Future{CallMeta}(mresult, interp, sv) do result, interp, sv - (; rt, exct, edge, effects, volatile_inf_result) = result + (; rt, exct, effects, edge, volatile_inf_result) = result res = nothing sig = match.spec_types argtypesβ€² = invoke_rewrite(argtypes) @@ -2250,16 +2221,19 @@ function abstract_invoke(interp::AbstractInterpreter, arginfo::ArgInfo, si::Stmt result, f, arginfo, si, match, sv, invokecall) const_result = volatile_inf_result if const_call_result !== nothing + const_edge = nothing if const_call_result.rt βŠ‘ rt - (; rt, effects, const_result, edge) = const_call_result + (; rt, effects, const_result, const_edge) = const_call_result end if const_call_result.exct β‹€ exct - (; exct, const_result, edge) = const_call_result + (; exct, const_result, const_edge) = const_call_result + end + if const_edge !== nothing + edge = const_edge end end rt = from_interprocedural!(interp, rt, sv, arginfo, sig) - info = InvokeCallInfo(match, const_result) - edge !== nothing && add_invoke_backedge!(sv, lookupsig, edge) + info = InvokeCallInfo(edge, match, const_result, lookupsig) if !match.fully_covers effects = Effects(effects; nothrow=false) exct = exct βŠ” TypeError @@ -2328,7 +2302,8 @@ function abstract_call_known(interp::AbstractInterpreter, @nospecialize(f), return abstract_apply(interp, argtypes, si, sv, max_methods) elseif f === invoke return abstract_invoke(interp, arginfo, si, sv) - elseif f === modifyfield! || f === Core.modifyglobal! || f === Core.memoryrefmodify! || f === atomic_pointermodify + elseif f === modifyfield! || f === Core.modifyglobal! || + f === Core.memoryrefmodify! || f === atomic_pointermodify return abstract_modifyop!(interp, f, argtypes, si, sv) elseif f === Core.finalizer return abstract_finalizer(interp, argtypes, sv) @@ -2455,19 +2430,23 @@ function abstract_call_opaque_closure(interp::AbstractInterpreter, mresult = abstract_call_method(interp, ocmethod, sig, Core.svec(), false, si, sv) ocsig_box = Core.Box(ocsig) return Future{CallMeta}(mresult, interp, sv) do result, interp, sv - (; rt, exct, edge, effects, volatile_inf_result, edgecycle) = result + (; rt, exct, effects, volatile_inf_result, edge, edgecycle) = result π•ƒβ‚š = ipo_lattice(interp) βŠ‘, β‹€, βŠ” = partialorder(π•ƒβ‚š), strictneqpartialorder(π•ƒβ‚š), join(π•ƒβ‚š) const_result = volatile_inf_result if !edgecycle const_call_result = abstract_call_method_with_const_args(interp, result, - nothing, arginfo, si, match, sv) + #=f=#nothing, arginfo, si, match, sv) if const_call_result !== nothing + const_edge = nothing if const_call_result.rt βŠ‘ rt - (; rt, effects, const_result, edge) = const_call_result + (; rt, effects, const_result, const_edge) = const_call_result end if const_call_result.exct β‹€ exct - (; exct, const_result, edge) = const_call_result + (; exct, const_result, const_edge) = const_call_result + end + if const_edge !== nothing + edge = const_edge end end end @@ -2481,8 +2460,7 @@ function abstract_call_opaque_closure(interp::AbstractInterpreter, end end rt = from_interprocedural!(interp, rt, sv, arginfo, match.spec_types) - info = OpaqueClosureCallInfo(match, const_result) - edge !== nothing && add_backedge!(sv, edge) + info = OpaqueClosureCallInfo(edge, match, const_result) return CallMeta(rt, exct, effects, info) end end @@ -3432,7 +3410,6 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState, nextr while currpc < bbend currpc += 1 frame.currpc = currpc - empty_backedges!(frame, currpc) stmt = frame.src.code[currpc] # If we're at the end of the basic block ... if currpc == bbend diff --git a/base/compiler/inferencestate.jl b/base/compiler/inferencestate.jl index a200d5ced4d93..43ada89f23133 100644 --- a/base/compiler/inferencestate.jl +++ b/base/compiler/inferencestate.jl @@ -247,7 +247,7 @@ mutable struct InferenceState # TODO: Could keep this sparsely by doing structural liveness analysis ahead of time. bb_vartables::Vector{Union{Nothing,VarTable}} # nothing if not analyzed yet ssavaluetypes::Vector{Any} - stmt_edges::Vector{Vector{Any}} + edges::Vector{Any} stmt_info::Vector{CallInfo} #= intermediate states for interprocedural abstract interpretation =# @@ -302,7 +302,7 @@ mutable struct InferenceState nssavalues = src.ssavaluetypes::Int ssavalue_uses = find_ssavalue_uses(code, nssavalues) nstmts = length(code) - stmt_edges = Vector{Vector{Any}}(undef, nstmts) + edges = [] stmt_info = CallInfo[ NoCallInfo() for i = 1:nstmts ] nslots = length(src.slotflags) @@ -327,7 +327,7 @@ mutable struct InferenceState unreachable = BitSet() pclimitations = IdSet{InferenceState}() limitations = IdSet{InferenceState}() - cycle_backedges = Vector{Tuple{InferenceState,Int}}() + cycle_backedges = Tuple{InferenceState,Int}[] callstack = AbsIntState[] tasks = WorkThunk[] @@ -350,10 +350,12 @@ mutable struct InferenceState restrict_abstract_call_sites = isa(def, Module) + parentid = frameid = cycleid = 0 + this = new( mi, world, mod, sptypes, slottypes, src, cfg, spec_info, - currbb, currpc, ip, handler_info, ssavalue_uses, bb_vartables, ssavaluetypes, stmt_edges, stmt_info, - tasks, pclimitations, limitations, cycle_backedges, callstack, 0, 0, 0, + currbb, currpc, ip, handler_info, ssavalue_uses, bb_vartables, ssavaluetypes, edges, stmt_info, + tasks, pclimitations, limitations, cycle_backedges, callstack, parentid, frameid, cycleid, result, unreachable, valid_worlds, bestguess, exc_bestguess, ipo_effects, restrict_abstract_call_sites, cache_mode, insert_coverage, interp) @@ -754,30 +756,6 @@ function record_ssa_assign!(𝕃ᡒ::AbstractLattice, ssa_id::Int, @nospecialize return nothing end -function add_cycle_backedge!(caller::InferenceState, frame::InferenceState) - update_valid_age!(caller, frame.valid_worlds) - backedge = (caller, caller.currpc) - contains_is(frame.cycle_backedges, backedge) || push!(frame.cycle_backedges, backedge) - add_backedge!(caller, frame.linfo) - return frame -end - -function get_stmt_edges!(caller::InferenceState, currpc::Int=caller.currpc) - stmt_edges = caller.stmt_edges - if !isassigned(stmt_edges, currpc) - return stmt_edges[currpc] = Any[] - else - return stmt_edges[currpc] - end -end - -function empty_backedges!(frame::InferenceState, currpc::Int=frame.currpc) - if isassigned(frame.stmt_edges, currpc) - empty!(frame.stmt_edges[currpc]) - end - return nothing -end - function narguments(sv::InferenceState, include_va::Bool=true) nargs = Int(sv.src.nargs) if !include_va @@ -1008,32 +986,6 @@ function callers_in_cycle(sv::InferenceState) end callers_in_cycle(sv::IRInterpretationState) = AbsIntCycle(sv.callstack::Vector{AbsIntState}, 0, 0) -# temporarily accumulate our edges to later add as backedges in the callee -function add_backedge!(caller::InferenceState, mi::MethodInstance) - isa(caller.linfo.def, Method) || return nothing # don't add backedges to toplevel method instance - return push!(get_stmt_edges!(caller), mi) -end -function add_backedge!(irsv::IRInterpretationState, mi::MethodInstance) - return push!(irsv.edges, mi) -end - -function add_invoke_backedge!(caller::InferenceState, @nospecialize(invokesig::Type), mi::MethodInstance) - isa(caller.linfo.def, Method) || return nothing # don't add backedges to toplevel method instance - return push!(get_stmt_edges!(caller), invokesig, mi) -end -function add_invoke_backedge!(irsv::IRInterpretationState, @nospecialize(invokesig::Type), mi::MethodInstance) - return push!(irsv.edges, invokesig, mi) -end - -# used to temporarily accumulate our no method errors to later add as backedges in the callee method table -function add_mt_backedge!(caller::InferenceState, mt::MethodTable, @nospecialize(typ)) - isa(caller.linfo.def, Method) || return nothing # don't add backedges to toplevel method instance - return push!(get_stmt_edges!(caller), mt, typ) -end -function add_mt_backedge!(irsv::IRInterpretationState, mt::MethodTable, @nospecialize(typ)) - return push!(irsv.edges, mt, typ) -end - get_curr_ssaflag(sv::InferenceState) = sv.src.ssaflags[sv.currpc] get_curr_ssaflag(sv::IRInterpretationState) = sv.ir.stmts[sv.curridx][:flag] diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl index c5606f80468c0..e8508ade88b6c 100644 --- a/base/compiler/optimize.jl +++ b/base/compiler/optimize.jl @@ -141,8 +141,7 @@ struct InliningState{Interp<:AbstractInterpreter} interp::Interp end function InliningState(sv::InferenceState, interp::AbstractInterpreter) - edges = sv.stmt_edges[1] - return InliningState(edges, sv.world, interp) + return InliningState(sv.edges, sv.world, interp) end function InliningState(interp::AbstractInterpreter) return InliningState(Any[], get_inference_world(interp), interp) @@ -225,6 +224,7 @@ include("compiler/ssair/irinterp.jl") function ir_to_codeinf!(opt::OptimizationState) (; linfo, src) = opt src = ir_to_codeinf!(src, opt.ir::IRCode) + src.edges = opt.inlining.edges opt.ir = nothing maybe_validate_code(linfo, src, "optimized") return src diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl index dfdd317f74d87..ae4c04241fa13 100644 --- a/base/compiler/ssair/inlining.jl +++ b/base/compiler/ssair/inlining.jl @@ -28,7 +28,8 @@ end struct ConstantCase val::Any - ConstantCase(@nospecialize val) = new(val) + edge::CodeInstance + ConstantCase(@nospecialize(val), edge::CodeInstance) = new(val, edge) end struct SomeCase @@ -68,11 +69,12 @@ struct InliningEdgeTracker new(state.edges, invokesig) end -function add_inlining_backedge!((; edges, invokesig)::InliningEdgeTracker, mi::MethodInstance) +function add_inlining_edge!(et::InliningEdgeTracker, edge::Union{CodeInstance,MethodInstance}) + (; edges, invokesig) = et if invokesig === nothing - push!(edges, mi) + add_one_edge!(edges, edge) else # invoke backedge - push!(edges, invoke_signature(invokesig), mi) + add_invoke_edge!(edges, invoke_signature(invokesig), edge) end return nothing end @@ -784,10 +786,10 @@ function rewrite_apply_exprargs!(todo::Vector{Pair{Int,Any}}, end function compileable_specialization(mi::MethodInstance, effects::Effects, - et::InliningEdgeTracker, @nospecialize(info::CallInfo); compilesig_invokes::Bool=true) + et::InliningEdgeTracker, @nospecialize(info::CallInfo), state::InliningState) mi_invoke = mi method, atype, sparams = mi.def::Method, mi.specTypes, mi.sparam_vals - if compilesig_invokes + if OptimizationParams(state.interp).compilesig_invokes new_atype = get_compileable_sig(method, atype, sparams) new_atype === nothing && return nothing if atype !== new_atype @@ -805,43 +807,41 @@ function compileable_specialization(mi::MethodInstance, effects::Effects, return nothing end end - add_inlining_backedge!(et, mi) # to the dispatch lookup - mi_invoke !== mi && push!(et.edges, method.sig, mi_invoke) # add_inlining_backedge to the invoke call, if that is different + add_inlining_edge!(et, mi) # to the dispatch lookup + if mi_invoke !== mi + add_invoke_edge!(et.edges, method.sig, mi_invoke) # add_inlining_edge to the invoke call, if that is different + end return InvokeCase(mi_invoke, effects, info) end -function compileable_specialization(match::MethodMatch, effects::Effects, - et::InliningEdgeTracker, @nospecialize(info::CallInfo); compilesig_invokes::Bool=true) - mi = specialize_method(match) - return compileable_specialization(mi, effects, et, info; compilesig_invokes) -end - struct InferredResult src::Any # CodeInfo or IRCode effects::Effects - InferredResult(@nospecialize(src), effects::Effects) = new(src, effects) + edge::CodeInstance + InferredResult(@nospecialize(src), effects::Effects, edge::CodeInstance) = new(src, effects, edge) end @inline function get_cached_result(state::InliningState, mi::MethodInstance) code = get(code_cache(state), mi, nothing) if code isa CodeInstance if use_const_api(code) # in this case function can be inlined to a constant - return ConstantCase(quoted(code.rettype_const)) + return ConstantCase(quoted(code.rettype_const), code) end return code end return nothing end @inline function get_local_result(inf_result::InferenceResult) + @assert isdefined(inf_result, :ci_as_edge) "InferenceResult without ci_as_edge" effects = inf_result.ipo_effects if is_foldable_nothrow(effects) res = inf_result.result if isa(res, Const) && is_inlineable_constant(res.val) # use constant calling convention - return ConstantCase(quoted(res.val)) + return ConstantCase(quoted(res.val), inf_result.ci_as_edge) end end - return InferredResult(inf_result.src, effects) + return InferredResult(inf_result.src, effects, inf_result.ci_as_edge) end # the general resolver for usual and const-prop'ed calls @@ -861,30 +861,28 @@ function resolve_todo(mi::MethodInstance, result::Union{Nothing,InferenceResult, inferred_result = get_cached_result(state, mi) end if inferred_result isa ConstantCase - add_inlining_backedge!(et, mi) + add_inlining_edge!(et, inferred_result.edge) return inferred_result elseif inferred_result isa InferredResult - (; src, effects) = inferred_result + (; src, effects, edge) = inferred_result elseif inferred_result isa CodeInstance src = @atomic :monotonic inferred_result.inferred effects = decode_effects(inferred_result.ipo_purity_bits) + edge = inferred_result else # there is no cached source available, bail out - return compileable_specialization(mi, Effects(), et, info; - compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes) + return compileable_specialization(mi, Effects(), et, info, state) end # the duplicated check might have been done already within `analyze_method!`, but still # we need it here too since we may come here directly using a constant-prop' result if !OptimizationParams(state.interp).inlining || is_stmt_noinline(flag) - return compileable_specialization(mi, effects, et, info; - compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes) + return compileable_specialization(edge.def, effects, et, info, state) end src_inlining_policy(state.interp, src, info, flag) || - return compileable_specialization(mi, effects, et, info; - compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes) + return compileable_specialization(edge.def, effects, et, info, state) - add_inlining_backedge!(et, mi) + add_inlining_edge!(et, edge) if inferred_result isa CodeInstance ir, spec_info, debuginfo = retrieve_ir_for_inlining(inferred_result, src) else @@ -904,7 +902,7 @@ function resolve_todo(mi::MethodInstance, @nospecialize(info::CallInfo), flag::U cached_result = get_cached_result(state, mi) if cached_result isa ConstantCase - add_inlining_backedge!(et, mi) + add_inlining_edge!(et, cached_result.edge) return cached_result elseif cached_result isa CodeInstance src = @atomic :monotonic cached_result.inferred @@ -915,7 +913,7 @@ function resolve_todo(mi::MethodInstance, @nospecialize(info::CallInfo), flag::U src_inlining_policy(state.interp, src, info, flag) || return nothing ir, spec_info, debuginfo = retrieve_ir_for_inlining(cached_result, src) - add_inlining_backedge!(et, mi) + add_inlining_edge!(et, cached_result) return InliningTodo(mi, ir, spec_info, debuginfo, effects) end @@ -1119,7 +1117,7 @@ function inline_apply!(todo::Vector{Pair{Int,Any}}, # e.g. rewrite `((t::Tuple)...,)` to `t` nonempty_idx = 0 𝕃ₒ = optimizer_lattice(state.interp) - for i = (arg_start + 1):length(argtypes) + for i = (arg_start+1):length(argtypes) ti = argtypes[i] βŠ‘(𝕃ₒ, ti, Tuple{}) && continue if βŠ‘(𝕃ₒ, ti, Tuple) && nonempty_idx == 0 @@ -1137,7 +1135,7 @@ function inline_apply!(todo::Vector{Pair{Int,Any}}, # Try to figure out the signature of the function being called # and if rewrite_apply_exprargs can deal with this form arginfos = MaybeAbstractIterationInfo[] - for i = (arg_start + 1):length(argtypes) + for i = (arg_start+1):length(argtypes) thisarginfo = nothing if !is_valid_type_for_apply_rewrite(argtypes[i], OptimizationParams(state.interp)) isa(info, ApplyCallInfo) || return nothing @@ -1403,9 +1401,7 @@ function compute_inlining_cases(@nospecialize(info::CallInfo), flag::UInt32, sig result, match, argtypes, info, flag, state; allow_typevars=true) end if !fully_covered - atype = argtypes_to_type(sig.argtypes) - # We will emit an inline MethodError so we need a backedge to the MethodTable - add_uncovered_edges!(state.edges, info, atype) + # We will emit an inline MethodError in this case, but that info already came inference, so we must already have the uncovered edge for it end elseif !isempty(cases) # if we've not seen all candidates, union split is valid only for dispatch tuples @@ -1453,7 +1449,7 @@ end function semiconcrete_result_item(result::SemiConcreteResult, @nospecialize(info::CallInfo), flag::UInt32, state::InliningState) - mi = result.mi + mi = result.edge.def et = InliningEdgeTracker(state) if (!OptimizationParams(state.interp).inlining || is_stmt_noinline(flag) || @@ -1461,14 +1457,12 @@ function semiconcrete_result_item(result::SemiConcreteResult, # a `@noinline`-declared method when it's marked as `@constprop :aggressive`. # Suppress the inlining here (unless inlining is requested at the callsite). (is_declared_noinline(mi.def::Method) && !is_stmt_inline(flag))) - return compileable_specialization(mi, result.effects, et, info; - compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes) + return compileable_specialization(mi, result.effects, et, info, state) end src_inlining_policy(state.interp, result.ir, info, flag) || - return compileable_specialization(mi, result.effects, et, info; - compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes) + return compileable_specialization(mi, result.effects, et, info, state) - add_inlining_backedge!(et, mi) + add_inlining_edge!(et, result.edge) preserve_local_sources = OptimizationParams(state.interp).preserve_local_sources ir, _, debuginfo = retrieve_ir_for_inlining(mi, result.ir, preserve_local_sources) return InliningTodo(mi, ir, result.spec_info, debuginfo, result.effects) @@ -1476,7 +1470,7 @@ end function handle_semi_concrete_result!(cases::Vector{InliningCase}, result::SemiConcreteResult, match::MethodMatch, @nospecialize(info::CallInfo), flag::UInt32, state::InliningState) - mi = result.mi + mi = result.edge.def validate_sparams(mi.sparam_vals) || return false item = semiconcrete_result_item(result, info, flag, state) item === nothing && return false @@ -1499,11 +1493,10 @@ function concrete_result_item(result::ConcreteResult, @nospecialize(info::CallIn invokesig::Union{Nothing,Vector{Any}}=nothing) if !may_inline_concrete_result(result) et = InliningEdgeTracker(state, invokesig) - return compileable_specialization(result.mi, result.effects, et, info; - compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes) + return compileable_specialization(result.edge.def, result.effects, et, info, state) end @assert result.effects === EFFECTS_TOTAL - return ConstantCase(quoted(result.result)) + return ConstantCase(quoted(result.result), result.edge) end function handle_cases!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, stmt::Expr, @@ -1552,11 +1545,16 @@ function handle_modifyop!_call!(ir::IRCode, idx::Int, stmt::Expr, info::ModifyOp info isa MethodResultPure && (info = info.info) info isa ConstCallInfo && (info = info.call) info isa MethodMatchInfo || return nothing - length(info.results) == 1 || return nothing + length(info.edges) == length(info.results) == 1 || return nothing match = info.results[1]::MethodMatch match.fully_covers || return nothing - case = compileable_specialization(match, Effects(), InliningEdgeTracker(state), info; - compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes) + edge = info.edges[1] + if edge === nothing + edge = specialize_method(match) + else + edge = edge.def + end + case = compileable_specialization(edge, Effects(), InliningEdgeTracker(state), info, state) case === nothing && return nothing stmt.head = :invoke_modify pushfirst!(stmt.args, case.invoke) diff --git a/base/compiler/ssair/irinterp.jl b/base/compiler/ssair/irinterp.jl index f9565f3971733..0a8239dc590db 100644 --- a/base/compiler/ssair/irinterp.jl +++ b/base/compiler/ssair/irinterp.jl @@ -450,12 +450,6 @@ function ir_abstract_constant_propagation(interp::AbstractInterpreter, irsv::IRI (nothrow | noub) || break end - if last(irsv.valid_worlds) >= get_world_counter() - # if we aren't cached, we don't need this edge - # but our caller might, so let's just make it anyways - store_backedges(frame_instance(irsv), irsv.edges) - end - if irsv.frameid != 0 callstack = irsv.callstack::Vector{AbsIntState} @assert callstack[end] === irsv && length(callstack) == irsv.frameid diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 4ad5dcfb2a3c8..bfe33c23871fe 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1529,7 +1529,7 @@ function try_inline_finalizer!(ir::IRCode, argexprs::Vector{Any}, idx::Int, if code isa CodeInstance if use_const_api(code) # No code in the function - Nothing to do - add_inlining_backedge!(et, mi) + add_inlining_edge!(et, code) return true end src = @atomic :monotonic code.inferred @@ -1544,7 +1544,7 @@ function try_inline_finalizer!(ir::IRCode, argexprs::Vector{Any}, idx::Int, length(src.cfg.blocks) == 1 || return false # Ok, we're committed to inlining the finalizer - add_inlining_backedge!(et, mi) + add_inlining_edge!(et, code) # TODO: Should there be a special line number node for inlined finalizers? inline_at = ir[SSAValue(idx)][:line] diff --git a/base/compiler/stmtinfo.jl b/base/compiler/stmtinfo.jl index 9dba7a4459f9e..4cbd2ab39fd46 100644 --- a/base/compiler/stmtinfo.jl +++ b/base/compiler/stmtinfo.jl @@ -22,27 +22,114 @@ struct CallMeta end struct NoCallInfo <: CallInfo end +add_edges_impl(::Vector{Any}, ::NoCallInfo) = nothing """ info::MethodMatchInfo <: CallInfo -Captures the result of a `:jl_matching_methods` lookup for the given call (`info.results`). -This info may then be used by the optimizer to inline the matches, without having -to re-consult the method table. This info is illegal on any statement that is -not a call to a generic function. +Captures the essential arguments and result of a `:jl_matching_methods` lookup +for the given call (`info.results`). This info may then be used by the +optimizer, without having to re-consult the method table. +This info is illegal on any statement that is not a call to a generic function. """ struct MethodMatchInfo <: CallInfo results::MethodLookupResult mt::MethodTable + atype fullmatch::Bool + edges::Vector{Union{Nothing,CodeInstance}} + function MethodMatchInfo( + results::MethodLookupResult, mt::MethodTable, @nospecialize(atype), fullmatch::Bool) + edges = fill!(Vector{Union{Nothing,CodeInstance}}(undef, length(results)), nothing) + return new(results, mt, atype, fullmatch, edges) + end +end +add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo) = _add_edges_impl(edges, info) +function _add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo, mi_edge::Bool=false) + if !fully_covering(info) + # add legacy-style missing backedge info also + exists = false + for i in 1:length(edges) + if edges[i] === info.mt && edges[i+1] == info.atype + exists = true + break + end + end + if !exists + push!(edges, info.mt, info.atype) + end + end + nmatches = length(info.results) + if nmatches == length(info.edges) == 1 + # try the optimized format for the representation, if possible and applicable + # if this doesn't succeed, the backedge will be less precise, + # but the forward edge will maintain the precision + edge = info.edges[1] + m = info.results[1] + if edge === nothing + mi = specialize_method(m) # don't allow `Method`-edge for this optimized format + edge = mi + else + mi = edge.def + end + if mi.specTypes === m.spec_types + add_one_edge!(edges, edge) + return nothing + end + end + # add check for whether this lookup already existed in the edges list + for i in 1:length(edges) + if edges[i] === nmatches && edges[i+1] == info.atype + # TODO: must also verify the CodeInstance match too + return nothing + end + end + push!(edges, nmatches, info.atype) + for i = 1:nmatches + edge = info.edges[i] + m = info.results[i] + if edge === nothing + edge = mi_edge ? specialize_method(m) : m.method + else + @assert edge.def.def === m.method + end + push!(edges, edge) + end + nothing +end +function add_one_edge!(edges::Vector{Any}, edge::MethodInstance) + for i in 1:length(edges) + edgeα΅’ = edges[i] + edgeα΅’ isa CodeInstance && (edgeα΅’ = edgeα΅’.def) + edgeα΅’ isa MethodInstance || continue + if edgeα΅’ === edge && !(i > 1 && edges[i-1] isa Type) + return # found existing covered edge + end + end + push!(edges, edge) + nothing +end +function add_one_edge!(edges::Vector{Any}, edge::CodeInstance) + for i in 1:length(edges) + edgeα΅’_orig = edgeα΅’ = edges[i] + edgeα΅’ isa CodeInstance && (edgeα΅’ = edgeα΅’.def) + edgeα΅’ isa MethodInstance || continue + if edgeα΅’ === edge.def && !(i > 1 && edges[i-1] isa Type) + if edgeα΅’_orig isa MethodInstance + # found edge we can upgrade + edges[i] = edge + return + elseif true # XXX compare `CodeInstance` identify? + return + end + end + end + push!(edges, edge) + nothing end nsplit_impl(info::MethodMatchInfo) = 1 getsplit_impl(info::MethodMatchInfo, idx::Int) = (@assert idx == 1; info.results) getresult_impl(::MethodMatchInfo, ::Int) = nothing -function add_uncovered_edges_impl(edges::Vector{Any}, info::MethodMatchInfo, @nospecialize(atype)) - fully_covering(info) || push!(edges, info.mt, atype) - nothing -end """ info::UnionSplitInfo <: CallInfo @@ -56,25 +143,13 @@ This info is illegal on any statement that is not a call to a generic function. struct UnionSplitInfo <: CallInfo split::Vector{MethodMatchInfo} end - -nmatches(info::MethodMatchInfo) = length(info.results) -function nmatches(info::UnionSplitInfo) - n = 0 - for mminfo in info.split - n += nmatches(mminfo) - end - return n -end +add_edges_impl(edges::Vector{Any}, info::UnionSplitInfo) = + _add_edges_impl(edges, info) +_add_edges_impl(edges::Vector{Any}, info::UnionSplitInfo, mi_edge::Bool=false) = + for split in info.split; _add_edges_impl(edges, split, mi_edge); end nsplit_impl(info::UnionSplitInfo) = length(info.split) getsplit_impl(info::UnionSplitInfo, idx::Int) = getsplit(info.split[idx], 1) getresult_impl(::UnionSplitInfo, ::Int) = nothing -function add_uncovered_edges_impl(edges::Vector{Any}, info::UnionSplitInfo, @nospecialize(atype)) - all(fully_covering, info.split) && return nothing - # add mt backedges with removing duplications - for mt in uncovered_method_tables(info) - push!(edges, mt, atype) - end -end abstract type ConstResult end @@ -83,15 +158,15 @@ struct ConstPropResult <: ConstResult end struct ConcreteResult <: ConstResult - mi::MethodInstance + edge::CodeInstance effects::Effects result - ConcreteResult(mi::MethodInstance, effects::Effects) = new(mi, effects) - ConcreteResult(mi::MethodInstance, effects::Effects, @nospecialize val) = new(mi, effects, val) + ConcreteResult(edge::CodeInstance, effects::Effects) = new(edge, effects) + ConcreteResult(edge::CodeInstance, effects::Effects, @nospecialize val) = new(edge, effects, val) end struct SemiConcreteResult <: ConstResult - mi::MethodInstance + edge::CodeInstance ir::IRCode effects::Effects spec_info::SpecInfo @@ -116,17 +191,15 @@ struct ConstCallInfo <: CallInfo call::Union{MethodMatchInfo,UnionSplitInfo} results::Vector{Union{Nothing,ConstResult}} end +add_edges_impl(edges::Vector{Any}, info::ConstCallInfo) = add_edges!(edges, info.call) nsplit_impl(info::ConstCallInfo) = nsplit(info.call) getsplit_impl(info::ConstCallInfo, idx::Int) = getsplit(info.call, idx) getresult_impl(info::ConstCallInfo, idx::Int) = info.results[idx] -add_uncovered_edges_impl(edges::Vector{Any}, info::ConstCallInfo, @nospecialize(atype)) = add_uncovered_edges!(edges, info.call, atype) """ info::MethodResultPure <: CallInfo -This struct represents a method result constant was proven to be -effect-free, including being no-throw (typically because the value was computed -by calling an `@pure` function). +This struct represents a method result constant was proven to be effect-free. """ struct MethodResultPure <: CallInfo info::CallInfo @@ -135,6 +208,7 @@ let instance = MethodResultPure(NoCallInfo()) global MethodResultPure MethodResultPure() = instance end +add_edges_impl(edges::Vector{Any}, info::MethodResultPure) = add_edges!(edges, info.info) """ ainfo::AbstractIterationInfo @@ -161,10 +235,19 @@ not an `_apply_iterate` call. """ struct ApplyCallInfo <: CallInfo # The info for the call itself - call::Any + call::CallInfo # AbstractIterationInfo for each argument, if applicable arginfo::Vector{MaybeAbstractIterationInfo} end +function add_edges_impl(edges::Vector{Any}, info::ApplyCallInfo) + add_edges!(edges, info.call) + for arg in info.arginfo + arg === nothing && continue + for edge in arg.each + add_edges!(edges, edge.info) + end + end +end """ info::UnionSplitApplyCallInfo <: CallInfo @@ -175,6 +258,8 @@ This info is illegal on any statement that is not an `_apply_iterate` call. struct UnionSplitApplyCallInfo <: CallInfo infos::Vector{ApplyCallInfo} end +add_edges_impl(edges::Vector{Any}, info::UnionSplitApplyCallInfo) = + for split in info.infos; add_edges!(edges, split); end """ info::InvokeCallInfo @@ -184,8 +269,56 @@ the method that has been processed. Optionally keeps `info.result::InferenceResult` that keeps constant information. """ struct InvokeCallInfo <: CallInfo + edge::Union{Nothing,CodeInstance} match::MethodMatch result::Union{Nothing,ConstResult} + atype # ::Type +end +add_edges_impl(edges::Vector{Any}, info::InvokeCallInfo) = + _add_edges_impl(edges, info) +function _add_edges_impl(edges::Vector{Any}, info::InvokeCallInfo, mi_edge::Bool=false) + edge = info.edge + if edge === nothing + edge = mi_edge ? specialize_method(info.match) : info.match.method + end + add_invoke_edge!(edges, info.atype, edge) + nothing +end +function add_invoke_edge!(edges::Vector{Any}, @nospecialize(atype), edge::Union{MethodInstance,Method}) + for i in 2:length(edges) + edgeα΅’ = edges[i] + edgeα΅’ isa CodeInstance && (edgeα΅’ = edgeα΅’.def) + edgeα΅’ isa MethodInstance || edgeα΅’ isa Method || continue + if edgeα΅’ === edge + edge_minus_1 = edges[i-1] + if edge_minus_1 isa Type && edge_minus_1 == atype + return # found existing covered edge + end + end + end + push!(edges, atype, edge) + nothing +end +function add_invoke_edge!(edges::Vector{Any}, @nospecialize(atype), edge::CodeInstance) + for i in 2:length(edges) + edgeα΅’_orig = edgeα΅’ = edges[i] + edgeα΅’ isa CodeInstance && (edgeα΅’ = edgeα΅’.def) + if ((edgeα΅’ isa MethodInstance && edgeα΅’ === edge.def) || + (edgeα΅’ isa Method && edgeα΅’ === edge.def.def)) + edge_minus_1 = edges[i-1] + if edge_minus_1 isa Type && edge_minus_1 == atype + if edgeα΅’_orig isa MethodInstance || edgeα΅’_orig isa Method + # found edge we can upgrade + edges[i] = edge + return + elseif true # XXX compare `CodeInstance` identify? + return + end + end + end + end + push!(edges, atype, edge) + nothing end """ @@ -196,9 +329,17 @@ the method that has been processed. Optionally keeps `info.result::InferenceResult` that keeps constant information. """ struct OpaqueClosureCallInfo <: CallInfo + edge::Union{Nothing,CodeInstance} match::MethodMatch result::Union{Nothing,ConstResult} end +function add_edges_impl(edges::Vector{Any}, info::OpaqueClosureCallInfo) + edge = info.edge + if edge !== nothing + add_one_edge!(edges, edge) + end + nothing +end """ info::OpaqueClosureCreateInfo <: CallInfo @@ -215,6 +356,9 @@ struct OpaqueClosureCreateInfo <: CallInfo return new(unspec) end end +# merely creating the object implies edges for OC, unlike normal objects, +# since calling them doesn't normally have edges in contrast +add_edges_impl(edges::Vector{Any}, info::OpaqueClosureCreateInfo) = add_edges!(edges, info.unspec.info) # Stmt infos that are used by external consumers, but not by optimization. # These are not produced by default and must be explicitly opted into by @@ -230,6 +374,7 @@ was supposed to analyze. struct ReturnTypeCallInfo <: CallInfo info::CallInfo end +add_edges_impl(edges::Vector{Any}, info::ReturnTypeCallInfo) = add_edges!(edges, info.info) """ info::FinalizerInfo <: CallInfo @@ -241,6 +386,8 @@ struct FinalizerInfo <: CallInfo info::CallInfo # the callinfo for the finalizer call effects::Effects # the effects for the finalizer call end +# merely allocating a finalizer does not imply edges (unless it gets inlined later) +add_edges_impl(::Vector{Any}, ::FinalizerInfo) = nothing """ info::ModifyOpInfo <: CallInfo @@ -256,5 +403,12 @@ Represents a resolved call of one of: struct ModifyOpInfo <: CallInfo info::CallInfo # the callinfo for the `op(getval(), x)` call end +add_edges_impl(edges::Vector{Any}, info::ModifyOpInfo) = add_edges!(edges, info.info) + +struct VirtualMethodMatchInfo <: CallInfo + info::Union{MethodMatchInfo,UnionSplitInfo,InvokeCallInfo} +end +add_edges_impl(edges::Vector{Any}, info::VirtualMethodMatchInfo) = + _add_edges_impl(edges, info.info, #=mi_edge=#true) @specialize diff --git a/base/compiler/tfuncs.jl b/base/compiler/tfuncs.jl index 2f78348b79844..80e252dde3a02 100644 --- a/base/compiler/tfuncs.jl +++ b/base/compiler/tfuncs.jl @@ -1432,7 +1432,7 @@ end return Future{CallMeta}(callinfo, interp, sv) do callinfo, interp, sv TF = TF.contents RT = RT.contents - TF2 = tmeet(callinfo.rt, widenconst(TF)) + TF2 = tmeet(ipo_lattice(interp), callinfo.rt, widenconst(TF)) if TF2 === Bottom RT = Bottom elseif isconcretetype(RT) && has_nontrivial_extended_info(𝕃ᡒ, TF2) # isconcrete condition required to form a PartialStruct @@ -2959,7 +2959,7 @@ function return_type_tfunc(interp::AbstractInterpreter, argtypes::Vector{Any}, s if isa(sv, InferenceState) sv.restrict_abstract_call_sites = old_restrict end - info = verbose_stmt_info(interp) ? MethodResultPure(ReturnTypeCallInfo(call.info)) : MethodResultPure() + info = MethodResultPure(ReturnTypeCallInfo(call.info)) rt = widenslotwrapper(call.rt) if isa(rt, Const) # output was computed to be constant @@ -2989,11 +2989,12 @@ end # a simplified model of abstract_call_gf_by_type for applicable function abstract_applicable(interp::AbstractInterpreter, argtypes::Vector{Any}, sv::AbsIntState, max_methods::Int) - length(argtypes) < 2 && return Future(CallMeta(Bottom, Any, EFFECTS_THROWS, NoCallInfo())) - isvarargtype(argtypes[2]) && return Future(CallMeta(Bool, Any, EFFECTS_THROWS, NoCallInfo())) + length(argtypes) < 2 && return Future(CallMeta(Bottom, ArgumentError, EFFECTS_THROWS, NoCallInfo())) + isvarargtype(argtypes[2]) && return Future(CallMeta(Bool, ArgumentError, EFFECTS_THROWS, NoCallInfo())) argtypes = argtypes[2:end] atype = argtypes_to_type(argtypes) matches = find_method_matches(interp, argtypes, atype; max_methods) + info = NoCallInfo() if isa(matches, FailedMethodMatch) rt = Bool # too many matches to analyze else @@ -3009,17 +3010,10 @@ function abstract_applicable(interp::AbstractInterpreter, argtypes::Vector{Any}, rt = Const(true) # has applicable matches end if rt !== Bool - for i in 1:napplicable - match = applicable[i]::MethodMatch - edge = specialize_method(match) - add_backedge!(sv, edge) - end - # also need an edge to the method table in case something gets - # added that did not intersect with any existing method - add_uncovered_edges!(sv, matches, atype) + info = VirtualMethodMatchInfo(matches.info) end end - return Future(CallMeta(rt, Union{}, EFFECTS_TOTAL, NoCallInfo())) + return Future(CallMeta(rt, Union{}, EFFECTS_TOTAL, info)) end add_tfunc(applicable, 1, INT_INF, @nospecs((𝕃::AbstractLattice, f, args...)->Bool), 40) @@ -3053,13 +3047,14 @@ function _hasmethod_tfunc(interp::AbstractInterpreter, argtypes::Vector{Any}, sv update_valid_age!(sv, valid_worlds) if match === nothing rt = Const(false) - add_mt_backedge!(sv, mt, types) # this should actually be an invoke-type backedge + vresults = MethodLookupResult(Any[], valid_worlds, true) + vinfo = MethodMatchInfo(vresults, mt, types, false) # XXX: this should actually be an info with invoke-type edge else rt = Const(true) - edge = specialize_method(match)::MethodInstance - add_invoke_backedge!(sv, types, edge) + vinfo = InvokeCallInfo(nothing, match, nothing, types) end - return CallMeta(rt, Any, EFFECTS_TOTAL, NoCallInfo()) + info = VirtualMethodMatchInfo(vinfo) + return CallMeta(rt, Union{}, EFFECTS_TOTAL, info) end # N.B.: typename maps type equivalence classes to a single value diff --git a/base/compiler/typeinfer.jl b/base/compiler/typeinfer.jl index 2a3bbf3854302..11337d5a4d047 100644 --- a/base/compiler/typeinfer.jl +++ b/base/compiler/typeinfer.jl @@ -95,26 +95,34 @@ const __measure_typeinf__ = RefValue{Bool}(false) function finish!(interp::AbstractInterpreter, caller::InferenceState; can_discard_trees::Bool=may_discard_trees(interp)) result = caller.result - valid_worlds = result.valid_worlds - if last(valid_worlds) >= get_world_counter() - # if we aren't cached, we don't need this edge - # but our caller might, so let's just make it anyways - store_backedges(result, caller.stmt_edges[1]) - end opt = result.src if opt isa OptimizationState - result.src = opt = ir_to_codeinf!(opt) + result.src = ir_to_codeinf!(opt) end + #@assert last(result.valid_worlds) <= get_world_counter() || isempty(caller.edges) if isdefined(result, :ci) ci = result.ci + # if we aren't cached, we don't need this edge + # but our caller might, so let's just make it anyways + if last(result.valid_worlds) >= get_world_counter() + # TODO: this should probably come after all store_backedges (after optimizations) for the entire graph in finish_cycle + # since we should be requiring that all edges first get their backedges set, as a batch + result.valid_worlds = WorldRange(first(result.valid_worlds), typemax(UInt)) + end + if last(result.valid_worlds) == typemax(UInt) + # if we can record all of the backedges in the global reverse-cache, + # we can now widen our applicability in the global cache too + store_backedges(ci, caller.edges) + end inferred_result = nothing relocatability = 0x1 const_flag = is_result_constabi_eligible(result) if !can_discard_trees || (is_cached(caller) && !const_flag) - inferred_result = transform_result_for_cache(interp, result.linfo, result.valid_worlds, result, can_discard_trees) + inferred_result = transform_result_for_cache(interp, result) + # TODO: do we want to augment edges here with any :invoke targets that we got from inlining (such that we didn't have a direct edge to it already)? relocatability = 0x0 if inferred_result isa CodeInfo - edges = inferred_result.debuginfo + di = inferred_result.debuginfo uncompressed = inferred_result inferred_result = maybe_compress_codeinfo(interp, result.linfo, inferred_result, can_discard_trees) result.is_src_volatile |= uncompressed !== inferred_result @@ -129,14 +137,12 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState; end end # n.b. relocatability = isa(inferred_result, String) && inferred_result[end] - if !@isdefined edges - edges = DebugInfo(result.linfo) + if !@isdefined di + di = DebugInfo(result.linfo) end - ccall(:jl_update_codeinst, Cvoid, (Any, Any, Int32, UInt, UInt, UInt32, Any, UInt8, Any), - ci, inferred_result, const_flag, - first(result.valid_worlds), last(result.valid_worlds), - encode_effects(result.ipo_effects), result.analysis_results, - relocatability, edges) + ccall(:jl_update_codeinst, Cvoid, (Any, Any, Int32, UInt, UInt, UInt32, Any, UInt8, Any, Any), + ci, inferred_result, const_flag, first(result.valid_worlds), last(result.valid_worlds), encode_effects(result.ipo_effects), + result.analysis_results, relocatability, di, Core.svec(caller.edges...)) engine_reject(interp, ci) end return nothing @@ -160,8 +166,8 @@ end function finish_cycle(::AbstractInterpreter, frames::Vector{AbsIntState}, cycleid::Int) cycle_valid_worlds = WorldRange() cycle_valid_effects = EFFECTS_TOTAL - for caller in cycleid:length(frames) - caller = frames[caller]::InferenceState + for frameid = cycleid:length(frames) + caller = frames[frameid]::InferenceState @assert caller.cycleid == cycleid # converge the world age range and effects for this cycle here: # all frames in the cycle should have the same bits of `valid_worlds` and `effects` @@ -170,20 +176,20 @@ function finish_cycle(::AbstractInterpreter, frames::Vector{AbsIntState}, cyclei cycle_valid_worlds = intersect(cycle_valid_worlds, caller.valid_worlds) cycle_valid_effects = merge_effects(cycle_valid_effects, caller.ipo_effects) end - for caller in cycleid:length(frames) - caller = frames[caller]::InferenceState + for frameid = cycleid:length(frames) + caller = frames[frameid]::InferenceState adjust_cycle_frame!(caller, cycle_valid_worlds, cycle_valid_effects) finishinfer!(caller, caller.interp) end - for caller in cycleid:length(frames) - caller = frames[caller]::InferenceState + for frameid = cycleid:length(frames) + caller = frames[frameid]::InferenceState opt = caller.result.src if opt isa OptimizationState # implies `may_optimize(caller.interp) === true` optimize(caller.interp, opt, caller.result) end end - for caller in cycleid:length(frames) - caller = frames[caller]::InferenceState + for frameid = cycleid:length(frames) + caller = frames[frameid]::InferenceState finish!(caller.interp, caller) end resize!(frames, cycleid - 1) @@ -210,12 +216,7 @@ function is_result_constabi_eligible(result::InferenceResult) return isa(result_type, Const) && is_foldable_nothrow(result.ipo_effects) && is_inlineable_constant(result_type.val) end - -function transform_result_for_cache(interp::AbstractInterpreter, - ::MethodInstance, valid_worlds::WorldRange, result::InferenceResult, - can_discard_trees::Bool=may_discard_trees(interp)) - return result.src -end +transform_result_for_cache(::AbstractInterpreter, result::InferenceResult) = result.src function maybe_compress_codeinfo(interp::AbstractInterpreter, mi::MethodInstance, ci::CodeInfo, can_discard_trees::Bool=may_discard_trees(interp)) @@ -239,29 +240,20 @@ function maybe_compress_codeinfo(interp::AbstractInterpreter, mi::MethodInstance end end -function cache_result!(interp::AbstractInterpreter, result::InferenceResult) - if last(result.valid_worlds) == get_world_counter() - # if we've successfully recorded all of the backedges in the global reverse-cache, - # we can now widen our applicability in the global cache too - result.valid_worlds = WorldRange(first(result.valid_worlds), typemax(UInt)) - end - @assert isdefined(result.ci, :inferred) +function cache_result!(interp::AbstractInterpreter, result::InferenceResult, ci::CodeInstance) + @assert isdefined(ci, :inferred) # check if the existing linfo metadata is also sufficient to describe the current inference result # to decide if it is worth caching this right now mi = result.linfo - cache_results = true cache = WorldView(code_cache(interp), result.valid_worlds) - if cache_results && haskey(cache, mi) + if haskey(cache, mi) ci = cache[mi] # n.b.: accurate edge representation might cause the CodeInstance for this to be constructed later @assert isdefined(ci, :inferred) - cache_results = false + return false end - - if cache_results - code_cache(interp)[mi] = result.ci - end - return cache_results + code_cache(interp)[mi] = ci + return true end function cycle_fix_limited(@nospecialize(typ), sv::InferenceState) @@ -387,21 +379,13 @@ function refine_exception_type(@nospecialize(exc_bestguess), ipo_effects::Effect return exc_bestguess end +const empty_edges = Core.svec() + # inference completed on `me` # update the MethodInstance function finishinfer!(me::InferenceState, interp::AbstractInterpreter) # prepare to run optimization passes on fulltree @assert isempty(me.ip) - s_edges = get_stmt_edges!(me, 1) - for i = 2:length(me.stmt_edges) - isassigned(me.stmt_edges, i) || continue - edges = me.stmt_edges[i] - append!(s_edges, edges) - empty!(edges) - end - if me.src.edges !== nothing - append!(s_edges, me.src.edges::Vector) - end # inspect whether our inference had a limited result accuracy, # else it may be suitable to cache bestguess = me.bestguess = cycle_fix_limited(me.bestguess, me) @@ -426,6 +410,8 @@ function finishinfer!(me::InferenceState, interp::AbstractInterpreter) me.src.rettype = widenconst(ignorelimited(bestguess)) me.src.min_world = first(me.valid_worlds) me.src.max_world = last(me.valid_worlds) + istoplevel = !(me.linfo.def isa Method) + istoplevel || compute_edges!(me) # don't add backedges to toplevel method instance if limited_ret # a parent may be cached still, but not this intermediate work: @@ -481,17 +467,19 @@ function finishinfer!(me::InferenceState, interp::AbstractInterpreter) const_flags = 0x2 else rettype_const = nothing - const_flags = 0x00 + const_flags = 0x0 end relocatability = 0x0 - edges = nothing - ccall(:jl_fill_codeinst, Cvoid, (Any, Any, Any, Any, Int32, UInt, UInt, UInt32, Any, Any), - result.ci, widenconst(result_type), widenconst(result.exc_result), rettype_const, const_flags, - first(result.valid_worlds), last(result.valid_worlds), - encode_effects(result.ipo_effects), result.analysis_results, edges) + di = nothing + edges = empty_edges # `edges` will be updated within `finish!` + ci = result.ci + ccall(:jl_fill_codeinst, Cvoid, (Any, Any, Any, Any, Int32, UInt, UInt, UInt32, Any, Any, Any), + ci, widenconst(result_type), widenconst(result.exc_result), rettype_const, const_flags, + first(result.valid_worlds), last(result.valid_worlds), + encode_effects(result.ipo_effects), result.analysis_results, di, edges) if is_cached(me) - cached_results = cache_result!(me.interp, me.result) - if !cached_results + cached_result = cache_result!(me.interp, result, ci) + if !cached_result me.cache_mode = CACHE_MODE_NULL end end @@ -500,19 +488,29 @@ function finishinfer!(me::InferenceState, interp::AbstractInterpreter) end # record the backedges -store_backedges(caller::InferenceResult, edges::Vector{Any}) = store_backedges(caller.linfo, edges) -function store_backedges(caller::MethodInstance, edges::Vector{Any}) - isa(caller.def, Method) || return nothing # don't add backedges to toplevel method instance +function store_backedges(caller::CodeInstance, edges::Vector{Any}) + isa(caller.def.def, Method) || return # don't add backedges to toplevel method instance for itr in BackedgeIterator(edges) callee = itr.caller if isa(callee, MethodInstance) ccall(:jl_method_instance_add_backedge, Cvoid, (Any, Any, Any), callee, itr.sig, caller) else - typeassert(callee, MethodTable) ccall(:jl_method_table_add_backedge, Cvoid, (Any, Any, Any), callee, itr.sig, caller) end end - return nothing + nothing +end + +function compute_edges!(sv::InferenceState) + edges = sv.edges + for i in 1:length(sv.stmt_info) + add_edges!(edges, sv.stmt_info[i]) + end + user_edges = sv.src.edges + if user_edges !== nothing && user_edges !== empty_edges + append!(edges, user_edges) + end + nothing end function record_slot_assign!(sv::InferenceState) @@ -617,7 +615,7 @@ function type_annotate!(interp::AbstractInterpreter, sv::InferenceState) return nothing end -function merge_call_chain!(interp::AbstractInterpreter, parent::InferenceState, child::InferenceState) +function merge_call_chain!(::AbstractInterpreter, parent::InferenceState, child::InferenceState) # add backedge of parent <- child # then add all backedges of parent <- parent.parent frames = parent.callstack::Vector{AbsIntState} @@ -630,14 +628,21 @@ function merge_call_chain!(interp::AbstractInterpreter, parent::InferenceState, parent = frame_parent(child)::InferenceState end # ensure that walking the callstack has the same cycleid (DAG) - for frame = reverse(ancestorid:length(frames)) - frame = frames[frame]::InferenceState + for frameid = reverse(ancestorid:length(frames)) + frame = frames[frameid]::InferenceState frame.cycleid == ancestorid && break @assert frame.cycleid > ancestorid frame.cycleid = ancestorid end end +function add_cycle_backedge!(caller::InferenceState, frame::InferenceState) + update_valid_age!(caller, frame.valid_worlds) + backedge = (caller, caller.currpc) + contains_is(frame.cycle_backedges, backedge) || push!(frame.cycle_backedges, backedge) + return frame +end + function is_same_frame(interp::AbstractInterpreter, mi::MethodInstance, frame::InferenceState) return mi === frame_instance(frame) && cache_owner(interp) === cache_owner(frame.interp) end @@ -661,8 +666,8 @@ function resolve_call_cycle!(interp::AbstractInterpreter, mi::MethodInstance, pa parent isa InferenceState || return false frames = parent.callstack::Vector{AbsIntState} uncached = false - for frame = reverse(1:length(frames)) - frame = frames[frame] + for frameid = reverse(1:length(frames)) + frame = frames[frameid] isa(frame, InferenceState) || break uncached |= !is_cached(frame) # ensure we never add an uncached frame to a cycle if is_same_frame(interp, mi, frame) @@ -682,31 +687,20 @@ end ipo_effects(code::CodeInstance) = decode_effects(code.ipo_purity_bits) -struct EdgeCallResult - rt - exct - edge::Union{Nothing,MethodInstance} - effects::Effects - volatile_inf_result::Union{Nothing,VolatileInferenceResult} - function EdgeCallResult(@nospecialize(rt), @nospecialize(exct), - edge::Union{Nothing,MethodInstance}, - effects::Effects, - volatile_inf_result::Union{Nothing,VolatileInferenceResult} = nothing) - return new(rt, exct, edge, effects, volatile_inf_result) - end -end - # return cached result of regular inference function return_cached_result(interp::AbstractInterpreter, method::Method, codeinst::CodeInstance, caller::AbsIntState, edgecycle::Bool, edgelimited::Bool) rt = cached_return_type(codeinst) + exct = codeinst.exctype effects = ipo_effects(codeinst) + edge = codeinst update_valid_age!(caller, WorldRange(min_world(codeinst), max_world(codeinst))) - return Future(EdgeCall_to_MethodCall_Result(interp, caller, method, EdgeCallResult(rt, codeinst.exctype, codeinst.def, effects), edgecycle, edgelimited)) + return Future(MethodCallResult(interp, caller, method, rt, exct, effects, edge, edgecycle, edgelimited)) end -function EdgeCall_to_MethodCall_Result(interp::AbstractInterpreter, sv::AbsIntState, method::Method, result::EdgeCallResult, edgecycle::Bool, edgelimited::Bool) - (; rt, exct, edge, effects, volatile_inf_result) = result - +function MethodCallResult(::AbstractInterpreter, sv::AbsIntState, method::Method, + @nospecialize(rt), @nospecialize(exct), effects::Effects, + edge::Union{Nothing,CodeInstance}, edgecycle::Bool, edgelimited::Bool, + volatile_inf_result::Union{Nothing,VolatileInferenceResult}=nothing) if edge === nothing edgecycle = edgelimited = true end @@ -729,14 +723,34 @@ function EdgeCall_to_MethodCall_Result(interp::AbstractInterpreter, sv::AbsIntSt end end - return MethodCallResult(rt, exct, edgecycle, edgelimited, edge, effects, volatile_inf_result) + return MethodCallResult(rt, exct, effects, edge, edgecycle, edgelimited, volatile_inf_result) +end + +# allocate a dummy `edge::CodeInstance` to be added by `add_edges!` +function codeinst_as_edge(interp::AbstractInterpreter, sv::InferenceState) + mi = sv.linfo + owner = cache_owner(interp) + min_world, max_world = first(sv.valid_worlds), last(sv.valid_worlds) + if max_world >= get_world_counter() + max_world = typemax(UInt) + end + ci = CodeInstance(mi, owner, Any, Any, nothing, nothing, zero(Int32), + min_world, max_world, zero(UInt32), nothing, zero(UInt8), nothing, Core.svec(sv.edges...)) + if max_world == typemax(UInt) + # if we can record all of the backedges in the global reverse-cache, + # we can now widen our applicability in the global cache too + # TODO: this should probably come after we decide this edge is even useful + store_backedges(ci, sv.edges) + end + return ci end # compute (and cache) an inferred AST and return the current best estimate of the result type function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize(atype), sparams::SimpleVector, caller::AbsIntState, edgecycle::Bool, edgelimited::Bool) - mi = specialize_method(method, atype, sparams)::MethodInstance + mi = specialize_method(method, atype, sparams) cache_mode = CACHE_MODE_GLOBAL # cache edge targets globally by default force_inline = is_stmt_inline(get_curr_ssaflag(caller)) + edge_ci = nothing let codeinst = get(code_cache(interp), mi, nothing) if codeinst isa CodeInstance # return existing rettype if the code is already inferred inferred = @atomic :monotonic codeinst.inferred @@ -745,6 +759,7 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize # nevertheless we re-infer it here again in order to propagate the re-inferred # source to the inliner as a volatile result cache_mode = CACHE_MODE_VOLATILE + edge_ci = codeinst else @assert codeinst.def === mi "MethodInstance for cached edge does not match" return return_cached_result(interp, method, codeinst, caller, edgecycle, edgelimited) @@ -753,7 +768,7 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize end if ccall(:jl_get_module_infer, Cint, (Any,), method.module) == 0 && !generating_output(#=incremental=#false) add_remark!(interp, caller, "[typeinf_edge] Inference is disabled for the target module") - return Future(EdgeCall_to_MethodCall_Result(interp, caller, method, EdgeCallResult(Any, Any, nothing, Effects()), edgecycle, edgelimited)) + return Future(MethodCallResult(interp, caller, method, Any, Any, Effects(), nothing, edgecycle, edgelimited)) end if !is_cached(caller) && frame_parent(caller) === nothing # this caller exists to return to the user @@ -765,32 +780,36 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize if frame === false # completely new, but check again after reserving in the engine if cache_mode == CACHE_MODE_GLOBAL - ci = engine_reserve(interp, mi) - let codeinst = get(code_cache(interp), mi, nothing) - if codeinst isa CodeInstance # return existing rettype if the code is already inferred - engine_reject(interp, ci) - inferred = @atomic :monotonic codeinst.inferred - if inferred === nothing && force_inline - cache_mode = CACHE_MODE_VOLATILE - else - @assert codeinst.def === mi "MethodInstance for cached edge does not match" - return return_cached_result(interp, method, codeinst, caller, edgecycle, edgelimited) - end + ci_from_engine = engine_reserve(interp, mi) + edge_ci = ci_from_engine + codeinst = get(code_cache(interp), mi, nothing) + if codeinst isa CodeInstance # return existing rettype if the code is already inferred + engine_reject(interp, ci_from_engine) + ci_from_engine = nothing + inferred = @atomic :monotonic codeinst.inferred + if inferred === nothing && force_inline + cache_mode = CACHE_MODE_VOLATILE + edge_ci = codeinst + else + @assert codeinst.def === mi "MethodInstance for cached edge does not match" + return return_cached_result(interp, method, codeinst, caller, edgecycle, edgelimited) end end + else + ci_from_engine = nothing end result = InferenceResult(mi, typeinf_lattice(interp)) - if cache_mode == CACHE_MODE_GLOBAL - result.ci = ci + if ci_from_engine !== nothing + result.ci = ci_from_engine end frame = InferenceState(result, cache_mode, interp) # always use the cache for edge targets if frame === nothing add_remark!(interp, caller, "[typeinf_edge] Failed to retrieve source") # can't get the source for this, so we know nothing - if cache_mode == CACHE_MODE_GLOBAL - engine_reject(interp, ci) + if ci_from_engine !== nothing + engine_reject(interp, ci_from_engine) end - return Future(EdgeCall_to_MethodCall_Result(interp, caller, method, EdgeCallResult(Any, Any, nothing, Effects()), edgecycle, edgelimited)) + return Future(MethodCallResult(interp, caller, method, Any, Any, Effects(), nothing, edgecycle, edgelimited)) end assign_parentchild!(frame, caller) # the actual inference task for this edge is going to be scheduled within `typeinf_local` via the callstack queue @@ -799,15 +818,19 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize push!(caller.tasks, function get_infer_result(interp, caller) update_valid_age!(caller, frame.valid_worlds) local isinferred = is_inferred(frame) - local edge = isinferred ? mi : nothing + local edge = isinferred ? edge_ci : nothing local effects = isinferred ? frame.result.ipo_effects : # effects are adjusted already within `finish` for ipo_effects adjust_effects(effects_for_cycle(frame.ipo_effects), method) + local bestguess = frame.bestguess local exc_bestguess = refine_exception_type(frame.exc_bestguess, effects) # propagate newly inferred source to the inliner, allowing efficient inlining w/o deserialization: # note that this result is cached globally exclusively, so we can use this local result destructively - local volatile_inf_result = isinferred ? VolatileInferenceResult(result) : nothing - local edgeresult = EdgeCallResult(frame.bestguess, exc_bestguess, edge, effects, volatile_inf_result) - mresult[] = EdgeCall_to_MethodCall_Result(interp, caller, method, edgeresult, edgecycle, edgelimited) + local volatile_inf_result = if isinferred && edge_ci isa CodeInstance + result.ci_as_edge = edge_ci # set the edge for the inliner usage + VolatileInferenceResult(result) + end + mresult[] = MethodCallResult(interp, caller, method, bestguess, exc_bestguess, effects, + edge, edgecycle, edgelimited, volatile_inf_result) return true end) return mresult @@ -815,15 +838,15 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize elseif frame === true # unresolvable cycle add_remark!(interp, caller, "[typeinf_edge] Unresolvable cycle") - return Future(EdgeCall_to_MethodCall_Result(interp, caller, method, EdgeCallResult(Any, Any, nothing, Effects()), edgecycle, edgelimited)) + return Future(MethodCallResult(interp, caller, method, Any, Any, Effects(), nothing, edgecycle, edgelimited)) end # return the current knowledge about this cycle frame = frame::InferenceState update_valid_age!(caller, frame.valid_worlds) effects = adjust_effects(effects_for_cycle(frame.ipo_effects), method) + bestguess = frame.bestguess exc_bestguess = refine_exception_type(frame.exc_bestguess, effects) - edgeresult = EdgeCallResult(frame.bestguess, exc_bestguess, nothing, effects) - return Future(EdgeCall_to_MethodCall_Result(interp, caller, method, edgeresult, edgecycle, edgelimited)) + return Future(MethodCallResult(interp, caller, method, bestguess, exc_bestguess, effects, nothing, edgecycle, edgelimited)) end # The `:terminates` effect bit must be conservatively tainted unless recursion cycle has @@ -871,6 +894,7 @@ function codeinfo_for_const(interp::AbstractInterpreter, mi::MethodInstance, @no tree.debuginfo = DebugInfo(mi) tree.ssaflags = UInt32[0] tree.rettype = Core.Typeof(val) + tree.edges = Core.svec() set_inlineable!(tree, true) tree.parent = mi return tree @@ -888,7 +912,7 @@ function codeinstance_for_const_with_code(interp::AbstractInterpreter, code::Cod return CodeInstance(code.def, cache_owner(interp), code.rettype, code.exctype, code.rettype_const, src, Int32(0x3), code.min_world, code.max_world, code.ipo_purity_bits, code.analysis_results, - code.relocatability, src.debuginfo) + code.relocatability, src.debuginfo, src.edges) end result_is_constabi(interp::AbstractInterpreter, result::InferenceResult) = @@ -951,13 +975,13 @@ function typeinf_frame(interp::AbstractInterpreter, mi::MethodInstance, run_opti if run_optimizer if result_is_constabi(interp, frame.result) rt = frame.result.result::Const - opt = codeinfo_for_const(interp, frame.linfo, rt.val) + src = codeinfo_for_const(interp, frame.linfo, rt.val) else opt = OptimizationState(frame, interp) optimize(interp, opt, frame.result) - opt = ir_to_codeinf!(opt) + src = ir_to_codeinf!(opt) end - result.src = frame.src = opt + result.src = frame.src = src end return frame end @@ -1050,7 +1074,7 @@ function typeinf_ext(interp::AbstractInterpreter, mi::MethodInstance, source_mod src isa CodeInfo || return nothing return CodeInstance(mi, cache_owner(interp), Any, Any, nothing, src, Int32(0), get_inference_world(interp), get_inference_world(interp), - UInt32(0), nothing, UInt8(0), src.debuginfo) + UInt32(0), nothing, UInt8(0), src.debuginfo, src.edges) end end ci = engine_reserve(interp, mi) diff --git a/base/compiler/types.jl b/base/compiler/types.jl index b6c976da48f67..8899e7673d753 100644 --- a/base/compiler/types.jl +++ b/base/compiler/types.jl @@ -91,21 +91,31 @@ There are two constructor available: for constant inference, with extended lattice information included in `result.argtypes`. """ mutable struct InferenceResult + #=== constant fields ===# const linfo::MethodInstance const argtypes::Vector{Any} const overridden_by_const::Union{Nothing,BitVector} - result # extended lattice element if inferred, nothing otherwise - exc_result # like `result`, but for the thrown value - src # ::Union{CodeInfo, IRCode, OptimizationState} if inferred copy is available, nothing otherwise - valid_worlds::WorldRange # if inference and optimization is finished - ipo_effects::Effects # if inference is finished - effects::Effects # if optimization is finished + + #=== mutable fields ===# + result # extended lattice element if inferred, nothing otherwise + exc_result # like `result`, but for the thrown value + src # ::Union{CodeInfo, IRCode, OptimizationState} if inferred copy is available, nothing otherwise + valid_worlds::WorldRange # if inference and optimization is finished + ipo_effects::Effects # if inference is finished + effects::Effects # if optimization is finished analysis_results::AnalysisResults # AnalysisResults with e.g. result::ArgEscapeCache if optimized, otherwise NULL_ANALYSIS_RESULTS - is_src_volatile::Bool # `src` has been cached globally as the compressed format already, allowing `src` to be used destructively - ci::CodeInstance # CodeInstance if this result may be added to the cache + is_src_volatile::Bool # `src` has been cached globally as the compressed format already, allowing `src` to be used destructively + + #=== uninitialized fields ===# + ci::CodeInstance # CodeInstance if this result may be added to the cache + ci_as_edge::CodeInstance # CodeInstance as the edge representing locally cached result function InferenceResult(mi::MethodInstance, argtypes::Vector{Any}, overridden_by_const::Union{Nothing,BitVector}) - return new(mi, argtypes, overridden_by_const, nothing, nothing, nothing, - WorldRange(), Effects(), Effects(), NULL_ANALYSIS_RESULTS, false) + result = exc_result = src = nothing + valid_worlds = WorldRange() + ipo_effects = effects = Effects() + analysis_results = NULL_ANALYSIS_RESULTS + return new(mi, argtypes, overridden_by_const, result, exc_result, src, + valid_worlds, ipo_effects, effects, analysis_results, #=is_src_volatile=#false) end end function InferenceResult(mi::MethodInstance, 𝕃::AbstractLattice=fallback_lattice) @@ -399,7 +409,6 @@ engine_reserve(mi::MethodInstance, @nospecialize owner) = ccall(:jl_engine_reser # engine_fulfill(::AbstractInterpreter, ci::CodeInstance, src::CodeInfo) = ccall(:jl_engine_fulfill, Cvoid, (Any, Any), ci, src) # currently the same as engine_reject, so just use that one engine_reject(::AbstractInterpreter, ci::CodeInstance) = ccall(:jl_engine_fulfill, Cvoid, (Any, Ptr{Cvoid}), ci, C_NULL) - function already_inferred_quick_test end function lock_mi_inference end function unlock_mi_inference end @@ -416,7 +425,6 @@ function add_remark! end may_optimize(::AbstractInterpreter) = true may_compress(::AbstractInterpreter) = true may_discard_trees(::AbstractInterpreter) = true -verbose_stmt_info(::AbstractInterpreter) = false """ method_table(interp::AbstractInterpreter) -> MethodTableView @@ -463,18 +471,23 @@ abstract type CallInfo end @nospecialize +function add_edges!(edges::Vector{Any}, info::CallInfo) + if info === NoCallInfo() + return nothing # just a minor optimization to avoid dynamic dispatch + end + add_edges_impl(edges, info) + nothing +end nsplit(info::CallInfo) = nsplit_impl(info)::Union{Nothing,Int} getsplit(info::CallInfo, idx::Int) = getsplit_impl(info, idx)::MethodLookupResult -add_uncovered_edges!(edges::Vector{Any}, info::CallInfo, @nospecialize(atype)) = add_uncovered_edges_impl(edges, info, atype) - -getresult(info::CallInfo, idx::Int) = getresult_impl(info, idx) +getresult(info::CallInfo, idx::Int) = getresult_impl(info, idx)#=::Union{Nothing,ConstResult}=# -# must implement `nsplit`, `getsplit`, and `add_uncovered_edges!` to opt in to inlining +add_edges_impl(::Vector{Any}, ::CallInfo) = error(""" + All `CallInfo` is required to implement `add_edges_impl(::Vector{Any}, ::CallInfo)`""") nsplit_impl(::CallInfo) = nothing -getsplit_impl(::CallInfo, ::Int) = error("unexpected call into `getsplit`") -add_uncovered_edges_impl(::Vector{Any}, ::CallInfo, _) = error("unexpected call into `add_uncovered_edges!`") - -# must implement `getresult` to opt in to extended lattice return information +getsplit_impl(::CallInfo, ::Int) = error(""" + A `info::CallInfo` that implements `nsplit_impl(info::CallInfo) -> Int` must implement `getsplit_impl(info::CallInfo, idx::Int) -> MethodLookupResult` + in order to correctly opt in to inlining""") getresult_impl(::CallInfo, ::Int) = nothing @specialize diff --git a/base/compiler/utilities.jl b/base/compiler/utilities.jl index f9202788b6360..5361ff26f997c 100644 --- a/base/compiler/utilities.jl +++ b/base/compiler/utilities.jl @@ -252,8 +252,6 @@ struct BackedgeIterator backedges::Vector{Any} end -const empty_backedge_iter = BackedgeIterator(Any[]) - struct BackedgePair sig # ::Union{Nothing,Type} caller::Union{MethodInstance,MethodTable} @@ -262,11 +260,36 @@ end function iterate(iter::BackedgeIterator, i::Int=1) backedges = iter.backedges - i > length(backedges) && return nothing - item = backedges[i] - isa(item, MethodInstance) && return BackedgePair(nothing, item), i+1 # regular dispatch - isa(item, MethodTable) && return BackedgePair(backedges[i+1], item), i+2 # abstract dispatch - return BackedgePair(item, backedges[i+1]::MethodInstance), i+2 # `invoke` calls + while true + i > length(backedges) && return nothing + item = backedges[i] + if item isa Int + i += 2 + continue # ignore the query information if present + elseif isa(item, Method) + # ignore `Method`-edges (from e.g. failed `abstract_call_method`) + i += 1 + continue + end + if isa(item, CodeInstance) + item = item.def + end + if isa(item, MethodInstance) # regular dispatch + return BackedgePair(nothing, item), i+1 + elseif isa(item, MethodTable) # abstract dispatch (legacy style edges) + return BackedgePair(backedges[i+1], item), i+2 + else # `invoke` call + callee = backedges[i+1] + if isa(callee, Method) + i += 2 + continue + end + if isa(callee, CodeInstance) + callee = callee.def + end + return BackedgePair(item, callee::MethodInstance), i+2 + end + end end ######### diff --git a/base/expr.jl b/base/expr.jl index e281d9b677297..f57331ef02e74 100644 --- a/base/expr.jl +++ b/base/expr.jl @@ -79,7 +79,7 @@ function copy(c::CodeInfo) cnew.slottypes = copy(cnew.slottypes::Vector{Any}) end cnew.ssaflags = copy(cnew.ssaflags) - cnew.edges = cnew.edges === nothing ? nothing : copy(cnew.edges::Vector) + cnew.edges = cnew.edges === nothing || cnew.edges isa Core.SimpleVector ? cnew.edges : copy(cnew.edges::Vector) ssavaluetypes = cnew.ssavaluetypes ssavaluetypes isa Vector{Any} && (cnew.ssavaluetypes = copy(ssavaluetypes)) return cnew diff --git a/base/show.jl b/base/show.jl index dbdd85f0608da..627982b2bcb1a 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1350,7 +1350,11 @@ function sourceinfo_slotnames(slotnames::Vector{Symbol}) return printnames end -show(io::IO, l::Core.MethodInstance) = show_mi(io, l) +show(io::IO, mi::Core.MethodInstance) = show_mi(io, mi) +function show(io::IO, codeinst::Core.CodeInstance) + print(io, "CodeInstance for ") + show_mi(io, codeinst.def) +end function show_mi(io::IO, mi::Core.MethodInstance, from_stackframe::Bool=false) def = mi.def diff --git a/src/common_symbols1.inc b/src/common_symbols1.inc index f54be52729a4f..3dfcf17a07b5c 100644 --- a/src/common_symbols1.inc +++ b/src/common_symbols1.inc @@ -88,5 +88,3 @@ jl_symbol("ifelse"), jl_symbol("Array"), jl_symbol("eq_int"), jl_symbol("throw_inexacterror"), -jl_symbol("|"), -jl_symbol("setproperty!"), diff --git a/src/common_symbols2.inc b/src/common_symbols2.inc index ee2a0e2edd9fe..2a6990bac52ff 100644 --- a/src/common_symbols2.inc +++ b/src/common_symbols2.inc @@ -1,3 +1,5 @@ +jl_symbol("|"), +jl_symbol("setproperty!"), jl_symbol("sext_int"), jl_symbol("String"), jl_symbol("Int"), @@ -244,5 +246,3 @@ jl_symbol("invokelatest"), jl_symbol("jl_array_del_end"), jl_symbol("_mod64"), jl_symbol("parameters"), -jl_symbol("monotonic"), -jl_symbol("regex.jl"), diff --git a/src/gf.c b/src/gf.c index 285942cd157c5..18141410fa625 100644 --- a/src/gf.c +++ b/src/gf.c @@ -322,7 +322,7 @@ jl_datatype_t *jl_mk_builtin_func(jl_datatype_t *dt, const char *name, jl_fptr_a jl_code_instance_t *codeinst = jl_new_codeinst(mi, jl_nothing, (jl_value_t*)jl_any_type, (jl_value_t*)jl_any_type, jl_nothing, jl_nothing, - 0, 1, ~(size_t)0, 0, jl_nothing, 0, NULL); + 0, 1, ~(size_t)0, 0, jl_nothing, 0, NULL, NULL); jl_mi_cache_insert(mi, codeinst); jl_atomic_store_relaxed(&codeinst->specptr.fptr1, fptr); jl_atomic_store_relaxed(&codeinst->invoke, jl_fptr_args); @@ -480,7 +480,7 @@ JL_DLLEXPORT jl_value_t *jl_call_in_typeinf_world(jl_value_t **args, int nargs) JL_DLLEXPORT jl_code_instance_t *jl_get_method_inferred( jl_method_instance_t *mi JL_PROPAGATES_ROOT, jl_value_t *rettype, - size_t min_world, size_t max_world, jl_debuginfo_t *edges) + size_t min_world, size_t max_world, jl_debuginfo_t *di, jl_svec_t *edges) { jl_value_t *owner = jl_nothing; // TODO: owner should be arg jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&mi->cache); @@ -489,27 +489,30 @@ JL_DLLEXPORT jl_code_instance_t *jl_get_method_inferred( jl_atomic_load_relaxed(&codeinst->max_world) == max_world && jl_egal(codeinst->owner, owner) && jl_egal(codeinst->rettype, rettype)) { - if (edges == NULL) + if (di == NULL) return codeinst; jl_debuginfo_t *debuginfo = jl_atomic_load_relaxed(&codeinst->debuginfo); - if (edges == debuginfo) - return codeinst; - if (debuginfo == NULL && jl_atomic_cmpswap_relaxed(&codeinst->debuginfo, &debuginfo, edges)) - return codeinst; - if (debuginfo && jl_egal((jl_value_t*)debuginfo, (jl_value_t*)edges)) + if (di != debuginfo) { + if (!(debuginfo == NULL && jl_atomic_cmpswap_relaxed(&codeinst->debuginfo, &debuginfo, di))) + if (!(debuginfo && jl_egal((jl_value_t*)debuginfo, (jl_value_t*)di))) + continue; + } + // TODO: this is implied by the matching worlds, since it is intrinsic, so do we really need to verify it? + jl_svec_t *e = jl_atomic_load_relaxed(&codeinst->edges); + if (e && jl_egal((jl_value_t*)e, (jl_value_t*)edges)) return codeinst; } codeinst = jl_atomic_load_relaxed(&codeinst->next); } codeinst = jl_new_codeinst( mi, owner, rettype, (jl_value_t*)jl_any_type, NULL, NULL, - 0, min_world, max_world, 0, jl_nothing, 0, edges); + 0, min_world, max_world, 0, jl_nothing, 0, di, edges); jl_mi_cache_insert(mi, codeinst); return codeinst; } JL_DLLEXPORT int jl_mi_cache_has_ci(jl_method_instance_t *mi, - jl_code_instance_t *ci) + jl_code_instance_t *ci) { jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&mi->cache); while (codeinst) { @@ -527,14 +530,16 @@ JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst( int32_t const_flags, size_t min_world, size_t max_world, uint32_t effects, jl_value_t *analysis_results, uint8_t relocatability, - jl_debuginfo_t *edges /* , int absolute_max*/) + jl_debuginfo_t *di, jl_svec_t *edges /*, int absolute_max*/) { - jl_task_t *ct = jl_current_task; assert(min_world <= max_world && "attempting to set invalid world constraints"); + //assert((!jl_is_method(mi->def.value) || max_world != ~(size_t)0 || min_world <= 1 || edges == NULL || jl_svec_len(edges) != 0) && "missing edges"); + jl_task_t *ct = jl_current_task; jl_code_instance_t *codeinst = (jl_code_instance_t*)jl_gc_alloc(ct->ptls, sizeof(jl_code_instance_t), jl_code_instance_type); codeinst->def = mi; codeinst->owner = owner; + jl_atomic_store_relaxed(&codeinst->edges, edges); jl_atomic_store_relaxed(&codeinst->min_world, min_world); jl_atomic_store_relaxed(&codeinst->max_world, max_world); codeinst->rettype = rettype; @@ -543,7 +548,7 @@ JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst( if ((const_flags & 2) == 0) inferred_const = NULL; codeinst->rettype_const = inferred_const; - jl_atomic_store_relaxed(&codeinst->debuginfo, (jl_value_t*)edges == jl_nothing ? NULL : edges); + jl_atomic_store_relaxed(&codeinst->debuginfo, (jl_value_t*)di == jl_nothing ? NULL : di); jl_atomic_store_relaxed(&codeinst->specptr.fptr, NULL); jl_atomic_store_relaxed(&codeinst->invoke, NULL); if ((const_flags & 1) != 0) { @@ -563,13 +568,17 @@ JL_DLLEXPORT void jl_update_codeinst( jl_code_instance_t *codeinst, jl_value_t *inferred, int32_t const_flags, size_t min_world, size_t max_world, uint32_t effects, jl_value_t *analysis_results, - uint8_t relocatability, jl_debuginfo_t *edges /* , int absolute_max*/) + uint8_t relocatability, jl_debuginfo_t *di, jl_svec_t *edges /* , int absolute_max*/) { + assert(min_world <= max_world && "attempting to set invalid world constraints"); + //assert((!jl_is_method(codeinst->def->def.value) || max_world != ~(size_t)0 || min_world <= 1 || jl_svec_len(edges) != 0) && "missing edges"); codeinst->relocatability = relocatability; codeinst->analysis_results = analysis_results; jl_gc_wb(codeinst, analysis_results); jl_atomic_store_relaxed(&codeinst->ipo_purity_bits, effects); - jl_atomic_store_relaxed(&codeinst->debuginfo, edges); + jl_atomic_store_relaxed(&codeinst->debuginfo, di); + jl_gc_wb(codeinst, di); + jl_atomic_store_relaxed(&codeinst->edges, edges); jl_gc_wb(codeinst, edges); if ((const_flags & 1) != 0) { assert(codeinst->rettype_const); @@ -587,9 +596,10 @@ JL_DLLEXPORT void jl_fill_codeinst( jl_value_t *inferred_const, int32_t const_flags, size_t min_world, size_t max_world, uint32_t effects, jl_value_t *analysis_results, - jl_debuginfo_t *edges /* , int absolute_max*/) + jl_debuginfo_t *di, jl_svec_t *edges /* , int absolute_max*/) { assert(min_world <= max_world && "attempting to set invalid world constraints"); + //assert((!jl_is_method(codeinst->def->def.value) || max_world != ~(size_t)0 || min_world <= 1 || jl_svec_len(edges) != 0) && "missing edges"); codeinst->rettype = rettype; jl_gc_wb(codeinst, rettype); codeinst->exctype = exctype; @@ -598,8 +608,12 @@ JL_DLLEXPORT void jl_fill_codeinst( codeinst->rettype_const = inferred_const; jl_gc_wb(codeinst, inferred_const); } - jl_atomic_store_relaxed(&codeinst->debuginfo, (jl_value_t*)edges == jl_nothing ? NULL : edges); + jl_atomic_store_relaxed(&codeinst->edges, edges); jl_gc_wb(codeinst, edges); + if ((jl_value_t*)di != jl_nothing) { + jl_atomic_store_relaxed(&codeinst->debuginfo, di); + jl_gc_wb(codeinst, di); + } if ((const_flags & 1) != 0) { // TODO: may want to follow ordering restrictions here (see jitlayers.cpp) assert(const_flags & 2); @@ -616,7 +630,7 @@ JL_DLLEXPORT void jl_fill_codeinst( JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst_uninit(jl_method_instance_t *mi, jl_value_t *owner) { - jl_code_instance_t *codeinst = jl_new_codeinst(mi, owner, NULL, NULL, NULL, NULL, 0, 0, 0, 0, NULL, 0, NULL); + jl_code_instance_t *codeinst = jl_new_codeinst(mi, owner, NULL, NULL, NULL, NULL, 0, 0, 0, 0, NULL, 0, NULL, NULL); jl_atomic_store_relaxed(&codeinst->min_world, 1); // make temporarily invalid before returning, so that jl_fill_codeinst is valid later return codeinst; } @@ -1745,35 +1759,29 @@ JL_DLLEXPORT jl_value_t *jl_debug_method_invalidation(int state) static void _invalidate_backedges(jl_method_instance_t *replaced_mi, size_t max_world, int depth); // recursively invalidate cached methods that had an edge to a replaced method -static void invalidate_method_instance(jl_method_instance_t *replaced, size_t max_world, int depth) +static void invalidate_code_instance(jl_code_instance_t *replaced, size_t max_world, int depth) { jl_timing_counter_inc(JL_TIMING_COUNTER_Invalidations, 1); if (_jl_debug_method_invalidation) { jl_value_t *boxeddepth = NULL; JL_GC_PUSH1(&boxeddepth); - jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)replaced); + jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)replaced->def); boxeddepth = jl_box_int32(depth); jl_array_ptr_1d_push(_jl_debug_method_invalidation, boxeddepth); JL_GC_POP(); } - //jl_static_show(JL_STDERR, (jl_value_t*)replaced); - if (!jl_is_method(replaced->def.method)) + //jl_static_show(JL_STDERR, (jl_value_t*)replaced->def); + if (!jl_is_method(replaced->def->def.method)) return; // shouldn't happen, but better to be safe - JL_LOCK(&replaced->def.method->writelock); - jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&replaced->cache); - while (codeinst) { - if (jl_atomic_load_relaxed(&codeinst->max_world) == ~(size_t)0) { - assert(jl_atomic_load_relaxed(&codeinst->min_world) - 1 <= max_world && "attempting to set illogical world constraints (probable race condition)"); - jl_atomic_store_release(&codeinst->max_world, max_world); - } - assert(jl_atomic_load_relaxed(&codeinst->max_world) <= max_world); - codeinst = jl_atomic_load_relaxed(&codeinst->next); + JL_LOCK(&replaced->def->def.method->writelock); + if (jl_atomic_load_relaxed(&replaced->max_world) == ~(size_t)0) { + assert(jl_atomic_load_relaxed(&replaced->min_world) - 1 <= max_world && "attempting to set illogical world constraints (probable race condition)"); + jl_atomic_store_release(&replaced->max_world, max_world); } - JL_GC_PUSH1(&replaced); + assert(jl_atomic_load_relaxed(&replaced->max_world) <= max_world); // recurse to all backedges to update their valid range also - _invalidate_backedges(replaced, max_world, depth + 1); - JL_GC_POP(); - JL_UNLOCK(&replaced->def.method->writelock); + _invalidate_backedges(replaced->def, max_world, depth + 1); + JL_UNLOCK(&replaced->def->def.method->writelock); } static void _invalidate_backedges(jl_method_instance_t *replaced_mi, size_t max_world, int depth) { @@ -1783,10 +1791,11 @@ static void _invalidate_backedges(jl_method_instance_t *replaced_mi, size_t max_ replaced_mi->backedges = NULL; JL_GC_PUSH1(&backedges); size_t i = 0, l = jl_array_nrows(backedges); - jl_method_instance_t *replaced; + jl_code_instance_t *replaced; while (i < l) { i = get_next_edge(backedges, i, NULL, &replaced); - invalidate_method_instance(replaced, max_world, depth); + JL_GC_PROMISE_ROOTED(replaced); // propagated by get_next_edge from backedges + invalidate_code_instance(replaced, max_world, depth); } JL_GC_POP(); } @@ -1808,11 +1817,14 @@ static void invalidate_backedges(jl_method_instance_t *replaced_mi, size_t max_w } // add a backedge from callee to caller -JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, jl_value_t *invokesig, jl_method_instance_t *caller) +JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, jl_value_t *invokesig, jl_code_instance_t *caller) { - JL_LOCK(&callee->def.method->writelock); if (invokesig == jl_nothing) invokesig = NULL; // julia uses `nothing` but C uses NULL (#undef) + assert(jl_is_method_instance(callee)); + assert(jl_is_code_instance(caller)); + assert(invokesig == NULL || jl_is_type(invokesig)); + JL_LOCK(&callee->def.method->writelock); int found = 0; // TODO: use jl_cache_type_(invokesig) like cache_method does to save memory if (!callee->backedges) { @@ -1843,8 +1855,9 @@ JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, } // add a backedge from a non-existent signature to caller -JL_DLLEXPORT void jl_method_table_add_backedge(jl_methtable_t *mt, jl_value_t *typ, jl_value_t *caller) +JL_DLLEXPORT void jl_method_table_add_backedge(jl_methtable_t *mt, jl_value_t *typ, jl_code_instance_t *caller) { + assert(jl_is_code_instance(caller)); JL_LOCK(&mt->writelock); if (!mt->backedges) { // lazy-init the backedges array @@ -1857,7 +1870,7 @@ JL_DLLEXPORT void jl_method_table_add_backedge(jl_methtable_t *mt, jl_value_t *t // check if the edge is already present and avoid adding a duplicate size_t i, l = jl_array_nrows(mt->backedges); for (i = 1; i < l; i += 2) { - if (jl_array_ptr_ref(mt->backedges, i) == caller) { + if (jl_array_ptr_ref(mt->backedges, i) == (jl_value_t*)caller) { if (jl_types_equal(jl_array_ptr_ref(mt->backedges, i - 1), typ)) { JL_UNLOCK(&mt->writelock); return; @@ -1867,7 +1880,7 @@ JL_DLLEXPORT void jl_method_table_add_backedge(jl_methtable_t *mt, jl_value_t *t // reuse an already cached instance of this type, if possible // TODO: use jl_cache_type_(tt) like cache_method does, instead of this linear scan? for (i = 1; i < l; i += 2) { - if (jl_array_ptr_ref(mt->backedges, i) != caller) { + if (jl_array_ptr_ref(mt->backedges, i) != (jl_value_t*)caller) { if (jl_types_equal(jl_array_ptr_ref(mt->backedges, i - 1), typ)) { typ = jl_array_ptr_ref(mt->backedges, i - 1); break; @@ -1875,7 +1888,7 @@ JL_DLLEXPORT void jl_method_table_add_backedge(jl_methtable_t *mt, jl_value_t *t } } jl_array_ptr_1d_push(mt->backedges, typ); - jl_array_ptr_1d_push(mt->backedges, caller); + jl_array_ptr_1d_push(mt->backedges, (jl_value_t*)caller); } JL_UNLOCK(&mt->writelock); } @@ -2161,6 +2174,7 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) size_t ins = 0; for (i = 1; i < na; i += 2) { jl_value_t *backedgetyp = backedges[i - 1]; + JL_GC_PROMISE_ROOTED(backedgetyp); int missing = 0; if (jl_type_intersection2(backedgetyp, (jl_value_t*)type, &isect, &isect2)) { // See if the intersection was actually already fully @@ -2189,8 +2203,9 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) } } if (missing) { - jl_method_instance_t *backedge = (jl_method_instance_t*)backedges[i]; - invalidate_method_instance(backedge, max_world, 0); + jl_code_instance_t *backedge = (jl_code_instance_t*)backedges[i]; + JL_GC_PROMISE_ROOTED(backedge); + invalidate_code_instance(backedge, max_world, 0); invalidated = 1; if (_jl_debug_method_invalidation) jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)backedgetyp); @@ -2253,13 +2268,14 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) if (backedges) { size_t ib = 0, insb = 0, nb = jl_array_nrows(backedges); jl_value_t *invokeTypes; - jl_method_instance_t *caller; + jl_code_instance_t *caller; while (ib < nb) { ib = get_next_edge(backedges, ib, &invokeTypes, &caller); + JL_GC_PROMISE_ROOTED(caller); // propagated by get_next_edge from backedges int replaced_edge; if (invokeTypes) { // n.b. normally we must have mi.specTypes <: invokeTypes <: m.sig (though it might not strictly hold), so we only need to check the other subtypes - if (jl_egal(invokeTypes, caller->def.method->sig)) + if (jl_egal(invokeTypes, caller->def->def.method->sig)) replaced_edge = 0; // if invokeTypes == m.sig, then the only way to change this invoke is to replace the method itself else replaced_edge = jl_subtype(invokeTypes, type) && is_replacing(ambig, type, m, d, n, invokeTypes, NULL, morespec); @@ -2268,7 +2284,7 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) replaced_edge = replaced_dispatch; } if (replaced_edge) { - invalidate_method_instance(caller, max_world, 1); + invalidate_code_instance(caller, max_world, 1); invalidated = 1; } else { @@ -2685,8 +2701,10 @@ jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t jl_code_instance_t *codeinst2 = jl_compile_method_internal(mi2, world); jl_code_instance_t *codeinst = jl_get_method_inferred( mi, codeinst2->rettype, - jl_atomic_load_relaxed(&codeinst2->min_world), jl_atomic_load_relaxed(&codeinst2->max_world), - jl_atomic_load_relaxed(&codeinst2->debuginfo)); + jl_atomic_load_relaxed(&codeinst2->min_world), + jl_atomic_load_relaxed(&codeinst2->max_world), + jl_atomic_load_relaxed(&codeinst2->debuginfo), + jl_atomic_load_relaxed(&codeinst2->edges)); if (jl_atomic_load_relaxed(&codeinst->invoke) == NULL) { codeinst->rettype_const = codeinst2->rettype_const; jl_gc_wb(codeinst, codeinst->rettype_const); @@ -2743,7 +2761,7 @@ jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t if (unspec && (unspec_invoke = jl_atomic_load_acquire(&unspec->invoke))) { jl_code_instance_t *codeinst = jl_new_codeinst(mi, jl_nothing, (jl_value_t*)jl_any_type, (jl_value_t*)jl_any_type, NULL, NULL, - 0, 1, ~(size_t)0, 0, jl_nothing, 0, NULL); + 0, 1, ~(size_t)0, 0, jl_nothing, 0, NULL, NULL); codeinst->rettype_const = unspec->rettype_const; uint8_t specsigflags; jl_callptr_t invoke; @@ -2768,7 +2786,7 @@ jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t if (!jl_code_requires_compiler(src, 0)) { jl_code_instance_t *codeinst = jl_new_codeinst(mi, jl_nothing, (jl_value_t*)jl_any_type, (jl_value_t*)jl_any_type, NULL, NULL, - 0, 1, ~(size_t)0, 0, jl_nothing, 0, NULL); + 0, 1, ~(size_t)0, 0, jl_nothing, 0, NULL, NULL); jl_atomic_store_release(&codeinst->invoke, jl_fptr_interpret_call); jl_mi_cache_insert(mi, codeinst); record_precompile_statement(mi, 0, 0); @@ -2828,7 +2846,7 @@ jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t jl_method_instance_t *unspec = jl_get_unspecialized(def); if (unspec == NULL) unspec = mi; - jl_code_instance_t *ucache = jl_get_method_inferred(unspec, (jl_value_t*)jl_any_type, 1, ~(size_t)0, NULL); + jl_code_instance_t *ucache = jl_get_method_inferred(unspec, (jl_value_t*)jl_any_type, 1, ~(size_t)0, NULL, NULL); // ask codegen to make the fptr for unspec jl_callptr_t ucache_invoke = jl_atomic_load_acquire(&ucache->invoke); if (ucache_invoke == NULL) { @@ -2848,7 +2866,7 @@ jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t } codeinst = jl_new_codeinst(mi, jl_nothing, (jl_value_t*)jl_any_type, (jl_value_t*)jl_any_type, NULL, NULL, - 0, 1, ~(size_t)0, 0, jl_nothing, 0, NULL); + 0, 1, ~(size_t)0, 0, jl_nothing, 0, NULL, NULL); codeinst->rettype_const = ucache->rettype_const; uint8_t specsigflags; jl_callptr_t invoke; diff --git a/src/ircode.c b/src/ircode.c index 873d33d2d7523..bec8d46513eef 100644 --- a/src/ircode.c +++ b/src/ircode.c @@ -25,6 +25,7 @@ typedef struct { ios_t *s; // method we're compressing for jl_method_t *method; + jl_svec_t *edges; jl_ptls_t ptls; uint8_t relocatability; } jl_ircode_state; @@ -72,7 +73,7 @@ static void literal_val_id(rle_reference *rr, jl_ircode_state *s, jl_value_t *v) { jl_array_t *rs = s->method->roots; int i, l = jl_array_nrows(rs); - if (jl_is_symbol(v) || jl_is_concrete_type(v)) { + if (jl_is_symbol(v) || jl_is_concrete_type(v)) { // TODO: or more generally, any ptr-egal value for (i = 0; i < l; i++) { if (jl_array_ptr_ref(rs, i) == v) return tagged_root(rr, s, i); @@ -84,6 +85,12 @@ static void literal_val_id(rle_reference *rr, jl_ircode_state *s, jl_value_t *v) return tagged_root(rr, s, i); } } + for (size_t i = 0; i < jl_svec_len(s->edges); i++) { + if (jl_svecref(s->edges, i) == v) { + rr->index = i; + return; + } + } jl_add_method_root(s->method, jl_precompile_toplevel_module, v); return tagged_root(rr, s, jl_array_nrows(rs) - 1); } @@ -102,13 +109,24 @@ static void jl_encode_int32(jl_ircode_state *s, int32_t x) static void jl_encode_as_indexed_root(jl_ircode_state *s, jl_value_t *v) { - rle_reference rr; + rle_reference rr = {.key = -1, .index = -1}; if (jl_is_string(v)) v = jl_as_global_root(v, 1); literal_val_id(&rr, s, v); int id = rr.index; assert(id >= 0); + if (rr.key == -1) { + if (id <= UINT8_MAX) { + write_uint8(s->s, TAG_EDGE); + write_uint8(s->s, id); + } + else { + write_uint8(s->s, TAG_LONG_EDGE); + write_uint32(s->s, id); + } + return; + } if (rr.key) { write_uint8(s->s, TAG_RELOC_METHODROOT); write_uint64(s->s, rr.key); @@ -689,6 +707,10 @@ static jl_value_t *jl_decode_value(jl_ircode_state *s) JL_GC_DISABLED return lookup_root(s->method, 0, read_uint8(s->s)); case TAG_LONG_METHODROOT: return lookup_root(s->method, 0, read_uint32(s->s)); + case TAG_EDGE: + return jl_svecref(s->edges, read_uint8(s->s)); + case TAG_LONG_EDGE: + return jl_svecref(s->edges, read_uint32(s->s)); case TAG_SVEC: JL_FALLTHROUGH; case TAG_LONG_SVEC: return jl_decode_value_svec(s, tag); case TAG_COMMONSYM: @@ -865,9 +887,11 @@ JL_DLLEXPORT jl_string_t *jl_compress_ir(jl_method_t *m, jl_code_info_t *code) m->roots = jl_alloc_vec_any(0); jl_gc_wb(m, m->roots); } + jl_value_t *edges = code->edges; jl_ircode_state s = { &dest, m, + (!isdef && jl_is_svec(edges)) ? (jl_svec_t*)edges : jl_emptysvec, jl_current_task->ptls, 1 }; @@ -950,6 +974,7 @@ JL_DLLEXPORT jl_code_info_t *jl_uncompress_ir(jl_method_t *m, jl_code_instance_t jl_ircode_state s = { &src, m, + metadata == NULL ? NULL : jl_atomic_load_relaxed(&metadata->edges), jl_current_task->ptls, 1 }; @@ -1015,6 +1040,8 @@ JL_DLLEXPORT jl_code_info_t *jl_uncompress_ir(jl_method_t *m, jl_code_instance_t jl_gc_wb(code, code->rettype); code->min_world = jl_atomic_load_relaxed(&metadata->min_world); code->max_world = jl_atomic_load_relaxed(&metadata->max_world); + code->edges = (jl_value_t*)s.edges; + jl_gc_wb(code, s.edges); } return code; diff --git a/src/jltypes.c b/src/jltypes.c index 11f1d11a14edc..71eaa003d7d4a 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3638,7 +3638,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_code_instance_type = jl_new_datatype(jl_symbol("CodeInstance"), core, jl_any_type, jl_emptysvec, - jl_perm_symsvec(17, + jl_perm_symsvec(18, "def", "owner", "next", @@ -3648,13 +3648,14 @@ void jl_init_types(void) JL_GC_DISABLED "exctype", "rettype_const", "inferred", - "debuginfo", // TODO: rename to edges? + "debuginfo", + "edges", //"absolute_max", "ipo_purity_bits", "analysis_results", "specsigflags", "precompile", "relocatability", "invoke", "specptr"), // function object decls - jl_svec(17, + jl_svec(18, jl_method_instance_type, jl_any_type, jl_any_type, @@ -3665,6 +3666,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_any_type, jl_any_type, jl_debuginfo_type, + jl_simplevector_type, //jl_bool_type, jl_uint32_type, jl_any_type, @@ -3675,8 +3677,8 @@ void jl_init_types(void) JL_GC_DISABLED jl_emptysvec, 0, 1, 1); jl_svecset(jl_code_instance_type->types, 2, jl_code_instance_type); - const static uint32_t code_instance_constfields[1] = { 0b00000100011100011 }; // Set fields 1, 2, 6-8, 12 as const - const static uint32_t code_instance_atomicfields[1] = { 0b11011011100011100 }; // Set fields 3-5, 9, 10, 11, 13-14, 16-17 as atomic + const static uint32_t code_instance_constfields[1] = { 0b000001000011100011 }; // Set fields 1, 2, 6-8, 13 as const + const static uint32_t code_instance_atomicfields[1] = { 0b110110111100011100 }; // Set fields 3-5, 9-12, 14-15, 17-18 as atomic // Fields 4-5 are only operated on by construction and deserialization, so are effectively const at runtime // Fields ipo_purity_bits and analysis_results are not currently threadsafe or reliable, as they get mutated after optimization, but are not declared atomic // and there is no way to tell (during inference) if their value is finalized yet (to wait for them to be narrowed if applicable) @@ -3826,8 +3828,8 @@ void jl_init_types(void) JL_GC_DISABLED jl_svecset(jl_method_type->types, 13, jl_method_instance_type); //jl_svecset(jl_debuginfo_type->types, 0, jl_method_instance_type); // union(jl_method_instance_type, jl_method_type, jl_symbol_type) jl_svecset(jl_method_instance_type->types, 4, jl_code_instance_type); - jl_svecset(jl_code_instance_type->types, 15, jl_voidpointer_type); jl_svecset(jl_code_instance_type->types, 16, jl_voidpointer_type); + jl_svecset(jl_code_instance_type->types, 17, jl_voidpointer_type); jl_svecset(jl_binding_type->types, 0, jl_globalref_type); jl_svecset(jl_binding_partition_type->types, 3, jl_binding_partition_type); diff --git a/src/julia.h b/src/julia.h index a710192d5756c..bfb641d38374b 100644 --- a/src/julia.h +++ b/src/julia.h @@ -408,7 +408,7 @@ struct _jl_method_instance_t { } def; // pointer back to the context for this code jl_value_t *specTypes; // argument types this was specialized for jl_svec_t *sparam_vals; // static parameter values, indexed by def.method->sig - jl_array_t *backedges; // list of method-instances which call this method-instance; `invoke` records (invokesig, caller) pairs + jl_array_t *backedges; // list of code-instances which call this method-instance; `invoke` records (invokesig, caller) pairs _Atomic(struct _jl_code_instance_t*) cache; uint8_t cache_with_orig; // !cache_with_specTypes @@ -453,6 +453,7 @@ typedef struct _jl_code_instance_t { // - null, indicating that inference was not yet completed or did not succeed _Atomic(jl_value_t *) inferred; _Atomic(jl_debuginfo_t *) debuginfo; // stored information about edges from this object (set once, with a happens-before both source and invoke) + _Atomic(jl_svec_t *) edges; // forward edge info //TODO: uint8_t absolute_max; // whether true max world is unknown // purity results @@ -789,7 +790,7 @@ typedef struct _jl_methtable_t { _Atomic(jl_typemap_t*) cache; _Atomic(intptr_t) max_args; // max # of non-vararg arguments in a signature jl_module_t *module; // sometimes used for debug printing - jl_array_t *backedges; // (sig, caller::MethodInstance) pairs + jl_array_t *backedges; // (sig, caller::CodeInstance) pairs jl_mutex_t writelock; uint8_t offs; // 0, or 1 to skip splitting typemap on first (function) argument uint8_t frozen; // whether this accepts adding new methods diff --git a/src/julia_internal.h b/src/julia_internal.h index ade5940f30687..9a8750bbc2500 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -675,7 +675,7 @@ JL_DLLEXPORT jl_code_info_t *jl_gdbcodetyped1(jl_method_instance_t *mi, size_t w JL_DLLEXPORT jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *meth JL_PROPAGATES_ROOT, size_t world); JL_DLLEXPORT jl_code_instance_t *jl_get_method_inferred( jl_method_instance_t *mi JL_PROPAGATES_ROOT, jl_value_t *rettype, - size_t min_world, size_t max_world, jl_debuginfo_t *edges); + size_t min_world, size_t max_world, jl_debuginfo_t *di, jl_svec_t *edges); JL_DLLEXPORT jl_method_instance_t *jl_get_unspecialized(jl_method_t *def JL_PROPAGATES_ROOT); JL_DLLEXPORT void jl_read_codeinst_invoke(jl_code_instance_t *ci, uint8_t *specsigflags, jl_callptr_t *invoke, void **specptr, int waitcompile) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_method_instance_t *jl_method_match_to_mi(jl_method_match_t *match, size_t world, size_t min_valid, size_t max_valid, int mt_cache); @@ -687,7 +687,7 @@ JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst( jl_value_t *inferred_const, jl_value_t *inferred, int32_t const_flags, size_t min_world, size_t max_world, uint32_t effects, jl_value_t *analysis_results, - uint8_t relocatability, jl_debuginfo_t *edges /* , int absolute_max*/); + uint8_t relocatability, jl_debuginfo_t *di, jl_svec_t *edges /* , int absolute_max*/); JL_DLLEXPORT const char *jl_debuginfo_file(jl_debuginfo_t *debuginfo) JL_NOTSAFEPOINT; JL_DLLEXPORT const char *jl_debuginfo_file1(jl_debuginfo_t *debuginfo) JL_NOTSAFEPOINT; @@ -705,9 +705,9 @@ JL_DLLEXPORT jl_code_info_t *jl_new_code_info_uninit(void); JL_DLLEXPORT void jl_resolve_globals_in_ir(jl_array_t *stmts, jl_module_t *m, jl_svec_t *sparam_vals, int binding_effects); -int get_next_edge(jl_array_t *list, int i, jl_value_t** invokesig, jl_method_instance_t **caller) JL_NOTSAFEPOINT; -int set_next_edge(jl_array_t *list, int i, jl_value_t *invokesig, jl_method_instance_t *caller); -void push_edge(jl_array_t *list, jl_value_t *invokesig, jl_method_instance_t *caller); +int get_next_edge(jl_array_t *list, int i, jl_value_t** invokesig, jl_code_instance_t **caller) JL_NOTSAFEPOINT; +int set_next_edge(jl_array_t *list, int i, jl_value_t *invokesig, jl_code_instance_t *caller); +void push_edge(jl_array_t *list, jl_value_t *invokesig, jl_code_instance_t *caller); JL_DLLEXPORT void jl_add_method_root(jl_method_t *m, jl_module_t *mod, jl_value_t* root); void jl_append_method_roots(jl_method_t *m, uint64_t modid, jl_array_t* roots); @@ -727,6 +727,7 @@ JL_DLLEXPORT void jl_typeassert(jl_value_t *x, jl_value_t *t); #define JL_CALLABLE(name) \ JL_DLLEXPORT jl_value_t *name(jl_value_t *F, jl_value_t **args, uint32_t nargs) +JL_CALLABLE(jl_f_svec); JL_CALLABLE(jl_f_tuple); JL_CALLABLE(jl_f_intrinsic_call); JL_CALLABLE(jl_f_opaque_closure_call); @@ -1185,8 +1186,8 @@ JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_methtable_t *mt JL_PROPAGATES_RO JL_DLLEXPORT jl_method_instance_t *jl_specializations_get_linfo( jl_method_t *m JL_PROPAGATES_ROOT, jl_value_t *type, jl_svec_t *sparams); jl_method_instance_t *jl_specializations_get_or_insert(jl_method_instance_t *mi_ins); -JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, jl_value_t *invokesig, jl_method_instance_t *caller); -JL_DLLEXPORT void jl_method_table_add_backedge(jl_methtable_t *mt, jl_value_t *typ, jl_value_t *caller); +JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, jl_value_t *invokesig, jl_code_instance_t *caller); +JL_DLLEXPORT void jl_method_table_add_backedge(jl_methtable_t *mt, jl_value_t *typ, jl_code_instance_t *caller); JL_DLLEXPORT void jl_mi_cache_insert(jl_method_instance_t *mi JL_ROOTING_ARGUMENT, jl_code_instance_t *ci JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED); JL_DLLEXPORT int jl_mi_try_insert(jl_method_instance_t *mi JL_ROOTING_ARGUMENT, diff --git a/src/method.c b/src/method.c index 629816319b334..8e3bb7d0060b7 100644 --- a/src/method.c +++ b/src/method.c @@ -650,10 +650,10 @@ JL_DLLEXPORT jl_code_info_t *jl_new_code_info_uninit(void) src->slotnames = NULL; src->slottypes = jl_nothing; src->rettype = (jl_value_t*)jl_any_type; + src->edges = (jl_value_t*)jl_emptysvec; src->parent = (jl_method_instance_t*)jl_nothing; src->min_world = 1; src->max_world = ~(size_t)0; - src->edges = jl_nothing; src->propagate_inbounds = 0; src->has_fcall = 0; src->nospecializeinfer = 0; @@ -754,9 +754,10 @@ JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *mi, size_t assert(jl_is_method(def)); jl_code_info_t *func = NULL; jl_value_t *ex = NULL; + jl_value_t *kind = NULL; jl_code_info_t *uninferred = NULL; jl_code_instance_t *ci = NULL; - JL_GC_PUSH4(&ex, &func, &uninferred, &ci); + JL_GC_PUSH5(&ex, &func, &uninferred, &ci, &kind); jl_task_t *ct = jl_current_task; int last_lineno = jl_lineno; int last_in = ct->ptls->in_pure_callback; @@ -792,6 +793,7 @@ JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *mi, size_t // but currently our isva determination is non-syntactic func->isva = def->isva; } + ex = NULL; // If this generated function has an opaque closure, cache it for // correctness of method identity. In particular, other methods that call @@ -815,7 +817,7 @@ JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *mi, size_t } } - if (func->edges == jl_nothing && func->max_world == ~(size_t)0) { + if ((func->edges == jl_nothing || func->edges == (jl_value_t*)jl_emptysvec) && func->max_world == ~(size_t)0) { if (func->min_world != 1) { jl_error("Generated function result with `edges == nothing` and `max_world == typemax(UInt)` must have `min_world == 1`"); } @@ -824,27 +826,41 @@ JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *mi, size_t if (cache || needs_cache_for_correctness) { uninferred = (jl_code_info_t*)jl_copy_ast((jl_value_t*)func); ci = jl_new_codeinst_for_uninferred(mi, uninferred); - - if (uninferred->edges != jl_nothing) { - // N.B.: This needs to match `store_backedges` on the julia side - jl_array_t *edges = (jl_array_t*)uninferred->edges; - for (size_t i = 0; i < jl_array_len(edges); ++i) { - jl_value_t *kind = jl_array_ptr_ref(edges, i); - if (jl_is_method_instance(kind)) { - jl_method_instance_add_backedge((jl_method_instance_t*)kind, jl_nothing, mi); - } else if (jl_is_mtable(kind)) { - jl_method_table_add_backedge((jl_methtable_t*)kind, jl_array_ptr_ref(edges, ++i), (jl_value_t*)mi); - } else { - jl_method_instance_add_backedge((jl_method_instance_t*)jl_array_ptr_ref(edges, ++i), kind, mi); - } - } - } - jl_code_instance_t *cached_ci = jl_cache_uninferred(mi, cache_ci, world, ci); if (cached_ci != ci) { func = (jl_code_info_t*)jl_copy_ast(jl_atomic_load_relaxed(&cached_ci->inferred)); assert(jl_is_code_info(func)); } + else if (uninferred->edges != jl_nothing) { + // N.B.: This needs to match `store_backedges` on the julia side + jl_value_t *edges = uninferred->edges; + size_t l; + jl_value_t **data; + if (jl_is_svec(edges)) { + l = jl_svec_len(edges); + data = jl_svec_data(edges); + } + else { + l = jl_array_dim0(edges); + data = jl_array_data(edges, jl_value_t*); + } + for (size_t i = 0; i < l; ) { + kind = data[i++]; + if (jl_is_method_instance(kind)) { + jl_method_instance_add_backedge((jl_method_instance_t*)kind, jl_nothing, ci); + } + else if (jl_is_mtable(kind)) { + assert(i < l); + ex = data[i++]; + jl_method_table_add_backedge((jl_methtable_t*)kind, ex, ci); + } + else { + assert(i < l); + ex = data[i++]; + jl_method_instance_add_backedge((jl_method_instance_t*)ex, kind, ci); + } + } + } if (cache) *cache = cached_ci; } @@ -1056,27 +1072,27 @@ JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t *module) // it will be the signature supplied in an `invoke` call. // If you don't need `invokesig`, you can set it to NULL on input. // Initialize iteration with `i = 0`. Returns `i` for the next backedge to be extracted. -int get_next_edge(jl_array_t *list, int i, jl_value_t** invokesig, jl_method_instance_t **caller) JL_NOTSAFEPOINT +int get_next_edge(jl_array_t *list, int i, jl_value_t** invokesig, jl_code_instance_t **caller) JL_NOTSAFEPOINT { jl_value_t *item = jl_array_ptr_ref(list, i); - if (jl_is_method_instance(item)) { - // Not an `invoke` call, it's just the MethodInstance + if (jl_is_code_instance(item)) { + // Not an `invoke` call, it's just the CodeInstance if (invokesig != NULL) *invokesig = NULL; - *caller = (jl_method_instance_t*)item; + *caller = (jl_code_instance_t*)item; return i + 1; } assert(jl_is_type(item)); // An `invoke` call, it's a (sig, MethodInstance) pair if (invokesig != NULL) *invokesig = item; - *caller = (jl_method_instance_t*)jl_array_ptr_ref(list, i + 1); + *caller = (jl_code_instance_t*)jl_array_ptr_ref(list, i + 1); if (*caller) - assert(jl_is_method_instance(*caller)); + assert(jl_is_code_instance(*caller)); return i + 2; } -int set_next_edge(jl_array_t *list, int i, jl_value_t *invokesig, jl_method_instance_t *caller) +int set_next_edge(jl_array_t *list, int i, jl_value_t *invokesig, jl_code_instance_t *caller) { if (invokesig) jl_array_ptr_set(list, i++, invokesig); @@ -1084,7 +1100,7 @@ int set_next_edge(jl_array_t *list, int i, jl_value_t *invokesig, jl_method_inst return i; } -void push_edge(jl_array_t *list, jl_value_t *invokesig, jl_method_instance_t *caller) +void push_edge(jl_array_t *list, jl_value_t *invokesig, jl_code_instance_t *caller) { if (invokesig) jl_array_ptr_1d_push(list, invokesig); diff --git a/src/opaque_closure.c b/src/opaque_closure.c index 9fe36f32d2030..65773f88a3951 100644 --- a/src/opaque_closure.c +++ b/src/opaque_closure.c @@ -113,8 +113,8 @@ static jl_opaque_closure_t *new_opaque_closure(jl_tupletype_t *argt, jl_value_t if (specptr == NULL) { jl_method_instance_t *mi_generic = jl_specializations_get_linfo(jl_opaque_closure_method, sigtype, jl_emptysvec); - // OC wrapper methods are not world dependent - ci = jl_get_method_inferred(mi_generic, selected_rt, 1, ~(size_t)0, NULL); + // OC wrapper methods are not world dependent and have no edges or other info + ci = jl_get_method_inferred(mi_generic, selected_rt, 1, ~(size_t)0, NULL, NULL); if (!jl_atomic_load_acquire(&ci->invoke)) jl_compile_codeinst(ci); specptr = jl_atomic_load_relaxed(&ci->specptr.fptr); @@ -145,7 +145,8 @@ JL_DLLEXPORT jl_opaque_closure_t *jl_new_opaque_closure_from_code_info(jl_tuplet { jl_value_t *root = NULL, *sigtype = NULL; jl_code_instance_t *inst = NULL; - JL_GC_PUSH3(&root, &sigtype, &inst); + jl_svec_t *edges = NULL; + JL_GC_PUSH4(&root, &sigtype, &inst, &edges); root = jl_box_long(lineno); root = jl_new_struct(jl_linenumbernode_type, root, file); jl_method_t *meth = jl_make_opaque_closure_method(mod, jl_nothing, nargs, root, ci, isva, isinferred); @@ -159,8 +160,11 @@ JL_DLLEXPORT jl_opaque_closure_t *jl_new_opaque_closure_from_code_info(jl_tuplet jl_value_t *argslotty = jl_array_ptr_ref(ci->slottypes, 0); sigtype = jl_argtype_with_function_type(argslotty, (jl_value_t*)argt); jl_method_instance_t *mi = jl_specializations_get_linfo((jl_method_t*)root, sigtype, jl_emptysvec); + edges = (jl_svec_t*)ci->edges; + if (!jl_is_svec(edges)) + edges = jl_emptysvec; // OC doesn't really have edges, so just drop them for now inst = jl_new_codeinst(mi, jl_nothing, rt_ub, (jl_value_t*)jl_any_type, NULL, (jl_value_t*)ci, - 0, world, world, 0, jl_nothing, 0, ci->debuginfo); + 0, world, world, 0, jl_nothing, 0, ci->debuginfo, edges); jl_mi_cache_insert(mi, inst); } diff --git a/src/serialize.h b/src/serialize.h index 3d3eb4df5e862..3aa82a1d09a9b 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -23,50 +23,52 @@ extern "C" { #define TAG_LONG_PHINODE 15 #define TAG_LONG_PHICNODE 16 #define TAG_METHODROOT 17 -#define TAG_STRING 18 -#define TAG_SHORT_INT64 19 -#define TAG_SHORT_GENERAL 20 -#define TAG_CNULL 21 -#define TAG_ARRAY1D 22 -#define TAG_SINGLETON 23 -#define TAG_MODULE 24 -#define TAG_TVAR 25 -#define TAG_METHOD_INSTANCE 26 -#define TAG_METHOD 27 -#define TAG_CODE_INSTANCE 28 -#define TAG_COMMONSYM 29 -#define TAG_NEARBYGLOBAL 30 -#define TAG_GLOBALREF 31 -#define TAG_CORE 32 -#define TAG_BASE 33 -#define TAG_BITYPENAME 34 -#define TAG_NEARBYMODULE 35 -#define TAG_INT32 36 -#define TAG_INT64 37 -#define TAG_UINT8 38 -#define TAG_VECTORTY 39 -#define TAG_PTRTY 40 -#define TAG_LONG_SSAVALUE 41 -#define TAG_LONG_METHODROOT 42 -#define TAG_SHORTER_INT64 43 -#define TAG_SHORT_INT32 44 -#define TAG_CALL1 45 -#define TAG_CALL2 46 -#define TAG_SHORT_BACKREF 47 -#define TAG_BACKREF 48 -#define TAG_UNIONALL 49 -#define TAG_GOTONODE 50 -#define TAG_QUOTENODE 51 -#define TAG_GENERAL 52 -#define TAG_GOTOIFNOT 53 -#define TAG_RETURNNODE 54 -#define TAG_ARGUMENT 55 -#define TAG_RELOC_METHODROOT 56 -#define TAG_BINDING 57 -#define TAG_MEMORYT 58 -#define TAG_ENTERNODE 59 - -#define LAST_TAG 59 +#define TAG_EDGE 18 +#define TAG_STRING 19 +#define TAG_SHORT_INT64 20 +#define TAG_SHORT_GENERAL 21 +#define TAG_CNULL 22 +#define TAG_ARRAY1D 23 +#define TAG_SINGLETON 24 +#define TAG_MODULE 25 +#define TAG_TVAR 26 +#define TAG_METHOD_INSTANCE 27 +#define TAG_METHOD 28 +#define TAG_CODE_INSTANCE 29 +#define TAG_COMMONSYM 30 +#define TAG_NEARBYGLOBAL 31 +#define TAG_GLOBALREF 32 +#define TAG_CORE 33 +#define TAG_BASE 34 +#define TAG_BITYPENAME 35 +#define TAG_NEARBYMODULE 36 +#define TAG_INT32 37 +#define TAG_INT64 38 +#define TAG_UINT8 39 +#define TAG_VECTORTY 40 +#define TAG_PTRTY 41 +#define TAG_LONG_SSAVALUE 42 +#define TAG_LONG_METHODROOT 43 +#define TAG_LONG_EDGE 44 +#define TAG_SHORTER_INT64 45 +#define TAG_SHORT_INT32 46 +#define TAG_CALL1 47 +#define TAG_CALL2 48 +#define TAG_SHORT_BACKREF 49 +#define TAG_BACKREF 50 +#define TAG_UNIONALL 51 +#define TAG_GOTONODE 52 +#define TAG_QUOTENODE 53 +#define TAG_GENERAL 54 +#define TAG_GOTOIFNOT 55 +#define TAG_RETURNNODE 56 +#define TAG_ARGUMENT 57 +#define TAG_RELOC_METHODROOT 58 +#define TAG_BINDING 59 +#define TAG_MEMORYT 60 +#define TAG_ENTERNODE 61 + +#define LAST_TAG 61 #define write_uint8(s, n) ios_putc((n), (s)) #define read_uint8(s) ((uint8_t)ios_getc((s))) diff --git a/src/staticdata.c b/src/staticdata.c index af3477a25128e..0d609db03aebc 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -548,9 +548,6 @@ typedef struct { jl_array_t *link_ids_gvars; jl_array_t *link_ids_external_fnvars; jl_ptls_t ptls; - // Set (implemented has a hasmap of MethodInstances to themselves) of which MethodInstances have (forward) edges - // to other MethodInstances. - htable_t callers_with_edges; jl_image_t *image; int8_t incremental; } jl_serializer_state; @@ -1767,6 +1764,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED if (s->incremental) { if (jl_atomic_load_relaxed(&ci->max_world) == ~(size_t)0) { if (jl_atomic_load_relaxed(&newci->min_world) > 1) { + //assert(jl_atomic_load_relaxed(&ci->edges) != jl_emptysvec); // some code (such as !==) might add a method lookup restriction but not keep the edges jl_atomic_store_release(&newci->min_world, ~(size_t)0); jl_atomic_store_release(&newci->max_world, WORLD_AGE_REVALIDATION_SENTINEL); arraylist_push(&s->fixup_objs, (void*)reloc_offset); @@ -2801,25 +2799,21 @@ JL_DLLEXPORT jl_value_t *jl_as_global_root(jl_value_t *val, int insert) } static void jl_prepare_serialization_data(jl_array_t *mod_array, jl_array_t *newly_inferred, uint64_t worklist_key, - /* outputs */ jl_array_t **extext_methods, jl_array_t **new_ext_cis, - jl_array_t **method_roots_list, jl_array_t **ext_targets, jl_array_t **edges) + /* outputs */ jl_array_t **extext_methods JL_REQUIRE_ROOTED_SLOT, + jl_array_t **new_ext_cis JL_REQUIRE_ROOTED_SLOT, + jl_array_t **method_roots_list JL_REQUIRE_ROOTED_SLOT, + jl_array_t **edges JL_REQUIRE_ROOTED_SLOT) { // extext_methods: [method1, ...], worklist-owned "extending external" methods added to functions owned by modules outside the worklist - // ext_targets: [invokesig1, callee1, matches1, ...] non-worklist callees of worklist-owned methods - // ordinary dispatch: invokesig=NULL, callee is MethodInstance - // `invoke` dispatch: invokesig is signature, callee is MethodInstance - // abstract call: callee is signature - // edges: [caller1, ext_targets_indexes1, ...] for worklist-owned methods calling external methods - assert(edges_map == NULL); + // edges: [caller1, ext_targets, ...] for worklist-owned methods calling external methods // Save the inferred code from newly inferred, external methods *new_ext_cis = queue_external_cis(newly_inferred); // Collect method extensions and edges data - JL_GC_PUSH1(&edges_map); - if (edges) - edges_map = jl_alloc_memory_any(0); *extext_methods = jl_alloc_vec_any(0); + internal_methods = jl_alloc_vec_any(0); + JL_GC_PUSH1(&internal_methods); jl_collect_methtable_from_mod(jl_type_type_mt, *extext_methods); jl_collect_methtable_from_mod(jl_nonfunction_mt, *extext_methods); size_t i, len = jl_array_len(mod_array); @@ -2832,18 +2826,14 @@ static void jl_prepare_serialization_data(jl_array_t *mod_array, jl_array_t *new if (edges) { size_t world = jl_atomic_load_acquire(&jl_world_counter); - jl_collect_missing_backedges(jl_type_type_mt); - jl_collect_missing_backedges(jl_nonfunction_mt); - // jl_collect_extext_methods_from_mod and jl_collect_missing_backedges also accumulate data in callers_with_edges. - // Process this to extract `edges` and `ext_targets`. - *ext_targets = jl_alloc_vec_any(0); - *edges = jl_alloc_vec_any(0); + // Extract `new_ext_cis` and `edges` now (from info prepared by jl_collect_methcache_from_mod) *method_roots_list = jl_alloc_vec_any(0); // Collect the new method roots for external specializations jl_collect_new_roots(&relocatable_ext_cis, *method_roots_list, *new_ext_cis, worklist_key); - jl_collect_edges(*edges, *ext_targets, *new_ext_cis, world); + *edges = jl_alloc_vec_any(0); + jl_collect_internal_cis(*edges, world); } - assert(edges_map == NULL); // jl_collect_edges clears this when done + internal_methods = NULL; JL_GC_POP(); } @@ -2852,7 +2842,7 @@ static void jl_prepare_serialization_data(jl_array_t *mod_array, jl_array_t *new static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_array_t *worklist, jl_array_t *extext_methods, jl_array_t *new_ext_cis, jl_array_t *method_roots_list, - jl_array_t *ext_targets, jl_array_t *edges) JL_GC_DISABLED + jl_array_t *edges) JL_GC_DISABLED { htable_new(&field_replace, 0); htable_new(&bits_replace, 0); @@ -2972,7 +2962,6 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, s.link_ids_gctags = jl_alloc_array_1d(jl_array_int32_type, 0); s.link_ids_gvars = jl_alloc_array_1d(jl_array_int32_type, 0); s.link_ids_external_fnvars = jl_alloc_array_1d(jl_array_int32_type, 0); - htable_new(&s.callers_with_edges, 0); jl_value_t **const*const tags = get_tags(); // worklist == NULL ? get_tags() : NULL; @@ -3012,12 +3001,9 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, // Queue the worklist itself as the first item we serialize jl_queue_for_serialization(&s, worklist); jl_queue_for_serialization(&s, jl_module_init_order); - // Classify the CodeInstances with respect to their need for validation - classify_callers(&s.callers_with_edges, edges); } // step 1.1: as needed, serialize the data needed for insertion into the running system if (extext_methods) { - assert(ext_targets); assert(edges); // Queue method extensions jl_queue_for_serialization(&s, extext_methods); @@ -3026,7 +3012,6 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, // Queue the new roots jl_queue_for_serialization(&s, method_roots_list); // Queue the edges - jl_queue_for_serialization(&s, ext_targets); jl_queue_for_serialization(&s, edges); } jl_serialize_reachable(&s); @@ -3117,7 +3102,6 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, ); jl_exit(1); } - htable_free(&s.callers_with_edges); // step 3: combine all of the sections into one file assert(ios_pos(f) % JL_CACHE_BYTE_ALIGNMENT == 0); @@ -3206,7 +3190,6 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_write_value(&s, extext_methods); jl_write_value(&s, new_ext_cis); jl_write_value(&s, method_roots_list); - jl_write_value(&s, ext_targets); jl_write_value(&s, edges); } write_uint32(f, jl_array_len(s.link_ids_gctags)); @@ -3297,18 +3280,18 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli } jl_array_t *mod_array = NULL, *extext_methods = NULL, *new_ext_cis = NULL; - jl_array_t *method_roots_list = NULL, *ext_targets = NULL, *edges = NULL; + jl_array_t *method_roots_list = NULL, *edges = NULL; int64_t checksumpos = 0; int64_t checksumpos_ff = 0; int64_t datastartpos = 0; - JL_GC_PUSH6(&mod_array, &extext_methods, &new_ext_cis, &method_roots_list, &ext_targets, &edges); + JL_GC_PUSH5(&mod_array, &extext_methods, &new_ext_cis, &method_roots_list, &edges); if (worklist) { mod_array = jl_get_loaded_modules(); // __toplevel__ modules loaded in this session (from Base.loaded_modules_array) // Generate _native_data` if (_native_data != NULL) { jl_prepare_serialization_data(mod_array, newly_inferred, jl_worklist_key(worklist), - &extext_methods, &new_ext_cis, NULL, NULL, NULL); + &extext_methods, &new_ext_cis, NULL, NULL); jl_precompile_toplevel_module = (jl_module_t*)jl_array_ptr_ref(worklist, jl_array_len(worklist)-1); *_native_data = jl_precompile_worklist(worklist, extext_methods, new_ext_cis); jl_precompile_toplevel_module = NULL; @@ -3341,7 +3324,7 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli if (worklist) { htable_new(&relocatable_ext_cis, 0); jl_prepare_serialization_data(mod_array, newly_inferred, jl_worklist_key(worklist), - &extext_methods, &new_ext_cis, &method_roots_list, &ext_targets, &edges); + &extext_methods, &new_ext_cis, &method_roots_list, &edges); if (!emit_split) { write_int32(f, 0); // No clone_targets write_padding(f, LLT_ALIGN(ios_pos(f), JL_CACHE_BYTE_ALIGNMENT) - ios_pos(f)); @@ -3353,7 +3336,7 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli } if (_native_data != NULL) native_functions = *_native_data; - jl_save_system_image_to_stream(ff, mod_array, worklist, extext_methods, new_ext_cis, method_roots_list, ext_targets, edges); + jl_save_system_image_to_stream(ff, mod_array, worklist, extext_methods, new_ext_cis, method_roots_list, edges); if (_native_data != NULL) native_functions = NULL; if (worklist) @@ -3443,7 +3426,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl /* outputs */ jl_array_t **restored, jl_array_t **init_order, jl_array_t **extext_methods, jl_array_t **internal_methods, jl_array_t **new_ext_cis, jl_array_t **method_roots_list, - jl_array_t **ext_targets, jl_array_t **edges, + jl_array_t **edges, char **base, arraylist_t *ccallable_list, pkgcachesizes *cachesizes) JL_GC_DISABLED { jl_task_t *ct = jl_current_task; @@ -3514,7 +3497,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl assert(!ios_eof(f)); s.s = f; uintptr_t offset_restored = 0, offset_init_order = 0, offset_extext_methods = 0, offset_new_ext_cis = 0, offset_method_roots_list = 0; - uintptr_t offset_ext_targets = 0, offset_edges = 0; + uintptr_t offset_edges = 0; if (!s.incremental) { size_t i; for (i = 0; tags[i] != NULL; i++) { @@ -3547,7 +3530,6 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl offset_extext_methods = jl_read_offset(&s); offset_new_ext_cis = jl_read_offset(&s); offset_method_roots_list = jl_read_offset(&s); - offset_ext_targets = jl_read_offset(&s); offset_edges = jl_read_offset(&s); } s.buildid_depmods_idxs = depmod_to_imageidx(depmods); @@ -3574,13 +3556,12 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl uint32_t external_fns_begin = read_uint32(f); jl_read_arraylist(s.s, ccallable_list ? ccallable_list : &s.ccallable_list); if (s.incremental) { - assert(restored && init_order && extext_methods && internal_methods && new_ext_cis && method_roots_list && ext_targets && edges); + assert(restored && init_order && extext_methods && internal_methods && new_ext_cis && method_roots_list && edges); *restored = (jl_array_t*)jl_delayed_reloc(&s, offset_restored); *init_order = (jl_array_t*)jl_delayed_reloc(&s, offset_init_order); *extext_methods = (jl_array_t*)jl_delayed_reloc(&s, offset_extext_methods); *new_ext_cis = (jl_array_t*)jl_delayed_reloc(&s, offset_new_ext_cis); *method_roots_list = (jl_array_t*)jl_delayed_reloc(&s, offset_method_roots_list); - *ext_targets = (jl_array_t*)jl_delayed_reloc(&s, offset_ext_targets); *edges = (jl_array_t*)jl_delayed_reloc(&s, offset_edges); *internal_methods = jl_alloc_vec_any(0); } @@ -4021,9 +4002,9 @@ static jl_value_t *jl_restore_package_image_from_stream(void* pkgimage_handle, i arraylist_t ccallable_list; jl_value_t *restored = NULL; - jl_array_t *init_order = NULL, *extext_methods = NULL, *internal_methods = NULL, *new_ext_cis = NULL, *method_roots_list = NULL, *ext_targets = NULL, *edges = NULL; + jl_array_t *init_order = NULL, *extext_methods = NULL, *internal_methods = NULL, *new_ext_cis = NULL, *method_roots_list = NULL, *edges = NULL; jl_svec_t *cachesizes_sv = NULL; - JL_GC_PUSH9(&restored, &init_order, &extext_methods, &internal_methods, &new_ext_cis, &method_roots_list, &ext_targets, &edges, &cachesizes_sv); + JL_GC_PUSH8(&restored, &init_order, &extext_methods, &internal_methods, &new_ext_cis, &method_roots_list, &edges, &cachesizes_sv); { // make a permanent in-memory copy of f (excluding the header) ios_bufmode(f, bm_none); @@ -4048,7 +4029,7 @@ static jl_value_t *jl_restore_package_image_from_stream(void* pkgimage_handle, i ios_static_buffer(f, sysimg, len); pkgcachesizes cachesizes; jl_restore_system_image_from_stream_(f, image, depmods, checksum, (jl_array_t**)&restored, &init_order, &extext_methods, &internal_methods, &new_ext_cis, &method_roots_list, - &ext_targets, &edges, &base, &ccallable_list, &cachesizes); + &edges, &base, &ccallable_list, &cachesizes); JL_SIGATOMIC_END(); // No special processing of `new_ext_cis` is required because recaching handled it @@ -4062,7 +4043,7 @@ static jl_value_t *jl_restore_package_image_from_stream(void* pkgimage_handle, i // allow users to start running in this updated world jl_atomic_store_release(&jl_world_counter, world); // but one of those immediate users is going to be our cache updates - jl_insert_backedges((jl_array_t*)edges, (jl_array_t*)ext_targets, (jl_array_t*)new_ext_cis, world); // restore external backedges (needs to be last) + jl_insert_backedges((jl_array_t*)edges, (jl_array_t*)new_ext_cis, world); // restore external backedges (needs to be last) // now permit more methods to be added again JL_UNLOCK(&world_counter_lock); // reinit ccallables @@ -4078,9 +4059,9 @@ static jl_value_t *jl_restore_package_image_from_stream(void* pkgimage_handle, i jl_svecset(cachesizes_sv, 4, jl_box_long(cachesizes.reloclist)); jl_svecset(cachesizes_sv, 5, jl_box_long(cachesizes.gvarlist)); jl_svecset(cachesizes_sv, 6, jl_box_long(cachesizes.fptrlist)); - restored = (jl_value_t*)jl_svec(8, restored, init_order, extext_methods, + restored = (jl_value_t*)jl_svec(7, restored, init_order, extext_methods, new_ext_cis ? (jl_value_t*)new_ext_cis : jl_nothing, - method_roots_list, ext_targets, edges, cachesizes_sv); + method_roots_list, edges, cachesizes_sv); } else { restored = (jl_value_t*)jl_svec(2, restored, init_order); @@ -4095,7 +4076,7 @@ static jl_value_t *jl_restore_package_image_from_stream(void* pkgimage_handle, i static void jl_restore_system_image_from_stream(ios_t *f, jl_image_t *image, uint32_t checksum) { JL_TIMING(LOAD_IMAGE, LOAD_Sysimg); - jl_restore_system_image_from_stream_(f, image, NULL, checksum | ((uint64_t)0xfdfcfbfa << 32), NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); + jl_restore_system_image_from_stream_(f, image, NULL, checksum | ((uint64_t)0xfdfcfbfa << 32), NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(void* pkgimage_handle, const char *buf, jl_image_t *image, size_t sz, jl_array_t *depmods, int completeinfo, const char *pkgname, int needs_permalloc) diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index 9a7653972ea7c..b69c1edb5429b 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -1,5 +1,5 @@ // inverse of backedges graph (caller=>callees hash) -jl_genericmemory_t *edges_map JL_GLOBALLY_ROOTED = NULL; // rooted for the duration of our uses of this +jl_array_t *internal_methods JL_GLOBALLY_ROOTED = NULL; // rooted for the duration of our uses of this static void write_float64(ios_t *s, double x) JL_NOTSAFEPOINT { @@ -185,9 +185,10 @@ static int has_backedge_to_worklist(jl_method_instance_t *mi, htable_t *visited, size_t i = 0, n = jl_array_nrows(mi->backedges); int cycle = depth; while (i < n) { - jl_method_instance_t *be; + jl_code_instance_t *be; i = get_next_edge(mi->backedges, i, NULL, &be); - int child_found = has_backedge_to_worklist(be, visited, stack); + JL_GC_PROMISE_ROOTED(be); // get_next_edge propagates the edge for us here + int child_found = has_backedge_to_worklist(be->def, visited, stack); if (child_found == 1 || child_found == 2) { // found what we were looking for, so terminate early found = 1; @@ -250,7 +251,7 @@ static jl_array_t *queue_external_cis(jl_array_t *list) continue; jl_method_instance_t *mi = ci->def; jl_method_t *m = mi->def.method; - if (jl_atomic_load_relaxed(&ci->inferred) && jl_is_method(m) && jl_object_in_image((jl_value_t*)m->module)) { + if (ci->owner == jl_nothing && jl_atomic_load_relaxed(&ci->inferred) && jl_is_method(m) && jl_object_in_image((jl_value_t*)m->module)) { int found = has_backedge_to_worklist(mi, &visited, &stack); assert(found == 0 || found == 1 || found == 2); assert(stack.len == 0); @@ -318,82 +319,19 @@ static void jl_collect_new_roots(htable_t *relocatable_ext_cis, jl_array_t *root htable_free(&mset); } -// Create the forward-edge map (caller => callees) -// the intent of these functions is to invert the backedges tree -// for anything that points to a method not part of the worklist -// -// from MethodTables -static void jl_collect_missing_backedges(jl_methtable_t *mt) -{ - jl_array_t *backedges = mt->backedges; - if (backedges) { - size_t i, l = jl_array_nrows(backedges); - for (i = 1; i < l; i += 2) { - jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(backedges, i); - jl_value_t *missing_callee = jl_array_ptr_ref(backedges, i - 1); // signature of abstract callee - jl_array_t *edges = (jl_array_t*)jl_eqtable_get(edges_map, (jl_value_t*)caller, NULL); - if (edges == NULL) { - edges = jl_alloc_vec_any(0); - JL_GC_PUSH1(&edges); - edges_map = jl_eqtable_put(edges_map, (jl_value_t*)caller, (jl_value_t*)edges, NULL); - JL_GC_POP(); - } - jl_array_ptr_1d_push(edges, NULL); - jl_array_ptr_1d_push(edges, missing_callee); - } - } -} - - -// from MethodInstances -static void collect_backedges(jl_method_instance_t *callee, int internal) -{ - jl_array_t *backedges = callee->backedges; - if (backedges) { - size_t i = 0, l = jl_array_nrows(backedges); - while (i < l) { - jl_value_t *invokeTypes; - jl_method_instance_t *caller; - i = get_next_edge(backedges, i, &invokeTypes, &caller); - jl_array_t *edges = (jl_array_t*)jl_eqtable_get(edges_map, (jl_value_t*)caller, NULL); - if (edges == NULL) { - edges = jl_alloc_vec_any(0); - JL_GC_PUSH1(&edges); - edges_map = jl_eqtable_put(edges_map, (jl_value_t*)caller, (jl_value_t*)edges, NULL); - JL_GC_POP(); - } - jl_array_ptr_1d_push(edges, invokeTypes); - jl_array_ptr_1d_push(edges, (jl_value_t*)callee); - } - } -} - -// For functions owned by modules not on the worklist, call this on each method. +// For every method: // - if the method is owned by a worklist module, add it to the list of things to be -// fully serialized -// - Collect all backedges (may be needed later when we invert this list). +// verified on reloading +// - if the method is extext, record that it needs to be reinserted later in the method table static int jl_collect_methcache_from_mod(jl_typemap_entry_t *ml, void *closure) { jl_array_t *s = (jl_array_t*)closure; jl_method_t *m = ml->func.method; - if (s && !jl_object_in_image((jl_value_t*)m->module)) { - jl_array_ptr_1d_push(s, (jl_value_t*)m); - } - if (edges_map == NULL) - return 1; - jl_value_t *specializations = jl_atomic_load_relaxed(&m->specializations); - if (!jl_is_svec(specializations)) { - jl_method_instance_t *callee = (jl_method_instance_t*)specializations; - collect_backedges(callee, !s); - } - else { - size_t i, l = jl_svec_len(specializations); - for (i = 0; i < l; i++) { - jl_method_instance_t *callee = (jl_method_instance_t*)jl_svecref(specializations, i); - if ((jl_value_t*)callee != jl_nothing) - collect_backedges(callee, !s); - } + if (!jl_object_in_image((jl_value_t*)m->module)) { + jl_array_ptr_1d_push(internal_methods, (jl_value_t*)m); + if (s) + jl_array_ptr_1d_push(s, (jl_value_t*)m); // extext } return 1; } @@ -401,10 +339,8 @@ static int jl_collect_methcache_from_mod(jl_typemap_entry_t *ml, void *closure) static int jl_collect_methtable_from_mod(jl_methtable_t *mt, void *env) { if (!jl_object_in_image((jl_value_t*)mt)) - env = NULL; // do not collect any methods from here + env = NULL; // mark internal, not extext jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), jl_collect_methcache_from_mod, env); - if (env && edges_map) - jl_collect_missing_backedges(mt); return 1; } @@ -416,169 +352,38 @@ static void jl_collect_extext_methods_from_mod(jl_array_t *s, jl_module_t *m) foreach_mtable_in_module(m, jl_collect_methtable_from_mod, s); } -static void jl_record_edges(jl_method_instance_t *caller, arraylist_t *wq, jl_array_t *edges) +static void jl_record_edges(jl_method_instance_t *caller, jl_array_t *edges) { - jl_array_t *callees = NULL; - JL_GC_PUSH2(&caller, &callees); - callees = (jl_array_t*)jl_eqtable_pop(edges_map, (jl_value_t*)caller, NULL, NULL); - if (callees != NULL) { - jl_array_ptr_1d_push(edges, (jl_value_t*)caller); - jl_array_ptr_1d_push(edges, (jl_value_t*)callees); - size_t i, l = jl_array_nrows(callees); - for (i = 1; i < l; i += 2) { - jl_method_instance_t *c = (jl_method_instance_t*)jl_array_ptr_ref(callees, i); - if (c && jl_is_method_instance(c)) { - arraylist_push(wq, c); - } - } + jl_code_instance_t *ci = jl_atomic_load_relaxed(&caller->cache); + while (ci != NULL) { + if (jl_atomic_load_relaxed(&ci->edges) && + jl_atomic_load_relaxed(&ci->edges) != jl_emptysvec && + jl_atomic_load_relaxed(&ci->max_world) == ~(size_t)0) + jl_array_ptr_1d_push(edges, (jl_value_t*)ci); + ci = jl_atomic_load_relaxed(&ci->next); } - JL_GC_POP(); } - // Extract `edges` and `ext_targets` from `edges_map` -// `edges` = [caller1, targets_indexes1, ...], the list of methods and their edges -// `ext_targets` is [invokesig1, callee1, matches1, ...], the edges for each target -static void jl_collect_edges(jl_array_t *edges, jl_array_t *ext_targets, jl_array_t *external_cis, size_t world) +// `edges` = [caller1, ...], the list of codeinstances internal to methods +static void jl_collect_internal_cis(jl_array_t *edges, size_t world) { - htable_t external_mis; - htable_new(&external_mis, 0); - if (external_cis) { - for (size_t i = 0; i < jl_array_nrows(external_cis); i++) { - jl_code_instance_t *ci = (jl_code_instance_t*)jl_array_ptr_ref(external_cis, i); - jl_method_instance_t *mi = ci->def; - ptrhash_put(&external_mis, (void*)mi, (void*)mi); - } - } - arraylist_t wq; - arraylist_new(&wq, 0); - void **table = (void**) edges_map->ptr; // edges_map is caller => callees - size_t table_size = edges_map->length; - for (size_t i = 0; i < table_size; i += 2) { - assert(table == edges_map->ptr && table_size == edges_map->length && - "edges_map changed during iteration"); - jl_method_instance_t *caller = (jl_method_instance_t*)table[i]; - jl_array_t *callees = (jl_array_t*)table[i + 1]; - if (callees == NULL) - continue; - assert(jl_is_method_instance(caller) && jl_is_method(caller->def.method)); - if (!jl_object_in_image((jl_value_t*)caller->def.method->module) || - ptrhash_get(&external_mis, caller) != HT_NOTFOUND) { - jl_record_edges(caller, &wq, edges); + for (size_t i = 0; i < jl_array_nrows(internal_methods); i++) { + jl_method_t *m = (jl_method_t*)jl_array_ptr_ref(internal_methods, i); + jl_value_t *specializations = jl_atomic_load_relaxed(&m->specializations); + if (!jl_is_svec(specializations)) { + jl_method_instance_t *mi = (jl_method_instance_t*)specializations; + jl_record_edges(mi, edges); } - } - htable_free(&external_mis); - while (wq.len) { - jl_method_instance_t *caller = (jl_method_instance_t*)arraylist_pop(&wq); - jl_record_edges(caller, &wq, edges); - } - arraylist_free(&wq); - edges_map = NULL; - htable_t edges_map2; - htable_new(&edges_map2, 0); - htable_t edges_ids; - size_t l = edges ? jl_array_nrows(edges) : 0; - htable_new(&edges_ids, l); - for (size_t i = 0; i < l / 2; i++) { - jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(edges, i * 2); - void *target = (void*)((char*)HT_NOTFOUND + i + 1); - ptrhash_put(&edges_ids, (void*)caller, target); - } - // process target list to turn it into a memoized validity table - // and compute the old methods list, ready for serialization - jl_value_t *matches = NULL; - jl_array_t *callee_ids = NULL; - jl_value_t *sig = NULL; - JL_GC_PUSH3(&matches, &callee_ids, &sig); - for (size_t i = 0; i < l; i += 2) { - jl_array_t *callees = (jl_array_t*)jl_array_ptr_ref(edges, i + 1); - size_t l = jl_array_nrows(callees); - callee_ids = jl_alloc_array_1d(jl_array_int32_type, l + 1); - int32_t *idxs = jl_array_data(callee_ids, int32_t); - idxs[0] = 0; - size_t nt = 0; - for (size_t j = 0; j < l; j += 2) { - jl_value_t *invokeTypes = jl_array_ptr_ref(callees, j); - jl_value_t *callee = jl_array_ptr_ref(callees, j + 1); - assert(callee && "unsupported edge"); - - if (jl_is_method_instance(callee)) { - jl_methtable_t *mt = jl_method_get_table(((jl_method_instance_t*)callee)->def.method); - if (!jl_object_in_image((jl_value_t*)mt)) - continue; - } - - // (nullptr, c) => call - // (invokeTypes, c) => invoke - // (nullptr, invokeTypes) => missing call - // (invokeTypes, nullptr) => missing invoke (unused--inferred as Any) - void *target = ptrhash_get(&edges_map2, invokeTypes ? (void*)invokeTypes : (void*)callee); - if (target == HT_NOTFOUND) { - size_t min_valid = 0; - size_t max_valid = ~(size_t)0; - if (invokeTypes) { - assert(jl_is_method_instance(callee)); - jl_method_t *m = ((jl_method_instance_t*)callee)->def.method; - matches = (jl_value_t*)m; // valid because there is no method replacement permitted -#ifndef NDEBUG - jl_methtable_t *mt = jl_method_get_table(m); - if ((jl_value_t*)mt != jl_nothing) { - jl_value_t *matches = jl_gf_invoke_lookup_worlds(invokeTypes, (jl_value_t*)mt, world, &min_valid, &max_valid); - if (matches != jl_nothing) { - assert(m == ((jl_method_match_t*)matches)->method); - } - } -#endif - } - else { - if (jl_is_method_instance(callee)) { - jl_method_instance_t *mi = (jl_method_instance_t*)callee; - sig = jl_type_intersection(mi->def.method->sig, (jl_value_t*)mi->specTypes); - } - else { - sig = callee; - } - int ambig = 0; - matches = jl_matching_methods((jl_tupletype_t*)sig, jl_nothing, - INT32_MAX, 0, world, &min_valid, &max_valid, &ambig); - sig = NULL; - if (matches == jl_nothing) { - callee_ids = NULL; // invalid - break; - } - size_t k; - for (k = 0; k < jl_array_nrows(matches); k++) { - jl_method_match_t *match = (jl_method_match_t *)jl_array_ptr_ref(matches, k); - jl_array_ptr_set(matches, k, match->method); - } - } - jl_array_ptr_1d_push(ext_targets, invokeTypes); - jl_array_ptr_1d_push(ext_targets, callee); - jl_array_ptr_1d_push(ext_targets, matches); - target = (void*)((char*)HT_NOTFOUND + jl_array_nrows(ext_targets) / 3); - ptrhash_put(&edges_map2, (void*)callee, target); - } - idxs[++nt] = (char*)target - (char*)HT_NOTFOUND - 1; - } - jl_array_ptr_set(edges, i + 1, callee_ids); // swap callees for ids - if (!callee_ids) - continue; - idxs[0] = nt; - // record place of every method in edges - // add method edges to the callee_ids list - for (size_t j = 0; j < l; j += 2) { - jl_value_t *callee = jl_array_ptr_ref(callees, j + 1); - if (callee && jl_is_method_instance(callee)) { - void *target = ptrhash_get(&edges_ids, (void*)callee); - if (target != HT_NOTFOUND) { - idxs[++nt] = (char*)target - (char*)HT_NOTFOUND - 1; - } + else { + size_t j, l = jl_svec_len(specializations); + for (j = 0; j < l; j++) { + jl_method_instance_t *mi = (jl_method_instance_t*)jl_svecref(specializations, j); + if ((jl_value_t*)mi != jl_nothing) + jl_record_edges(mi, edges); } } - jl_array_del_end(callee_ids, l - nt); } - JL_GC_POP(); - htable_free(&edges_map2); } // Headers @@ -946,374 +751,324 @@ static void jl_copy_roots(jl_array_t *method_roots_list, uint64_t key) } } - -// verify that these edges intersect with the same methods as before -static jl_array_t *jl_verify_edges(jl_array_t *targets, size_t minworld) +static size_t verify_invokesig(jl_value_t *invokesig, jl_method_t *expected, size_t minworld) { - JL_TIMING(VERIFY_IMAGE, VERIFY_Edges); - size_t i, l = jl_array_nrows(targets) / 3; - static jl_value_t *ulong_array JL_ALWAYS_LEAFTYPE = NULL; - if (ulong_array == NULL) - ulong_array = jl_apply_array_type((jl_value_t*)jl_ulong_type, 1); - jl_array_t *maxvalids = jl_alloc_array_1d(ulong_array, l); - memset(jl_array_data(maxvalids, size_t), 0, l * sizeof(size_t)); - jl_value_t *loctag = NULL; - jl_value_t *matches = NULL; - jl_value_t *sig = NULL; - JL_GC_PUSH4(&maxvalids, &matches, &sig, &loctag); - for (i = 0; i < l; i++) { - jl_value_t *invokesig = jl_array_ptr_ref(targets, i * 3); - jl_value_t *callee = jl_array_ptr_ref(targets, i * 3 + 1); - jl_value_t *expected = jl_array_ptr_ref(targets, i * 3 + 2); - size_t min_valid = 0; - size_t max_valid = ~(size_t)0; - if (invokesig) { - assert(callee && "unsupported edge"); - jl_method_t *m = ((jl_method_instance_t*)callee)->def.method; - if (jl_egal(invokesig, m->sig)) { - // the invoke match is `m` for `m->sig`, unless `m` is invalid - if (jl_atomic_load_relaxed(&m->deleted_world) < max_valid) - max_valid = 0; - } - else { - jl_methtable_t *mt = jl_method_get_table(m); - if ((jl_value_t*)mt == jl_nothing) { - max_valid = 0; - } - else { - matches = jl_gf_invoke_lookup_worlds(invokesig, (jl_value_t*)mt, minworld, &min_valid, &max_valid); - if (matches == jl_nothing) { - max_valid = 0; - } - else { - matches = (jl_value_t*)((jl_method_match_t*)matches)->method; - if (matches != expected) { - max_valid = 0; - } - } - } - } + assert(jl_is_type(invokesig)); + assert(jl_is_method(expected)); + size_t min_valid = 0; + size_t max_valid = ~(size_t)0; + if (jl_egal(invokesig, expected->sig)) { + // the invoke match is `expected` for `expected->sig`, unless `expected` is invalid + if (jl_atomic_load_relaxed(&expected->deleted_world) < max_valid) + max_valid = 0; + } + else { + jl_methtable_t *mt = jl_method_get_table(expected); + if ((jl_value_t*)mt == jl_nothing) { + max_valid = 0; } else { - if (jl_is_method_instance(callee)) { - jl_method_instance_t *mi = (jl_method_instance_t*)callee; - sig = jl_type_intersection(mi->def.method->sig, (jl_value_t*)mi->specTypes); - } - else { - sig = callee; - } - assert(jl_is_array(expected)); - int ambig = 0; - // TODO: possibly need to included ambiguities too (for the optimizer correctness)? - // len + 1 is to allow us to log causes of invalidation (SnoopCompile's @snoopr) - matches = jl_matching_methods((jl_tupletype_t*)sig, jl_nothing, - _jl_debug_method_invalidation ? INT32_MAX : jl_array_nrows(expected), - 0, minworld, &min_valid, &max_valid, &ambig); - sig = NULL; + jl_value_t *matches = jl_gf_invoke_lookup_worlds(invokesig, (jl_value_t*)mt, minworld, &min_valid, &max_valid); if (matches == jl_nothing) { max_valid = 0; } else { - // setdiff!(matches, expected) - size_t j, k, ins = 0; - if (jl_array_nrows(matches) != jl_array_nrows(expected)) { + if (((jl_method_match_t*)matches)->method != expected) { max_valid = 0; } - for (k = 0; k < jl_array_nrows(matches); k++) { - jl_method_t *match = ((jl_method_match_t*)jl_array_ptr_ref(matches, k))->method; - size_t l = jl_array_nrows(expected); - for (j = 0; j < l; j++) - if (match == (jl_method_t*)jl_array_ptr_ref(expected, j)) - break; - if (j == l) { - // intersection has a new method or a method was - // deleted--this is now probably no good, just invalidate - // everything about it now - max_valid = 0; - if (!_jl_debug_method_invalidation) - break; - jl_array_ptr_set(matches, ins++, match); - } - } - if (max_valid != ~(size_t)0 && _jl_debug_method_invalidation) - jl_array_del_end((jl_array_t*)matches, jl_array_nrows(matches) - ins); } } - jl_array_data(maxvalids, size_t)[i] = max_valid; - if (max_valid != ~(size_t)0 && _jl_debug_method_invalidation) { - jl_array_ptr_1d_push(_jl_debug_method_invalidation, invokesig ? (jl_value_t*)invokesig : callee); - loctag = jl_cstr_to_string("insert_backedges_callee"); - jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); - loctag = jl_box_int32((int32_t)i); - jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); - jl_array_ptr_1d_push(_jl_debug_method_invalidation, matches); - } - //jl_static_show((JL_STREAM*)ios_stderr, (jl_value_t*)invokesig); - //jl_static_show((JL_STREAM*)ios_stderr, (jl_value_t*)callee); - //ios_puts(max_valid == ~(size_t)0 ? "valid\n" : "INVALID\n", ios_stderr); } - JL_GC_POP(); - return maxvalids; + return max_valid; } -// Combine all edges relevant to a method to initialize the maxvalids list -static jl_array_t *jl_verify_methods(jl_array_t *edges, jl_array_t *maxvalids) +static size_t verify_call(jl_value_t *sig, jl_svec_t *expecteds, size_t i, size_t n, size_t minworld, jl_value_t **matches JL_REQUIRE_ROOTED_SLOT) { - JL_TIMING(VERIFY_IMAGE, VERIFY_Methods); - jl_value_t *loctag = NULL; - jl_array_t *maxvalids2 = NULL; - JL_GC_PUSH2(&loctag, &maxvalids2); - size_t i, l = jl_array_nrows(edges) / 2; - maxvalids2 = jl_alloc_array_1d(jl_typeof(maxvalids), l); - size_t *maxvalids2_data = jl_array_data(maxvalids2, size_t); - memset(maxvalids2_data, 0, l * sizeof(size_t)); - for (i = 0; i < l; i++) { - jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(edges, 2 * i); - assert(jl_is_method_instance(caller) && jl_is_method(caller->def.method)); - jl_array_t *callee_ids = (jl_array_t*)jl_array_ptr_ref(edges, 2 * i + 1); - assert(jl_typetagis((jl_value_t*)callee_ids, jl_array_int32_type)); - if (callee_ids == NULL) { - // serializing the edges had failed - maxvalids2_data[i] = 0; + // verify that these edges intersect with the same methods as before + size_t min_valid = 0; + size_t max_valid = ~(size_t)0; + int ambig = 0; + // TODO: possibly need to included ambiguities too (for the optimizer correctness)? + jl_value_t *result = jl_matching_methods((jl_tupletype_t*)sig, jl_nothing, + _jl_debug_method_invalidation ? INT32_MAX : n, + 0, minworld, &min_valid, &max_valid, &ambig); + *matches = result; + if (result == jl_nothing) { + max_valid = 0; + } + else { + // setdiff!(result, expected) + size_t j, k, ins = 0; + if (jl_array_nrows(result) != n) { + max_valid = 0; } - else { - int32_t *idxs = jl_array_data(callee_ids, int32_t); - size_t j; - maxvalids2_data[i] = ~(size_t)0; - for (j = 0; j < idxs[0]; j++) { - int32_t idx = idxs[j + 1]; - size_t max_valid = jl_array_data(maxvalids, size_t)[idx]; - if (max_valid != ~(size_t)0 && _jl_debug_method_invalidation) { - jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)caller); - loctag = jl_cstr_to_string("verify_methods"); - jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); - loctag = jl_box_int32((int32_t)idx); - jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); + for (k = 0; k < jl_array_nrows(result); k++) { + jl_method_t *match = ((jl_method_match_t*)jl_array_ptr_ref(result, k))->method; + for (j = 0; j < n; j++) { + jl_value_t *t = jl_svecref(expecteds, j + i); + if (jl_is_code_instance(t)) + t = (jl_value_t*)((jl_code_instance_t*)t)->def; + jl_method_t *meth; + if (jl_is_method(t)) + meth = (jl_method_t*)t; + else { + assert(jl_is_method_instance(t)); + meth = ((jl_method_instance_t*)t)->def.method; } - if (max_valid < maxvalids2_data[i]) - maxvalids2_data[i] = max_valid; - if (max_valid == 0) + if (match == meth) + break; + } + if (j == n) { + // intersection has a new method or a method was + // deleted--this is now probably no good, just invalidate + // everything about it now + max_valid = 0; + if (!_jl_debug_method_invalidation) break; + jl_array_ptr_set(result, ins++, match); } } - //jl_static_show((JL_STREAM*)ios_stderr, (jl_value_t*)caller); - //ios_puts(maxvalids2_data[i] == ~(size_t)0 ? "valid\n" : "INVALID\n", ios_stderr); + if (max_valid != ~(size_t)0 && _jl_debug_method_invalidation) + jl_array_del_end((jl_array_t*)result, jl_array_nrows(result) - ins); } - JL_GC_POP(); - return maxvalids2; + return max_valid; } - -// Visit the entire call graph, starting from edges[idx] to determine if that method is valid -// Implements Tarjan's SCC (strongly connected components) algorithm, simplified to remove the count variable -// and slightly modified with an early termination option once the computation reaches its minimum -static int jl_verify_graph_edge(size_t *maxvalids2_data, jl_array_t *edges, size_t idx, arraylist_t *visited, arraylist_t *stack) +// Test all edges relevant to a method: +//// Visit the entire call graph, starting from edges[idx] to determine if that method is valid +//// Implements Tarjan's SCC (strongly connected components) algorithm, simplified to remove the count variable +//// and slightly modified with an early termination option once the computation reaches its minimum +static int jl_verify_method(jl_code_instance_t *codeinst, size_t minworld, size_t *maxworld, arraylist_t *stack, htable_t *visiting) { - if (maxvalids2_data[idx] == 0) { - visited->items[idx] = (void*)1; + size_t max_valid2 = jl_atomic_load_relaxed(&codeinst->max_world); + if (max_valid2 != WORLD_AGE_REVALIDATION_SENTINEL) { + *maxworld = max_valid2; return 0; } - size_t cycle = (size_t)visited->items[idx]; - if (cycle != 0) - return cycle - 1; // depth remaining - jl_value_t *cause = NULL; - arraylist_push(stack, (void*)idx); + assert(jl_is_method_instance(codeinst->def) && jl_is_method(codeinst->def->def.method)); + void **bp = ptrhash_bp(visiting, codeinst); + if (*bp != HT_NOTFOUND) + return (char*)*bp - (char*)HT_NOTFOUND; // cycle idx + arraylist_push(stack, (void*)codeinst); size_t depth = stack->len; - visited->items[idx] = (void*)(1 + depth); - jl_array_t *callee_ids = (jl_array_t*)jl_array_ptr_ref(edges, idx * 2 + 1); - assert(jl_typetagis((jl_value_t*)callee_ids, jl_array_int32_type)); - int32_t *idxs = jl_array_data(callee_ids, int32_t); - size_t i, n = jl_array_nrows(callee_ids); - cycle = depth; - for (i = idxs[0] + 1; i < n; i++) { - int32_t childidx = idxs[i]; - int child_cycle = jl_verify_graph_edge(maxvalids2_data, edges, childidx, visited, stack); - size_t child_max_valid = maxvalids2_data[childidx]; - if (child_max_valid < maxvalids2_data[idx]) { - maxvalids2_data[idx] = child_max_valid; - cause = jl_array_ptr_ref(edges, childidx * 2); + *bp = (char*)HT_NOTFOUND + depth; + JL_TIMING(VERIFY_IMAGE, VERIFY_Methods); + jl_value_t *loctag = NULL; + jl_value_t *sig = NULL; + jl_value_t *matches = NULL; + JL_GC_PUSH3(&loctag, &matches, &sig); + jl_svec_t *callees = jl_atomic_load_relaxed(&codeinst->edges); + assert(jl_is_svec((jl_value_t*)callees)); + // verify current edges + for (size_t j = 0; j < jl_svec_len(callees); ) { + jl_value_t *edge = jl_svecref(callees, j); + size_t max_valid2; + assert(!jl_is_method(edge)); // `Method`-edge isn't allowed for the optimized one-edge format + if (jl_is_code_instance(edge)) + edge = (jl_value_t*)((jl_code_instance_t*)edge)->def; + if (jl_is_method_instance(edge)) { + jl_method_instance_t *mi = (jl_method_instance_t*)edge; + sig = jl_type_intersection(mi->def.method->sig, (jl_value_t*)mi->specTypes); // TODO: ?? + max_valid2 = verify_call(sig, callees, j, 1, minworld, &matches); + sig = NULL; + j += 1; } - if (child_max_valid == 0) { - // found what we were looking for, so terminate early - break; + else if (jl_is_long(edge)) { + jl_value_t *sig = jl_svecref(callees, j + 1); + size_t nedges = jl_unbox_long(edge); + max_valid2 = verify_call(sig, callees, j + 2, nedges, minworld, &matches); + j += 2 + nedges; + edge = sig; } - else if (child_cycle && child_cycle < cycle) { - // record the cycle will resolve at depth "cycle" - cycle = child_cycle; + else if (jl_is_mtable(edge)) { + // skip the legacy edge (missing backedge) + j += 2; + continue; + } + else { + jl_method_instance_t *callee = (jl_method_instance_t*)jl_svecref(callees, j + 1); + jl_method_t *meth; + if (jl_is_code_instance(callee)) + callee = ((jl_code_instance_t*)callee)->def; + if (jl_is_method_instance(callee)) + meth = callee->def.method; + else { + assert(jl_is_method(callee)); + meth = (jl_method_t*)callee; + } + max_valid2 = verify_invokesig(edge, meth, minworld); + j += 2; + } + if (*maxworld > max_valid2) + *maxworld = max_valid2; + if (max_valid2 != ~(size_t)0 && _jl_debug_method_invalidation) { + jl_array_ptr_1d_push(_jl_debug_method_invalidation, edge); + loctag = jl_cstr_to_string("insert_backedges_callee"); + jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); + jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)codeinst); + jl_array_ptr_1d_push(_jl_debug_method_invalidation, matches); + } + //jl_static_show((JL_STREAM*)ios_stderr, (jl_value_t*)edge); + //ios_puts(max_valid2 == ~(size_t)0 ? "valid\n" : "INVALID\n", ios_stderr); + if (max_valid2 == 0 && !_jl_debug_method_invalidation) + break; + } + JL_GC_POP(); + // verify recursive edges (if valid, or debugging) + size_t cycle = depth; + jl_code_instance_t *cause = codeinst; + if (*maxworld == ~(size_t)0 || _jl_debug_method_invalidation) { + for (size_t j = 0; j < jl_svec_len(callees); j++) { + jl_value_t *edge = jl_svecref(callees, j); + if (!jl_is_code_instance(edge)) + continue; + jl_code_instance_t *callee = (jl_code_instance_t*)edge; + size_t max_valid2 = ~(size_t)0; + size_t child_cycle = jl_verify_method(callee, minworld, &max_valid2, stack, visiting); + if (*maxworld > max_valid2) { + cause = callee; + *maxworld = max_valid2; + } + if (max_valid2 == 0) { + // found what we were looking for, so terminate early + break; + } + else if (child_cycle && child_cycle < cycle) { + // record the cycle will resolve at depth "cycle" + cycle = child_cycle; + } } } - size_t max_valid = maxvalids2_data[idx]; - if (max_valid != 0 && cycle != depth) + if (*maxworld != 0 && cycle != depth) return cycle; // If we are the top of the current cycle, now mark all other parts of // our cycle with what we found. // Or if we found a failed edge, also mark all of the other parts of the - // cycle as also having an failed edge. + // cycle as also having a failed edge. while (stack->len >= depth) { - size_t childidx = (size_t)arraylist_pop(stack); - assert(visited->items[childidx] == (void*)(2 + stack->len)); - if (idx != childidx) { - if (max_valid < maxvalids2_data[childidx]) - maxvalids2_data[childidx] = max_valid; - } - visited->items[childidx] = (void*)1; - if (_jl_debug_method_invalidation && max_valid != ~(size_t)0) { - jl_method_instance_t *mi = (jl_method_instance_t*)jl_array_ptr_ref(edges, childidx * 2); - jl_value_t *loctag = NULL; - JL_GC_PUSH1(&loctag); - jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)mi); + jl_code_instance_t *child = (jl_code_instance_t*)arraylist_pop(stack); + if (*maxworld != jl_atomic_load_relaxed(&child->max_world)) + jl_atomic_store_relaxed(&child->max_world, *maxworld); + void **bp = ptrhash_bp(visiting, codeinst); + assert(*bp == (char*)HT_NOTFOUND + stack->len + 1); + *bp = HT_NOTFOUND; + if (_jl_debug_method_invalidation && *maxworld != ~(size_t)0) { + jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)child); loctag = jl_cstr_to_string("verify_methods"); + JL_GC_PUSH1(&loctag); jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)cause); JL_GC_POP(); } } + //jl_static_show((JL_STREAM*)ios_stderr, (jl_value_t*)codeinst->def); + //ios_puts(max_valid == ~(size_t)0 ? "valid\n\n" : "INVALID\n\n", ios_stderr); return 0; } -// Visit all entries in edges, verify if they are valid -static void jl_verify_graph(jl_array_t *edges, jl_array_t *maxvalids2) +static size_t jl_verify_method_graph(jl_code_instance_t *codeinst, size_t minworld, arraylist_t *stack, htable_t *visiting) { - JL_TIMING(VERIFY_IMAGE, VERIFY_Graph); - arraylist_t stack, visited; - arraylist_new(&stack, 0); - size_t i, n = jl_array_nrows(edges) / 2; - arraylist_new(&visited, n); - memset(visited.items, 0, n * sizeof(size_t)); - size_t *maxvalids2_data = jl_array_data(maxvalids2, size_t); - for (i = 0; i < n; i++) { - assert(visited.items[i] == (void*)0 || visited.items[i] == (void*)1); - int child_cycle = jl_verify_graph_edge(maxvalids2_data, edges, i, &visited, &stack); - assert(child_cycle == 0); (void)child_cycle; - assert(stack.len == 0); - assert(visited.items[i] == (void*)1); + assert(stack->len == 0); + for (size_t i = 0, hsz = visiting->size; i < hsz; i++) + assert(visiting->table[i] == HT_NOTFOUND); + size_t maxworld = ~(size_t)0; + int child_cycle = jl_verify_method(codeinst, minworld, &maxworld, stack, visiting); + assert(child_cycle == 0); (void)child_cycle; + assert(stack->len == 0); + for (size_t i = 0, hsz = visiting->size / 2; i < hsz; i++) { + assert(visiting->table[2 * i + 1] == HT_NOTFOUND); + visiting->table[2 * i] = HT_NOTFOUND; } - arraylist_free(&stack); - arraylist_free(&visited); + return maxworld; } // Restore backedges to external targets -// `edges` = [caller1, targets_indexes1, ...], the list of worklist-owned methods calling external methods. -// `ext_targets` is [invokesig1, callee1, matches1, ...], the global set of non-worklist callees of worklist-owned methods. -static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_targets, jl_array_t *ext_ci_list, size_t minworld) +// `edges` = [caller1, ...], the list of worklist-owned code instances internally +// `ext_ci_list` = [caller1, ...], the list of worklist-owned code instances externally +static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_ci_list, size_t minworld) { // determine which CodeInstance objects are still valid in our image - jl_array_t *valids = jl_verify_edges(ext_targets, minworld); - JL_GC_PUSH1(&valids); - valids = jl_verify_methods(edges, valids); // consumes edges valids, initializes methods valids - jl_verify_graph(edges, valids); // propagates methods valids for each edge - - size_t n_ext_cis = ext_ci_list ? jl_array_nrows(ext_ci_list) : 0; - htable_t cis_pending_validation; - htable_new(&cis_pending_validation, n_ext_cis); - - // next build a map from external MethodInstances to their CodeInstance for insertion - for (size_t i = 0; i < n_ext_cis; i++) { - jl_code_instance_t *ci = (jl_code_instance_t*)jl_array_ptr_ref(ext_ci_list, i); - if (jl_atomic_load_relaxed(&ci->max_world) == WORLD_AGE_REVALIDATION_SENTINEL) { - assert(jl_atomic_load_relaxed(&ci->min_world) == minworld); - void **bp = ptrhash_bp(&cis_pending_validation, (void*)ci->def); - assert(!jl_atomic_load_relaxed(&ci->next)); - if (*bp == HT_NOTFOUND) - *bp = (void*)ci; - else { - // Do ci->owner bifurcates the cache, we temporarily - // form a linked list of all the CI that need to be connected later - jl_code_instance_t *prev_ci = (jl_code_instance_t *)*bp; - jl_atomic_store_relaxed(&ci->next, prev_ci); - *bp = (void*)ci; - } - } - else { - assert(jl_atomic_load_relaxed(&ci->min_world) == 1); - assert(jl_atomic_load_relaxed(&ci->max_world) == ~(size_t)0); - jl_method_instance_t *caller = ci->def; - if (jl_atomic_load_relaxed(&ci->inferred) && jl_rettype_inferred(ci->owner, caller, minworld, ~(size_t)0) == jl_nothing) { - jl_mi_cache_insert(caller, ci); - } - //jl_static_show((JL_STREAM*)ios_stderr, (jl_value_t*)caller); - //ios_puts("free\n", ios_stderr); - } - } - - // next enable any applicable new codes - size_t nedges = jl_array_nrows(edges) / 2; - for (size_t i = 0; i < nedges; i++) { - jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(edges, 2 * i); - size_t maxvalid = jl_array_data(valids, size_t)[i]; - if (maxvalid == ~(size_t)0) { - // if this callee is still valid, add all the backedges - jl_array_t *callee_ids = (jl_array_t*)jl_array_ptr_ref(edges, 2 * i + 1); - int32_t *idxs = jl_array_data(callee_ids, int32_t); - for (size_t j = 0; j < idxs[0]; j++) { - int32_t idx = idxs[j + 1]; - jl_value_t *invokesig = jl_array_ptr_ref(ext_targets, idx * 3); - jl_value_t *callee = jl_array_ptr_ref(ext_targets, idx * 3 + 1); - if (callee && jl_is_method_instance(callee)) { - jl_method_instance_add_backedge((jl_method_instance_t*)callee, invokesig, caller); + // to enable any applicable new codes + arraylist_t stack; + arraylist_new(&stack, 0); + htable_t visiting; + htable_new(&visiting, 0); + for (size_t external = 0; external < (ext_ci_list ? 2 : 1); external++) { + if (external) + edges = ext_ci_list; + size_t nedges = jl_array_nrows(edges); + for (size_t i = 0; i < nedges; i++) { + jl_code_instance_t *codeinst = (jl_code_instance_t*)jl_array_ptr_ref(edges, i); + jl_svec_t *callees = jl_atomic_load_relaxed(&codeinst->edges); + jl_method_instance_t *caller = codeinst->def; + if (jl_atomic_load_relaxed(&codeinst->min_world) != minworld) { + if (external && jl_atomic_load_relaxed(&codeinst->max_world) != WORLD_AGE_REVALIDATION_SENTINEL) { + assert(jl_atomic_load_relaxed(&codeinst->min_world) == 1); + assert(jl_atomic_load_relaxed(&codeinst->max_world) == ~(size_t)0); } else { - jl_value_t *sig = callee == NULL ? invokesig : callee; - jl_methtable_t *mt = jl_method_table_for(sig); - // FIXME: rarely, `callee` has an unexpected `Union` signature, - // see https://github.com/JuliaLang/julia/pull/43990#issuecomment-1030329344 - // Fix the issue and turn this back into an `assert((jl_value_t*)mt != jl_nothing)` - // This workaround exposes us to (rare) 265-violations. - if ((jl_value_t*)mt != jl_nothing) - jl_method_table_add_backedge(mt, sig, (jl_value_t*)caller); + continue; } } - } - // then enable any methods associated with it - void *ci = ptrhash_get(&cis_pending_validation, (void*)caller); - //assert(ci != HT_NOTFOUND); - if (ci != HT_NOTFOUND) { - // Update any external CIs and add them to the cache. - assert(jl_is_code_instance(ci)); - jl_code_instance_t *codeinst = (jl_code_instance_t*)ci; - while (codeinst) { - jl_code_instance_t *next_ci = jl_atomic_load_relaxed(&codeinst->next); - jl_atomic_store_relaxed(&codeinst->next, NULL); - - jl_value_t *owner = codeinst->owner; - JL_GC_PROMISE_ROOTED(owner); - - assert(jl_atomic_load_relaxed(&codeinst->min_world) == minworld); - // See #53586, #53109 - // assert(jl_atomic_load_relaxed(&codeinst->max_world) == WORLD_AGE_REVALIDATION_SENTINEL); - assert(jl_atomic_load_relaxed(&codeinst->inferred)); - jl_atomic_store_relaxed(&codeinst->max_world, maxvalid); - - if (jl_rettype_inferred(owner, caller, minworld, maxvalid) != jl_nothing) { - // We already got a code instance for this world age range from somewhere else - we don't need - // this one. - } else { - jl_mi_cache_insert(caller, codeinst); + size_t maxvalid = jl_verify_method_graph(codeinst, minworld, &stack, &visiting); + assert(jl_atomic_load_relaxed(&codeinst->max_world) == maxvalid); + if (maxvalid == ~(size_t)0) { + // if this callee is still valid, add all the backedges + for (size_t j = 0; j < jl_svec_len(callees); ) { + jl_value_t *edge = jl_svecref(callees, j); + if (jl_is_long(edge)) { + j += 2; // skip over signature and count but not methods + continue; + } + else if (jl_is_method(edge)) { + j += 1; + continue; + } + if (jl_is_code_instance(edge)) + edge = (jl_value_t*)((jl_code_instance_t*)edge)->def; + if (jl_is_method_instance(edge)) { + jl_method_instance_add_backedge((jl_method_instance_t*)edge, NULL, codeinst); + j += 1; + } + else if (jl_is_mtable(edge)) { + jl_methtable_t *mt = (jl_methtable_t*)edge; + jl_value_t *sig = jl_svecref(callees, j + 1); + jl_method_table_add_backedge(mt, sig, codeinst); + j += 2; + } + else { + jl_value_t *callee = jl_svecref(callees, j + 1); + if (jl_is_code_instance(callee)) + callee = (jl_value_t*)((jl_code_instance_t*)callee)->def; + else if (jl_is_method(callee)) { + j += 2; + continue; + } + jl_method_instance_add_backedge((jl_method_instance_t*)callee, edge, codeinst); + j += 2; + } } - codeinst = next_ci; - } - } - else { - // Likely internal. Find the CI already in the cache hierarchy. - for (jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&caller->cache); codeinst; codeinst = jl_atomic_load_relaxed(&codeinst->next)) { - if (jl_atomic_load_relaxed(&codeinst->min_world) == minworld && jl_atomic_load_relaxed(&codeinst->max_world) == WORLD_AGE_REVALIDATION_SENTINEL) { - jl_atomic_store_relaxed(&codeinst->max_world, maxvalid); + if (external) { + jl_value_t *owner = codeinst->owner; + JL_GC_PROMISE_ROOTED(owner); + + // See #53586, #53109 + assert(jl_atomic_load_relaxed(&codeinst->inferred)); + + if (jl_rettype_inferred(owner, caller, minworld, maxvalid) != jl_nothing) { + // We already got a code instance for this world age range from somewhere else - we don't need + // this one. + } + else { + jl_mi_cache_insert(caller, codeinst); + } } } } } - htable_free(&cis_pending_validation); - - JL_GC_POP(); -} -static void classify_callers(htable_t *callers_with_edges, jl_array_t *edges) -{ - size_t l = edges ? jl_array_nrows(edges) / 2 : 0; - for (size_t i = 0; i < l; i++) { - jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(edges, 2 * i); - ptrhash_put(callers_with_edges, (void*)caller, (void*)caller); - } + htable_free(&visiting); + arraylist_free(&stack); } static jl_value_t *read_verify_mod_list(ios_t *s, jl_array_t *depmods) diff --git a/src/toplevel.c b/src/toplevel.c index 6dcab3095e320..017d61bbc8ceb 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -640,7 +640,7 @@ JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst_for_uninferred(jl_method_instan jl_code_instance_t *ci = jl_new_codeinst(mi, (jl_value_t*)jl_uninferred_sym, (jl_value_t*)jl_any_type, (jl_value_t*)jl_any_type, jl_nothing, (jl_value_t*)src, 0, src->min_world, src->max_world, - 0, NULL, 1, NULL); + 0, NULL, 1, NULL, NULL); return ci; } diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index d59a18e6d4f16..67191a024da73 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -588,7 +588,7 @@ CC.cache_owner(::REPLInterpreter) = REPLCacheToken() CC.may_optimize(::REPLInterpreter) = false # REPLInterpreter doesn't need any sources to be cached, so discard them aggressively -CC.transform_result_for_cache(::REPLInterpreter, ::Core.MethodInstance, ::CC.WorldRange, ::CC.InferenceResult) = nothing +CC.transform_result_for_cache(::REPLInterpreter, ::CC.InferenceResult) = nothing # REPLInterpreter analyzes a top-level frame, so better to not bail out from it CC.bail_out_toplevel_call(::REPLInterpreter, ::CC.InferenceLoopState, ::CC.InferenceState) = false @@ -673,8 +673,8 @@ function CC.concrete_eval_eligible(interp::REPLInterpreter, @nospecialize(f), sv::CC.InferenceState) if (interp.limit_aggressive_inference ? is_repl_frame(sv) : is_call_graph_uncached(sv)) neweffects = CC.Effects(result.effects; consistent=CC.ALWAYS_TRUE) - result = CC.MethodCallResult(result.rt, result.exct, result.edgecycle, result.edgelimited, - result.edge, neweffects) + result = CC.MethodCallResult(result.rt, result.exct, neweffects, result.edge, + result.edgecycle, result.edgelimited, result.volatile_inf_result) end ret = @invoke CC.concrete_eval_eligible(interp::CC.AbstractInterpreter, f::Any, result::CC.MethodCallResult, arginfo::CC.ArgInfo, diff --git a/test/compiler/AbstractInterpreter.jl b/test/compiler/AbstractInterpreter.jl index 009128b289ade..a49647ad4ea43 100644 --- a/test/compiler/AbstractInterpreter.jl +++ b/test/compiler/AbstractInterpreter.jl @@ -15,7 +15,7 @@ CC.may_optimize(::AbsIntOnlyInterp1) = false # it should work even if the interpreter discards inferred source entirely @newinterp AbsIntOnlyInterp2 CC.may_optimize(::AbsIntOnlyInterp2) = false -CC.transform_result_for_cache(::AbsIntOnlyInterp2, ::Core.MethodInstance, ::CC.WorldRange, ::CC.InferenceResult) = nothing +CC.transform_result_for_cache(::AbsIntOnlyInterp2, ::CC.InferenceResult) = nothing @test Base.infer_return_type(Base.init_stdio, (Ptr{Cvoid},); interp=AbsIntOnlyInterp2()) >: IO # OverlayMethodTable @@ -406,10 +406,10 @@ import .CC: CallInfo struct NoinlineCallInfo <: CallInfo info::CallInfo # wrapped call end +CC.add_edges_impl(edges::Vector{Any}, info::NoinlineCallInfo) = CC.add_edges!(edges, info.info) CC.nsplit_impl(info::NoinlineCallInfo) = CC.nsplit(info.info) CC.getsplit_impl(info::NoinlineCallInfo, idx::Int) = CC.getsplit(info.info, idx) CC.getresult_impl(info::NoinlineCallInfo, idx::Int) = CC.getresult(info.info, idx) -CC.add_uncovered_edges_impl(edges::Vector{Any}, info::NoinlineCallInfo, @nospecialize(atype)) = CC.add_uncovered_edges!(edges, info.info, atype) function CC.abstract_call(interp::NoinlineInterpreter, arginfo::CC.ArgInfo, si::CC.StmtInfo, sv::CC.InferenceState, max_methods::Int) @@ -497,10 +497,9 @@ struct CustomData inferred CustomData(@nospecialize inferred) = new(inferred) end -function CC.transform_result_for_cache(interp::CustomDataInterp, - mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult) - inferred_result = @invoke CC.transform_result_for_cache(interp::CC.AbstractInterpreter, - mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult) +function CC.transform_result_for_cache(interp::CustomDataInterp, result::CC.InferenceResult) + inferred_result = @invoke CC.transform_result_for_cache( + interp::CC.AbstractInterpreter, result::CC.InferenceResult) return CustomData(inferred_result) end function CC.src_inlining_policy(interp::CustomDataInterp, @nospecialize(src), diff --git a/test/compiler/EscapeAnalysis/EAUtils.jl b/test/compiler/EscapeAnalysis/EAUtils.jl index 1f0a84f1a8365..c71b821fd25f3 100644 --- a/test/compiler/EscapeAnalysis/EAUtils.jl +++ b/test/compiler/EscapeAnalysis/EAUtils.jl @@ -11,9 +11,8 @@ const EA = EscapeAnalysis # imports import .CC: - AbstractInterpreter, NativeInterpreter, WorldView, WorldRange, - InferenceParams, OptimizationParams, get_world_counter, get_inference_cache, - ipo_dataflow_analysis!, cache_result! + AbstractInterpreter, NativeInterpreter, WorldView, WorldRange, InferenceParams, + OptimizationParams, get_world_counter, get_inference_cache, ipo_dataflow_analysis! # usings using Core: CodeInstance, MethodInstance, CodeInfo diff --git a/test/compiler/contextual.jl b/test/compiler/contextual.jl index fc91a37c5bd9e..8d526fdefdc5b 100644 --- a/test/compiler/contextual.jl +++ b/test/compiler/contextual.jl @@ -88,8 +88,8 @@ module MiniCassette src = retrieve_code_info(mi, world) @assert isa(src, CodeInfo) src = copy(src) - @assert src.edges === nothing - src.edges = MethodInstance[mi] + @assert src.edges === Core.svec() + src.edges = Any[mi] transform!(mi, src, length(args), match.sparams) # TODO: this is mandatory: code_info.min_world = max(code_info.min_world, min_world[]) # TODO: this is mandatory: code_info.max_world = min(code_info.max_world, max_world[]) diff --git a/test/compiler/invalidation.jl b/test/compiler/invalidation.jl index 76cf3cbdc0796..55faa4287da24 100644 --- a/test/compiler/invalidation.jl +++ b/test/compiler/invalidation.jl @@ -95,7 +95,8 @@ end const GLOBAL_BUFFER = IOBuffer() # test backedge optimization when the callee's type and effects information are maximized -begin take!(GLOBAL_BUFFER) +begin + take!(GLOBAL_BUFFER) pr48932_callee(x) = (print(GLOBAL_BUFFER, x); Base.inferencebarrier(x)) pr48932_caller(x) = pr48932_callee(Base.inferencebarrier(x)) @@ -150,11 +151,11 @@ begin take!(GLOBAL_BUFFER) ci = mi.cache @test isdefined(ci, :next) @test ci.owner === nothing - @test ci.max_world == typemax(UInt) + @test_broken ci.max_world == typemax(UInt) ci = ci.next @test !isdefined(ci, :next) @test ci.owner === InvalidationTesterToken() - @test ci.max_world == typemax(UInt) + @test_broken ci.max_world == typemax(UInt) end @test isnothing(pr48932_caller(42)) @@ -213,11 +214,11 @@ begin take!(GLOBAL_BUFFER) ci = mi.cache @test isdefined(ci, :next) @test ci.owner === nothing - @test ci.max_world == typemax(UInt) + @test_broken ci.max_world == typemax(UInt) ci = ci.next @test !isdefined(ci, :next) @test ci.owner === InvalidationTesterToken() - @test ci.max_world == typemax(UInt) + @test_broken ci.max_world == typemax(UInt) end @test isnothing(pr48932_caller_unuse(42)) @test "foo" == String(take!(GLOBAL_BUFFER)) diff --git a/test/core.jl b/test/core.jl index 5ba0e99e730d4..4b5a674ba44b3 100644 --- a/test/core.jl +++ b/test/core.jl @@ -32,7 +32,7 @@ end # sanity tests that our built-in types are marked correctly for atomic fields for (T, c) in ( (Core.CodeInfo, []), - (Core.CodeInstance, [:next, :min_world, :max_world, :inferred, :debuginfo, :ipo_purity_bits, :invoke, :specptr, :specsigflags, :precompile]), + (Core.CodeInstance, [:next, :min_world, :max_world, :inferred, :edges, :debuginfo, :ipo_purity_bits, :invoke, :specptr, :specsigflags, :precompile]), (Core.Method, [:primary_world, :deleted_world]), (Core.MethodInstance, [:cache, :flags]), (Core.MethodTable, [:defs, :leafcache, :cache, :max_args]), diff --git a/test/precompile.jl b/test/precompile.jl index adf10363298ba..1607d4c6b502b 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -823,6 +823,7 @@ precompile_test_harness("code caching") do dir mispecs = minternal.specializations::Core.SimpleVector @test mispecs[1] === mi mi = mispecs[2]::Core.MethodInstance + mi.specTypes == Tuple{typeof(M.getelsize),Vector{M.X2}} ci = mi.cache @test ci.relocatability == 0 # PkgA loads PkgB, and both add roots to the same `push!` method (both before and after loading B) @@ -914,7 +915,7 @@ precompile_test_harness("code caching") do dir # external callers mods = Module[] for be in mi.backedges - push!(mods, be.def.module) + push!(mods, (be.def.def::Method).module) # XXX end @test MA ∈ mods @test MB ∈ mods @@ -923,7 +924,7 @@ precompile_test_harness("code caching") do dir # internal callers meths = Method[] for be in mi.backedges - push!(meths, be.def) + push!(meths, (be.def::Method).def) # XXX end @test which(M.g1, ()) ∈ meths @test which(M.g2, ()) ∈ meths @@ -1056,11 +1057,11 @@ precompile_test_harness("code caching") do dir idxs = findall(==("verify_methods"), invalidations) idxsbits = filter(idxs) do i mi = invalidations[i-1] - mi.def == m + mi.def.def === m end idx = only(idxsbits) tagbad = invalidations[idx+1] - @test isa(tagbad, Int32) + @test isa(tagbad, Core.CodeInstance) j = findfirst(==(tagbad), invalidations) @test invalidations[j-1] == "insert_backedges_callee" @test isa(invalidations[j-2], Type) @@ -1068,7 +1069,7 @@ precompile_test_harness("code caching") do dir m = only(methods(MB.useA2)) mi = only(Base.specializations(m)) @test !hasvalid(mi, world) - @test mi ∈ invalidations + @test any(x -> x isa Core.CodeInstance && x.def === mi, invalidations) m = only(methods(MB.map_nbits)) @test !hasvalid(m.specializations::Core.MethodInstance, world+1) # insert_backedges invalidations also trigger their backedges @@ -1178,12 +1179,7 @@ precompile_test_harness("invoke") do dir @eval using $CallerModule M = getfield(@__MODULE__, CallerModule) - function get_method_for_type(func, @nospecialize(T)) # return the method func(::T) - for m in methods(func) - m.sig.parameters[end] === T && return m - end - error("no ::Real method found for $func") - end + get_method_for_type(func, @nospecialize(T)) = which(func, (T,)) # return the method func(::T) function nvalid(mi::Core.MethodInstance) isdefined(mi, :cache) || return 0 ci = mi.cache @@ -1200,7 +1196,7 @@ precompile_test_harness("invoke") do dir mi = m.specializations::Core.MethodInstance @test length(mi.backedges) == 2 @test mi.backedges[1] === Tuple{typeof(func), Real} - @test isa(mi.backedges[2], Core.MethodInstance) + @test isa(mi.backedges[2], Core.CodeInstance) @test mi.cache.max_world == typemax(mi.cache.max_world) end for func in (M.q, M.qnc) @@ -1208,18 +1204,18 @@ precompile_test_harness("invoke") do dir mi = m.specializations::Core.MethodInstance @test length(mi.backedges) == 2 @test mi.backedges[1] === Tuple{typeof(func), Integer} - @test isa(mi.backedges[2], Core.MethodInstance) + @test isa(mi.backedges[2], Core.CodeInstance) @test mi.cache.max_world == typemax(mi.cache.max_world) end m = get_method_for_type(M.h, Real) - @test isempty(Base.specializations(m)) + @test nvalid(m.specializations::Core.MethodInstance) == 0 m = get_method_for_type(M.hnc, Real) - @test isempty(Base.specializations(m)) + @test nvalid(m.specializations::Core.MethodInstance) == 0 m = only(methods(M.callq)) - @test isempty(Base.specializations(m)) || nvalid(m.specializations::Core.MethodInstance) == 0 + @test nvalid(m.specializations::Core.MethodInstance) == 0 m = only(methods(M.callqnc)) - @test isempty(Base.specializations(m)) || nvalid(m.specializations::Core.MethodInstance) == 0 + @test nvalid(m.specializations::Core.MethodInstance) == 0 m = only(methods(M.callqi)) @test (m.specializations::Core.MethodInstance).specTypes == Tuple{typeof(M.callqi), Int} m = only(methods(M.callqnci)) @@ -1733,7 +1729,7 @@ precompile_test_harness("issue #46296") do load_path mi = first(Base.specializations(first(methods(identity)))) ci = Core.CodeInstance(mi, nothing, Any, Any, nothing, nothing, zero(Int32), typemin(UInt), typemax(UInt), zero(UInt32), nothing, 0x00, - Core.DebugInfo(mi)) + Core.DebugInfo(mi), Core.svec()) __init__() = @assert ci isa Core.CodeInstance diff --git a/test/precompile_absint1.jl b/test/precompile_absint1.jl index 7bc0382ffda85..ab36af163dc50 100644 --- a/test/precompile_absint1.jl +++ b/test/precompile_absint1.jl @@ -41,29 +41,33 @@ precompile_test_harness() do load_path let m = only(methods(TestAbsIntPrecompile1.basic_callee)) mi = only(Base.specializations(m)) ci = mi.cache - @test isdefined(ci, :next) + @test_broken isdefined(ci, :next) @test ci.owner === nothing @test ci.max_world == typemax(UInt) @test Base.module_build_id(TestAbsIntPrecompile1) == Base.object_build_id(ci) + @test_skip begin ci = ci.next @test !isdefined(ci, :next) @test ci.owner === cache_owner @test ci.max_world == typemax(UInt) @test Base.module_build_id(TestAbsIntPrecompile1) == Base.object_build_id(ci) + end end let m = only(methods(sum, (Vector{Float64},))) found = false for mi in Base.specializations(m) if mi isa Core.MethodInstance && mi.specTypes == Tuple{typeof(sum),Vector{Float64}} ci = mi.cache - @test isdefined(ci, :next) - @test ci.owner === cache_owner + @test_broken isdefined(ci, :next) + @test_broken ci.owner === cache_owner + @test_skip begin @test ci.max_world == typemax(UInt) @test Base.module_build_id(TestAbsIntPrecompile1) == Base.object_build_id(ci) ci = ci.next + end @test !isdefined(ci, :next) @test ci.owner === nothing @test ci.max_world == typemax(UInt) diff --git a/test/precompile_absint2.jl b/test/precompile_absint2.jl index 066dcbaece4c4..75b84e26e06c6 100644 --- a/test/precompile_absint2.jl +++ b/test/precompile_absint2.jl @@ -22,10 +22,9 @@ precompile_test_harness() do load_path inferred CustomData(@nospecialize inferred) = new(inferred) end - function CC.transform_result_for_cache(interp::PrecompileInterpreter, - mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult) - inferred_result = @invoke CC.transform_result_for_cache(interp::CC.AbstractInterpreter, - mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult) + function CC.transform_result_for_cache(interp::PrecompileInterpreter, result::CC.InferenceResult) + inferred_result = @invoke CC.transform_result_for_cache( + interp::CC.AbstractInterpreter, result::CC.InferenceResult) return CustomData(inferred_result) end function CC.src_inlining_policy(interp::PrecompileInterpreter, @nospecialize(src), @@ -64,29 +63,33 @@ precompile_test_harness() do load_path let m = only(methods(TestAbsIntPrecompile2.basic_callee)) mi = only(Base.specializations(m)) ci = mi.cache - @test isdefined(ci, :next) + @test_broken isdefined(ci, :next) @test ci.owner === nothing @test ci.max_world == typemax(UInt) @test Base.module_build_id(TestAbsIntPrecompile2) == Base.object_build_id(ci) + @test_skip begin ci = ci.next @test !isdefined(ci, :next) @test ci.owner === cache_owner @test ci.max_world == typemax(UInt) @test Base.module_build_id(TestAbsIntPrecompile2) == Base.object_build_id(ci) + end end let m = only(methods(sum, (Vector{Float64},))) found = false for mi = Base.specializations(m) if mi isa Core.MethodInstance && mi.specTypes == Tuple{typeof(sum),Vector{Float64}} ci = mi.cache - @test isdefined(ci, :next) - @test ci.owner === cache_owner + @test_broken isdefined(ci, :next) + @test_broken ci.owner === cache_owner + @test_skip begin @test ci.max_world == typemax(UInt) @test Base.module_build_id(TestAbsIntPrecompile2) == Base.object_build_id(ci) ci = ci.next + end @test !isdefined(ci, :next) @test ci.owner === nothing @test ci.max_world == typemax(UInt) diff --git a/test/stacktraces.jl b/test/stacktraces.jl index bc86479dbab4b..12da6d571013e 100644 --- a/test/stacktraces.jl +++ b/test/stacktraces.jl @@ -103,10 +103,7 @@ end end let src = Meta.lower(Main, quote let x = 1 end end).args[1]::Core.CodeInfo - li = ccall(:jl_new_method_instance_uninit, Ref{Core.MethodInstance}, ()) - @atomic li.cache = ccall(:jl_new_codeinst_for_uninferred, Ref{Core.CodeInstance}, (Any, Any), li, src) - li.specTypes = Tuple{} - li.def = @__MODULE__ + li = ccall(:jl_method_instance_for_thunk, Ref{Core.MethodInstance}, (Any, Any), src, @__MODULE__) sf = StackFrame(:a, :b, 3, li, false, false, 0) repr = string(sf) @test repr == "Toplevel MethodInstance thunk at b:3" From 85dc2c72110e44f68f483c2b7b2e4dc68c73a143 Mon Sep 17 00:00:00 2001 From: CY Han Date: Fri, 1 Nov 2024 22:20:10 +0800 Subject: [PATCH 09/10] docs: remove `dirname.c` from THIRDPARTY file (#56413) - `dirname.c` was removed by https://github.com/JuliaLang/julia/commit/c2cec7ad57102e4fbb733b8fb79d617a9524f0ae --- THIRDPARTY.md | 1 - 1 file changed, 1 deletion(-) diff --git a/THIRDPARTY.md b/THIRDPARTY.md index 30f53727c50ab..3a74afec4a283 100644 --- a/THIRDPARTY.md +++ b/THIRDPARTY.md @@ -6,7 +6,6 @@ for exceptions. - [crc32c.c](https://stackoverflow.com/questions/17645167/implementing-sse-4-2s-crc32c-in-software) (CRC-32c checksum code by Mark Adler) [[ZLib](https://opensource.org/licenses/Zlib)]. - [LDC](https://github.com/ldc-developers/ldc/blob/master/LICENSE) (for ccall/cfunction ABI definitions) [BSD-3]. The portion of code that Julia uses from LDC is [BSD-3] licensed. - [LLVM](https://releases.llvm.org/3.9.0/LICENSE.TXT) (for parts of src/disasm.cpp) [UIUC] -- [MINGW](https://sourceforge.net/p/mingw/mingw-org-wsl/ci/legacy/tree/mingwrt/mingwex/dirname.c) (for dirname implementation on Windows) [MIT] - [NetBSD](https://www.netbsd.org/about/redistribution.html) (for setjmp, longjmp, and strptime implementations on Windows) [BSD-3] - [Python](https://docs.python.org/3/license.html) (for strtod implementation on Windows) [PSF] - [FEMTOLISP](https://github.com/JeffBezanson/femtolisp) [BSD-3] From 7d81897aef774ce953ed67a9951ae1a051d38544 Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Fri, 1 Nov 2024 10:57:23 -0400 Subject: [PATCH 10/10] =?UTF-8?q?Allow=20ext=20=E2=86=92=20ext=20dependenc?= =?UTF-8?q?y=20if=20triggers=20are=20a=20strict=20superset=20(#56368)=20(#?= =?UTF-8?q?56402)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Forward port of #56368 - this was a pretty clean port, so it should be good to go once tests pass. --- base/loading.jl | 75 ++++++++++--------- base/precompilation.jl | 31 +++++--- test/loading.jl | 68 +++++++++++++++++ .../Manifest.toml | 32 ++++++++ .../Project.toml | 12 +++ .../ext/ExtAB.jl | 12 +++ .../src/CrossPackageExtToExtDependency.jl | 7 ++ .../Extensions/CyclicExtensions/Manifest.toml | 7 +- .../Extensions/EnvWithDeps/Manifest.toml | 7 +- .../EnvWithHasExtensions/Manifest.toml | 7 +- .../EnvWithHasExtensionsv2/Manifest.toml | 7 +- .../project/Extensions/ExtDep.jl/Project.toml | 1 + .../Extensions/ExtDep.jl/src/ExtDep.jl | 1 + .../ExtToExtDependency/Manifest.toml | 21 ++++++ .../ExtToExtDependency/Project.toml | 14 ++++ .../Extensions/ExtToExtDependency/ext/ExtA.jl | 6 ++ .../ExtToExtDependency/ext/ExtAB.jl | 12 +++ .../src/ExtToExtDependency.jl | 7 ++ .../HasDepWithExtensions.jl/Manifest.toml | 7 +- .../Extensions/SomeOtherPackage/Project.toml | 4 + .../SomeOtherPackage/src/SomeOtherPackage.jl | 5 ++ 21 files changed, 293 insertions(+), 50 deletions(-) create mode 100644 test/project/Extensions/CrossPackageExtToExtDependency/Manifest.toml create mode 100644 test/project/Extensions/CrossPackageExtToExtDependency/Project.toml create mode 100644 test/project/Extensions/CrossPackageExtToExtDependency/ext/ExtAB.jl create mode 100644 test/project/Extensions/CrossPackageExtToExtDependency/src/CrossPackageExtToExtDependency.jl create mode 100644 test/project/Extensions/ExtToExtDependency/Manifest.toml create mode 100644 test/project/Extensions/ExtToExtDependency/Project.toml create mode 100644 test/project/Extensions/ExtToExtDependency/ext/ExtA.jl create mode 100644 test/project/Extensions/ExtToExtDependency/ext/ExtAB.jl create mode 100644 test/project/Extensions/ExtToExtDependency/src/ExtToExtDependency.jl create mode 100644 test/project/Extensions/SomeOtherPackage/Project.toml create mode 100644 test/project/Extensions/SomeOtherPackage/src/SomeOtherPackage.jl diff --git a/base/loading.jl b/base/loading.jl index 28875b8713b35..7b45348f47009 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -756,8 +756,9 @@ function manifest_uuid_path(env::String, pkg::PkgId)::Union{Nothing,String,Missi proj = implicit_manifest_uuid_path(env, pkg) proj === nothing || return proj # if not found - parentid = get(EXT_PRIMED, pkg, nothing) - if parentid !== nothing + triggers = get(EXT_PRIMED, pkg, nothing) + if triggers !== nothing + parentid = triggers[1] _, parent_project_file = entry_point_and_project_file(env, parentid.name) if parent_project_file !== nothing parentproj = project_file_name_uuid(parent_project_file, parentid.name) @@ -1432,9 +1433,7 @@ function run_module_init(mod::Module, i::Int=1) end function run_package_callbacks(modkey::PkgId) - if !precompiling_extension - run_extension_callbacks(modkey) - end + run_extension_callbacks(modkey) assert_havelock(require_lock) unlock(require_lock) try @@ -1463,7 +1462,7 @@ mutable struct ExtensionId ntriggers::Int # how many more packages must be defined until this is loaded end -const EXT_PRIMED = Dict{PkgId, PkgId}() # Extension -> Parent +const EXT_PRIMED = Dict{PkgId,Vector{PkgId}}() # Extension -> Parent + Triggers (parent is always first) const EXT_DORMITORY = Dict{PkgId,Vector{ExtensionId}}() # Trigger -> Extensions that can be triggered by it const EXT_DORMITORY_FAILED = ExtensionId[] @@ -1554,7 +1553,7 @@ function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any} if haskey(EXT_PRIMED, id) || haskey(Base.loaded_modules, id) continue # extension is already primed or loaded, don't add it again end - EXT_PRIMED[id] = parent + EXT_PRIMED[id] = trigger_ids = PkgId[parent] gid = ExtensionId(id, parent, 1 + length(triggers), 1 + length(triggers)) trigger1 = get!(Vector{ExtensionId}, EXT_DORMITORY, parent) push!(trigger1, gid) @@ -1562,6 +1561,7 @@ function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any} # TODO: Better error message if this lookup fails? uuid_trigger = UUID(totaldeps[trigger]::String) trigger_id = PkgId(uuid_trigger, trigger) + push!(trigger_ids, trigger_id) if !haskey(Base.loaded_modules, trigger_id) || haskey(package_locks, trigger_id) trigger1 = get!(Vector{ExtensionId}, EXT_DORMITORY, trigger_id) push!(trigger1, gid) @@ -1573,6 +1573,7 @@ function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any} end loading_extension::Bool = false +loadable_extensions::Union{Nothing,Vector{PkgId}} = nothing precompiling_extension::Bool = false function run_extension_callbacks(extid::ExtensionId) assert_havelock(require_lock) @@ -1603,7 +1604,7 @@ function run_extension_callbacks(pkgid::PkgId) for extid in extids @assert extid.ntriggers > 0 extid.ntriggers -= 1 - if extid.ntriggers == 0 + if extid.ntriggers == 0 && (loadable_extensions === nothing || extid.id in loadable_extensions) push!(extids_to_load, extid) end end @@ -2645,7 +2646,17 @@ function __require_prelocked(pkg::PkgId, env) # double-check the search now that we have lock m = _require_search_from_serialized(pkg, path, UInt128(0), true) m isa Module && return m - return compilecache(pkg, path; reasons) + triggers = get(EXT_PRIMED, pkg, nothing) + loadable_exts = nothing + if triggers !== nothing # extension + loadable_exts = PkgId[] + for (extβ€², triggersβ€²) in EXT_PRIMED + if triggersβ€² ⊊ triggers + push!(loadable_exts, extβ€²) + end + end + end + return compilecache(pkg, path; reasons, loadable_exts) end loaded isa Module && return loaded if isnothing(loaded) # maybe_cachefile_lock returns nothing if it had to wait for another process @@ -2996,10 +3007,16 @@ function check_package_module_loaded(pkg::PkgId) return nothing end +# protects against PkgId and UUID being imported and losing Base prefix +_pkg_str(_pkg::PkgId) = (_pkg.uuid === nothing) ? "Base.PkgId($(repr(_pkg.name)))" : "Base.PkgId(Base.UUID(\"$(_pkg.uuid)\"), $(repr(_pkg.name)))" +_pkg_str(_pkg::Vector) = sprint(show, eltype(_pkg); context = :module=>nothing) * "[" * join(map(_pkg_str, _pkg), ",") * "]" +_pkg_str(_pkg::Pair{PkgId}) = _pkg_str(_pkg.first) * " => " * repr(_pkg.second) +_pkg_str(_pkg::Nothing) = "nothing" + const PRECOMPILE_TRACE_COMPILE = Ref{String}() function create_expr_cache(pkg::PkgId, input::String, output::String, output_o::Union{Nothing, String}, concrete_deps::typeof(_concrete_dependencies), flags::Cmd=``, cacheflags::CacheFlags=CacheFlags(), - internal_stderr::IO = stderr, internal_stdout::IO = stdout, isext::Bool=false) + internal_stderr::IO = stderr, internal_stdout::IO = stdout, loadable_exts::Union{Vector{PkgId},Nothing}=nothing) @nospecialize internal_stderr internal_stdout rm(output, force=true) # Remove file if it exists output_o === nothing || rm(output_o, force=true) @@ -3007,8 +3024,9 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: dl_load_path = String[abspath(x) for x in DL_LOAD_PATH] load_path = String[abspath(x) for x in Base.load_path()] # if pkg is a stdlib, append its parent Project.toml to the load path - parentid = get(EXT_PRIMED, pkg, nothing) - if parentid !== nothing + triggers = get(EXT_PRIMED, pkg, nothing) + if triggers !== nothing + parentid = triggers[1] for env in load_path project_file = env_project_file(env) if project_file === true @@ -3026,22 +3044,6 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: any(path -> path_sep in path, load_path) && error("LOAD_PATH entries cannot contain $(repr(path_sep))") - deps_strs = String[] - # protects against PkgId and UUID being imported and losing Base prefix - function pkg_str(_pkg::PkgId) - if _pkg.uuid === nothing - "Base.PkgId($(repr(_pkg.name)))" - else - "Base.PkgId(Base.UUID(\"$(_pkg.uuid)\"), $(repr(_pkg.name)))" - end - end - for (pkg, build_id) in concrete_deps - push!(deps_strs, "$(pkg_str(pkg)) => $(repr(build_id))") - end - deps_eltype = sprint(show, eltype(concrete_deps); context = :module=>nothing) - deps = deps_eltype * "[" * join(deps_strs, ",") * "]" - precomp_stack = "Base.PkgId[$(join(map(pkg_str, vcat(Base.precompilation_stack, pkg)), ", "))]" - if output_o === nothing # remove options that make no difference given the other cache options cacheflags = CacheFlags(cacheflags, opt_level=0) @@ -3072,10 +3074,11 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: # write data over stdin to avoid the (unlikely) case of exceeding max command line size write(io.in, """ empty!(Base.EXT_DORMITORY) # If we have a custom sysimage with `EXT_DORMITORY` prepopulated - Base.track_nested_precomp($precomp_stack) - Base.precompiling_extension = $(loading_extension | isext) - Base.include_package_for_output($(pkg_str(pkg)), $(repr(abspath(input))), $(repr(depot_path)), $(repr(dl_load_path)), - $(repr(load_path)), $deps, $(repr(source_path(nothing)))) + Base.track_nested_precomp($(_pkg_str(vcat(Base.precompilation_stack, pkg)))) + Base.loadable_extensions = $(_pkg_str(loadable_exts)) + Base.precompiling_extension = $(loading_extension) + Base.include_package_for_output($(_pkg_str(pkg)), $(repr(abspath(input))), $(repr(depot_path)), $(repr(dl_load_path)), + $(repr(load_path)), $(_pkg_str(concrete_deps)), $(repr(source_path(nothing)))) """) close(io.in) return io @@ -3130,18 +3133,18 @@ This can be used to reduce package load times. Cache files are stored in `DEPOT_PATH[1]/compiled`. See [Module initialization and precompilation](@ref) for important notes. """ -function compilecache(pkg::PkgId, internal_stderr::IO = stderr, internal_stdout::IO = stdout; flags::Cmd=``, reasons::Union{Dict{String,Int},Nothing}=Dict{String,Int}(), isext::Bool=false) +function compilecache(pkg::PkgId, internal_stderr::IO = stderr, internal_stdout::IO = stdout; flags::Cmd=``, reasons::Union{Dict{String,Int},Nothing}=Dict{String,Int}(), loadable_exts::Union{Vector{PkgId},Nothing}=nothing) @nospecialize internal_stderr internal_stdout path = locate_package(pkg) path === nothing && throw(ArgumentError("$(repr("text/plain", pkg)) not found during precompilation")) - return compilecache(pkg, path, internal_stderr, internal_stdout; flags, reasons, isext) + return compilecache(pkg, path, internal_stderr, internal_stdout; flags, reasons, loadable_exts) end const MAX_NUM_PRECOMPILE_FILES = Ref(10) function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, internal_stdout::IO = stdout, keep_loaded_modules::Bool = true; flags::Cmd=``, cacheflags::CacheFlags=CacheFlags(), - reasons::Union{Dict{String,Int},Nothing}=Dict{String,Int}(), isext::Bool=false) + reasons::Union{Dict{String,Int},Nothing}=Dict{String,Int}(), loadable_exts::Union{Vector{PkgId},Nothing}=nothing) @nospecialize internal_stderr internal_stdout # decide where to put the resulting cache file @@ -3181,7 +3184,7 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in close(tmpio_o) close(tmpio_so) end - p = create_expr_cache(pkg, path, tmppath, tmppath_o, concrete_deps, flags, cacheflags, internal_stderr, internal_stdout, isext) + p = create_expr_cache(pkg, path, tmppath, tmppath_o, concrete_deps, flags, cacheflags, internal_stderr, internal_stdout, loadable_exts) if success(p) if cache_objects diff --git a/base/precompilation.jl b/base/precompilation.jl index 54f13d298a462..edd8824ff8d68 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -436,6 +436,7 @@ function _precompilepkgs(pkgs::Vector{String}, return name end + triggers = Dict{Base.PkgId,Vector{Base.PkgId}}() for (dep, deps) in env.deps pkg = Base.PkgId(dep, env.names[dep]) Base.in_sysimage(pkg) && continue @@ -444,25 +445,22 @@ function _precompilepkgs(pkgs::Vector{String}, # add any extensions pkg_exts = Dict{Base.PkgId, Vector{Base.PkgId}}() for (ext_name, extdep_uuids) in env.extensions[dep] - ext_deps = Base.PkgId[] - push!(ext_deps, pkg) # depends on parent package + ext_uuid = Base.uuid5(pkg.uuid, ext_name) + ext = Base.PkgId(ext_uuid, ext_name) + triggers[ext] = Base.PkgId[pkg] # depends on parent package all_extdeps_available = true for extdep_uuid in extdep_uuids extdep_name = env.names[extdep_uuid] if extdep_uuid in keys(env.deps) - push!(ext_deps, Base.PkgId(extdep_uuid, extdep_name)) + push!(triggers[ext], Base.PkgId(extdep_uuid, extdep_name)) else all_extdeps_available = false break end end all_extdeps_available || continue - ext_uuid = Base.uuid5(pkg.uuid, ext_name) - ext = Base.PkgId(ext_uuid, ext_name) - filter!(!Base.in_sysimage, ext_deps) - depsmap[ext] = ext_deps exts[ext] = pkg.name - pkg_exts[ext] = ext_deps + pkg_exts[ext] = depsmap[ext] = filter(!Base.in_sysimage, triggers[ext]) end if !isempty(pkg_exts) pkg_exts_map[pkg] = collect(keys(pkg_exts)) @@ -478,6 +476,16 @@ function _precompilepkgs(pkgs::Vector{String}, append!(direct_deps, keys(filter(d->last(d) in keys(env.project_deps), exts))) @debug "precompile: deps collected" + + # An extension effectively depends on another extension if it has a strict superset of its triggers + for ext_a in keys(exts) + for ext_b in keys(exts) + if triggers[ext_a] βŠ‹ triggers[ext_b] + push!(depsmap[ext_a], ext_b) + end + end + end + # this loop must be run after the full depsmap has been populated for (pkg, pkg_exts) in pkg_exts_map # find any packages that depend on the extension(s)'s deps and replace those deps in their deps list with the extension(s), @@ -839,7 +847,12 @@ function _precompilepkgs(pkgs::Vector{String}, t = @elapsed ret = precompile_pkgs_maybe_cachefile_lock(io, print_lock, fancyprint, pkg_config, pkgspidlocked, hascolor) do Base.with_logger(Base.NullLogger()) do # The false here means we ignore loaded modules, so precompile for a fresh session - Base.compilecache(pkg, sourcepath, std_pipe, std_pipe, false; flags, cacheflags, isext = haskey(exts, pkg)) + keep_loaded_modules = false + # for extensions, any extension in our direct dependencies is one we have a right to load + # for packages, we may load any extension (all possible triggers are accounted for above) + loadable_exts = haskey(exts, pkg) ? filter((dep)->haskey(exts, dep), depsmap[pkg]) : nothing + Base.compilecache(pkg, sourcepath, std_pipe, std_pipe, keep_loaded_modules; + flags, cacheflags, loadable_exts) end end if ret isa Base.PrecompilableError diff --git a/test/loading.jl b/test/loading.jl index ecba64ca45a73..1cc20548d9bc8 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -1152,6 +1152,74 @@ end cmd = addenv(cmd, "JULIA_LOAD_PATH" => proj) @test occursin("Hello Cycles!", String(read(cmd))) + # Extension-to-extension dependencies + + mktempdir() do depot # Parallel pre-compilation + code = """ + Base.disable_parallel_precompile = false + using ExtToExtDependency + Base.get_extension(ExtToExtDependency, :ExtA) isa Module || error("expected extension to load") + Base.get_extension(ExtToExtDependency, :ExtAB) isa Module || error("expected extension to load") + ExtToExtDependency.greet() + """ + proj = joinpath(@__DIR__, "project", "Extensions", "ExtToExtDependency") + cmd = `$(Base.julia_cmd()) --startup-file=no -e $code` + cmd = addenv(cmd, + "JULIA_LOAD_PATH" => proj, + "JULIA_DEPOT_PATH" => depot * Base.Filesystem.pathsep(), + ) + @test occursin("Hello ext-to-ext!", String(read(cmd))) + end + mktempdir() do depot # Serial pre-compilation + code = """ + Base.disable_parallel_precompile = true + using ExtToExtDependency + Base.get_extension(ExtToExtDependency, :ExtA) isa Module || error("expected extension to load") + Base.get_extension(ExtToExtDependency, :ExtAB) isa Module || error("expected extension to load") + ExtToExtDependency.greet() + """ + proj = joinpath(@__DIR__, "project", "Extensions", "ExtToExtDependency") + cmd = `$(Base.julia_cmd()) --startup-file=no -e $code` + cmd = addenv(cmd, + "JULIA_LOAD_PATH" => proj, + "JULIA_DEPOT_PATH" => depot * Base.Filesystem.pathsep(), + ) + @test occursin("Hello ext-to-ext!", String(read(cmd))) + end + + mktempdir() do depot # Parallel pre-compilation + code = """ + Base.disable_parallel_precompile = false + using CrossPackageExtToExtDependency + Base.get_extension(CrossPackageExtToExtDependency.CyclicExtensions, :ExtA) isa Module || error("expected extension to load") + Base.get_extension(CrossPackageExtToExtDependency, :ExtAB) isa Module || error("expected extension to load") + CrossPackageExtToExtDependency.greet() + """ + proj = joinpath(@__DIR__, "project", "Extensions", "CrossPackageExtToExtDependency") + cmd = `$(Base.julia_cmd()) --startup-file=no -e $code` + cmd = addenv(cmd, + "JULIA_LOAD_PATH" => proj, + "JULIA_DEPOT_PATH" => depot * Base.Filesystem.pathsep(), + ) + @test occursin("Hello x-package ext-to-ext!", String(read(cmd))) + end + mktempdir() do depot # Serial pre-compilation + code = """ + Base.disable_parallel_precompile = true + using CrossPackageExtToExtDependency + Base.get_extension(CrossPackageExtToExtDependency.CyclicExtensions, :ExtA) isa Module || error("expected extension to load") + Base.get_extension(CrossPackageExtToExtDependency, :ExtAB) isa Module || error("expected extension to load") + CrossPackageExtToExtDependency.greet() + """ + proj = joinpath(@__DIR__, "project", "Extensions", "CrossPackageExtToExtDependency") + cmd = `$(Base.julia_cmd()) --startup-file=no -e $code` + cmd = addenv(cmd, + "JULIA_LOAD_PATH" => proj, + "JULIA_DEPOT_PATH" => depot * Base.Filesystem.pathsep(), + ) + @test occursin("Hello x-package ext-to-ext!", String(read(cmd))) + end + finally try rm(depot_path, force=true, recursive=true) diff --git a/test/project/Extensions/CrossPackageExtToExtDependency/Manifest.toml b/test/project/Extensions/CrossPackageExtToExtDependency/Manifest.toml new file mode 100644 index 0000000000000..5497fdb7091bb --- /dev/null +++ b/test/project/Extensions/CrossPackageExtToExtDependency/Manifest.toml @@ -0,0 +1,32 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.11.1" +manifest_format = "2.0" +project_hash = "dc35c2cf8c6b82fb5b9624c9713c2df34ca30499" + +[[deps.CyclicExtensions]] +deps = ["ExtDep"] +path = "../CyclicExtensions" +uuid = "17d4f0df-b55c-4714-ac4b-55fa23f7355c" +version = "0.1.0" +weakdeps = ["SomePackage"] + + [deps.CyclicExtensions.extensions] + ExtA = ["SomePackage"] + ExtB = ["SomePackage"] + +[[deps.ExtDep]] +deps = ["SomeOtherPackage", "SomePackage"] +path = "../ExtDep.jl" +uuid = "fa069be4-f60b-4d4c-8b95-f8008775090c" +version = "0.1.0" + +[[deps.SomeOtherPackage]] +path = "../SomeOtherPackage" +uuid = "178f68a2-4498-45ee-a775-452b36359b63" +version = "0.1.0" + +[[deps.SomePackage]] +path = "../SomePackage" +uuid = "678608ae-7bb3-42c7-98b1-82102067a3d8" +version = "0.1.0" diff --git a/test/project/Extensions/CrossPackageExtToExtDependency/Project.toml b/test/project/Extensions/CrossPackageExtToExtDependency/Project.toml new file mode 100644 index 0000000000000..76ffb7bd1c882 --- /dev/null +++ b/test/project/Extensions/CrossPackageExtToExtDependency/Project.toml @@ -0,0 +1,12 @@ +name = "CrossPackageExtToExtDependency" +uuid = "30f07f2e-c47e-40db-93a2-cbc4d1b301cc" +version = "0.1.0" + +[deps] +CyclicExtensions = "17d4f0df-b55c-4714-ac4b-55fa23f7355c" + +[weakdeps] +SomePackage = "678608ae-7bb3-42c7-98b1-82102067a3d8" + +[extensions] +ExtAB = ["CyclicExtensions", "SomePackage"] diff --git a/test/project/Extensions/CrossPackageExtToExtDependency/ext/ExtAB.jl b/test/project/Extensions/CrossPackageExtToExtDependency/ext/ExtAB.jl new file mode 100644 index 0000000000000..1ded9f2df5097 --- /dev/null +++ b/test/project/Extensions/CrossPackageExtToExtDependency/ext/ExtAB.jl @@ -0,0 +1,12 @@ +module ExtAB + +using CrossPackageExtToExtDependency +using SomePackage +using CyclicExtensions + +const ExtA = Base.get_extension(CyclicExtensions, :ExtA) +if !(ExtA isa Module) + error("expected extension to load") +end + +end diff --git a/test/project/Extensions/CrossPackageExtToExtDependency/src/CrossPackageExtToExtDependency.jl b/test/project/Extensions/CrossPackageExtToExtDependency/src/CrossPackageExtToExtDependency.jl new file mode 100644 index 0000000000000..28b229e2d61bf --- /dev/null +++ b/test/project/Extensions/CrossPackageExtToExtDependency/src/CrossPackageExtToExtDependency.jl @@ -0,0 +1,7 @@ +module CrossPackageExtToExtDependency + +using CyclicExtensions + +greet() = print("Hello x-package ext-to-ext!") + +end # module CrossPackageExtToTextDependency diff --git a/test/project/Extensions/CyclicExtensions/Manifest.toml b/test/project/Extensions/CyclicExtensions/Manifest.toml index a506825cf7995..0f280293c07b6 100644 --- a/test/project/Extensions/CyclicExtensions/Manifest.toml +++ b/test/project/Extensions/CyclicExtensions/Manifest.toml @@ -5,7 +5,7 @@ manifest_format = "2.0" project_hash = "ec25ff8df3a5e2212a173c3de2c7d716cc47cd36" [[deps.ExtDep]] -deps = ["SomePackage"] +deps = ["SomePackage", "SomeOtherPackage"] path = "../ExtDep.jl" uuid = "fa069be4-f60b-4d4c-8b95-f8008775090c" version = "0.1.0" @@ -15,6 +15,11 @@ path = "../ExtDep2" uuid = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d" version = "0.1.0" +[[deps.SomeOtherPackage]] +path = "../SomeOtherPackage" +uuid = "178f68a2-4498-45ee-a775-452b36359b63" +version = "0.1.0" + [[deps.SomePackage]] path = "../SomePackage" uuid = "678608ae-7bb3-42c7-98b1-82102067a3d8" diff --git a/test/project/Extensions/EnvWithDeps/Manifest.toml b/test/project/Extensions/EnvWithDeps/Manifest.toml index 85ff259f0a4d5..554a317b370eb 100644 --- a/test/project/Extensions/EnvWithDeps/Manifest.toml +++ b/test/project/Extensions/EnvWithDeps/Manifest.toml @@ -5,7 +5,7 @@ manifest_format = "2.0" project_hash = "ec25ff8df3a5e2212a173c3de2c7d716cc47cd36" [[deps.ExtDep]] -deps = ["SomePackage"] +deps = ["SomePackage", "SomeOtherPackage"] path = "../ExtDep.jl" uuid = "fa069be4-f60b-4d4c-8b95-f8008775090c" version = "0.1.0" @@ -15,6 +15,11 @@ path = "../ExtDep2" uuid = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d" version = "0.1.0" +[[deps.SomeOtherPackage]] +path = "../SomeOtherPackage" +uuid = "178f68a2-4498-45ee-a775-452b36359b63" +version = "0.1.0" + [[deps.SomePackage]] path = "../SomePackage" uuid = "678608ae-7bb3-42c7-98b1-82102067a3d8" diff --git a/test/project/Extensions/EnvWithHasExtensions/Manifest.toml b/test/project/Extensions/EnvWithHasExtensions/Manifest.toml index 004ef7892c173..ca2be57c61596 100644 --- a/test/project/Extensions/EnvWithHasExtensions/Manifest.toml +++ b/test/project/Extensions/EnvWithHasExtensions/Manifest.toml @@ -5,7 +5,7 @@ manifest_format = "2.0" project_hash = "a4c480cfa7da9610333d5c42623bf746bd286c5f" [[deps.ExtDep]] -deps = ["SomePackage"] +deps = ["SomePackage", "SomeOtherPackage"] path = "../ExtDep.jl" uuid = "fa069be4-f60b-4d4c-8b95-f8008775090c" version = "0.1.0" @@ -25,6 +25,11 @@ version = "0.1.0" ExtDep2 = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +[[deps.SomeOtherPackage]] +path = "../SomeOtherPackage" +uuid = "178f68a2-4498-45ee-a775-452b36359b63" +version = "0.1.0" + [[deps.SomePackage]] path = "../SomePackage" uuid = "678608ae-7bb3-42c7-98b1-82102067a3d8" diff --git a/test/project/Extensions/EnvWithHasExtensionsv2/Manifest.toml b/test/project/Extensions/EnvWithHasExtensionsv2/Manifest.toml index 66781a5701363..9f8c717041b6e 100644 --- a/test/project/Extensions/EnvWithHasExtensionsv2/Manifest.toml +++ b/test/project/Extensions/EnvWithHasExtensionsv2/Manifest.toml @@ -5,7 +5,7 @@ manifest_format = "2.0" project_hash = "caa716752e6dff3d77c3de929ebbb5d2024d04ef" [[deps.ExtDep]] -deps = ["SomePackage"] +deps = ["SomePackage", "SomeOtherPackage"] path = "../ExtDep.jl" uuid = "fa069be4-f60b-4d4c-8b95-f8008775090c" version = "0.1.0" @@ -19,6 +19,11 @@ weakdeps = ["ExtDep"] [deps.HasExtensions.extensions] Extension2 = "ExtDep" +[[deps.SomeOtherPackage]] +path = "../SomeOtherPackage" +uuid = "178f68a2-4498-45ee-a775-452b36359b63" +version = "0.1.0" + [[deps.SomePackage]] path = "../SomePackage" uuid = "678608ae-7bb3-42c7-98b1-82102067a3d8" diff --git a/test/project/Extensions/ExtDep.jl/Project.toml b/test/project/Extensions/ExtDep.jl/Project.toml index d246934b7f958..1ece7bf11f95a 100644 --- a/test/project/Extensions/ExtDep.jl/Project.toml +++ b/test/project/Extensions/ExtDep.jl/Project.toml @@ -4,3 +4,4 @@ version = "0.1.0" [deps] SomePackage = "678608ae-7bb3-42c7-98b1-82102067a3d8" +SomeOtherPackage = "178f68a2-4498-45ee-a775-452b36359b63" diff --git a/test/project/Extensions/ExtDep.jl/src/ExtDep.jl b/test/project/Extensions/ExtDep.jl/src/ExtDep.jl index 1c0022d879f51..2d3c6b7f28827 100644 --- a/test/project/Extensions/ExtDep.jl/src/ExtDep.jl +++ b/test/project/Extensions/ExtDep.jl/src/ExtDep.jl @@ -2,6 +2,7 @@ module ExtDep # loading this package makes the check for loading extensions trigger # which tests #47921 +using SomeOtherPackage using SomePackage struct ExtDepStruct end diff --git a/test/project/Extensions/ExtToExtDependency/Manifest.toml b/test/project/Extensions/ExtToExtDependency/Manifest.toml new file mode 100644 index 0000000000000..41546213cdd41 --- /dev/null +++ b/test/project/Extensions/ExtToExtDependency/Manifest.toml @@ -0,0 +1,21 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.11.1" +manifest_format = "2.0" +project_hash = "90b427e837c654fabb1434527ea698dabad46d29" + +[[deps.ExtDep]] +deps = ["SomeOtherPackage", "SomePackage"] +path = "../ExtDep.jl" +uuid = "fa069be4-f60b-4d4c-8b95-f8008775090c" +version = "0.1.0" + +[[deps.SomeOtherPackage]] +path = "../SomeOtherPackage" +uuid = "178f68a2-4498-45ee-a775-452b36359b63" +version = "0.1.0" + +[[deps.SomePackage]] +path = "../SomePackage" +uuid = "678608ae-7bb3-42c7-98b1-82102067a3d8" +version = "0.1.0" diff --git a/test/project/Extensions/ExtToExtDependency/Project.toml b/test/project/Extensions/ExtToExtDependency/Project.toml new file mode 100644 index 0000000000000..980db74c04dc4 --- /dev/null +++ b/test/project/Extensions/ExtToExtDependency/Project.toml @@ -0,0 +1,14 @@ +name = "ExtToExtDependency" +uuid = "594ddb71-72fb-4cfe-9471-775d48a5b70b" +version = "0.1.0" + +[deps] +ExtDep = "fa069be4-f60b-4d4c-8b95-f8008775090c" + +[weakdeps] +SomeOtherPackage = "178f68a2-4498-45ee-a775-452b36359b63" +SomePackage = "678608ae-7bb3-42c7-98b1-82102067a3d8" + +[extensions] +ExtA = ["SomePackage"] +ExtAB = ["SomePackage", "SomeOtherPackage"] diff --git a/test/project/Extensions/ExtToExtDependency/ext/ExtA.jl b/test/project/Extensions/ExtToExtDependency/ext/ExtA.jl new file mode 100644 index 0000000000000..71ed09795157c --- /dev/null +++ b/test/project/Extensions/ExtToExtDependency/ext/ExtA.jl @@ -0,0 +1,6 @@ +module ExtA + +using ExtToExtDependency +using SomePackage + +end diff --git a/test/project/Extensions/ExtToExtDependency/ext/ExtAB.jl b/test/project/Extensions/ExtToExtDependency/ext/ExtAB.jl new file mode 100644 index 0000000000000..a5b2c43cafd58 --- /dev/null +++ b/test/project/Extensions/ExtToExtDependency/ext/ExtAB.jl @@ -0,0 +1,12 @@ +module ExtAB + +using ExtToExtDependency +using SomePackage +using SomeOtherPackage + +const ExtA = Base.get_extension(ExtToExtDependency, :ExtA) +if !(ExtA isa Module) + error("expected extension to load") +end + +end diff --git a/test/project/Extensions/ExtToExtDependency/src/ExtToExtDependency.jl b/test/project/Extensions/ExtToExtDependency/src/ExtToExtDependency.jl new file mode 100644 index 0000000000000..ec2bf58f18641 --- /dev/null +++ b/test/project/Extensions/ExtToExtDependency/src/ExtToExtDependency.jl @@ -0,0 +1,7 @@ +module ExtToExtDependency + +using ExtDep + +greet() = print("Hello ext-to-ext!") + +end # module ExtToExtDependency diff --git a/test/project/Extensions/HasDepWithExtensions.jl/Manifest.toml b/test/project/Extensions/HasDepWithExtensions.jl/Manifest.toml index f659a59e0910b..98510dcb27733 100644 --- a/test/project/Extensions/HasDepWithExtensions.jl/Manifest.toml +++ b/test/project/Extensions/HasDepWithExtensions.jl/Manifest.toml @@ -5,7 +5,7 @@ manifest_format = "2.0" project_hash = "4e196b07f2ee7adc48ac9d528d42b3cf3737c7a0" [[deps.ExtDep]] -deps = ["SomePackage"] +deps = ["SomePackage", "SomeOtherPackage"] path = "../ExtDep.jl" uuid = "fa069be4-f60b-4d4c-8b95-f8008775090c" version = "0.1.0" @@ -32,6 +32,11 @@ weakdeps = ["ExtDep", "ExtDep2"] ExtensionDep = "ExtDep3" ExtensionFolder = ["ExtDep", "ExtDep2"] +[[deps.SomeOtherPackage]] +path = "../SomeOtherPackage" +uuid = "178f68a2-4498-45ee-a775-452b36359b63" +version = "0.1.0" + [[deps.SomePackage]] path = "../SomePackage" uuid = "678608ae-7bb3-42c7-98b1-82102067a3d8" diff --git a/test/project/Extensions/SomeOtherPackage/Project.toml b/test/project/Extensions/SomeOtherPackage/Project.toml new file mode 100644 index 0000000000000..6e7eee40c7be2 --- /dev/null +++ b/test/project/Extensions/SomeOtherPackage/Project.toml @@ -0,0 +1,4 @@ +name = "SomeOtherPackage" +uuid = "178f68a2-4498-45ee-a775-452b36359b63" +authors = ["Cody Tapscott "] +version = "0.1.0" diff --git a/test/project/Extensions/SomeOtherPackage/src/SomeOtherPackage.jl b/test/project/Extensions/SomeOtherPackage/src/SomeOtherPackage.jl new file mode 100644 index 0000000000000..ba23eb3914561 --- /dev/null +++ b/test/project/Extensions/SomeOtherPackage/src/SomeOtherPackage.jl @@ -0,0 +1,5 @@ +module SomeOtherPackage + +greet() = print("Hello World!") + +end # module SomeOtherPackage