diff --git a/src/codegen.cpp b/src/codegen.cpp index 04253b67d6b6d..540ee5ec74c4c 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -10050,29 +10050,19 @@ jl_llvm_functions_t jl_emit_codeinst( { JL_TIMING(CODEGEN, CODEGEN_Codeinst); jl_timing_show_method_instance(jl_get_ci_mi(codeinst), JL_TIMING_DEFAULT_BLOCK); - JL_GC_PUSH1(&src); if (!src) { - src = (jl_code_info_t*)jl_atomic_load_relaxed(&codeinst->inferred); jl_method_instance_t *mi = jl_get_ci_mi(codeinst); - jl_method_t *def = mi->def.method; - // Check if this is the generic method for opaque closure wrappers - - // if so, this must compile specptr such that it holds the specptr -> invoke wrapper + // Assert that this this is the generic method for opaque closure wrappers: + // this signals to instead compile specptr such that it holds the specptr -> invoke wrapper // to satisfy the dispatching implementation requirements of jl_f_opaque_closure_call - if (def == jl_opaque_closure_method) { - JL_GC_POP(); + if (mi->def.method == jl_opaque_closure_method) { return jl_emit_oc_wrapper(m, params, mi, codeinst->rettype); } - if (src && (jl_value_t*)src != jl_nothing && jl_is_method(def)) - src = jl_uncompress_ir(def, codeinst, (jl_value_t*)src); - if (!src || !jl_is_code_info(src)) { - JL_GC_POP(); - m = orc::ThreadSafeModule(); - return jl_llvm_functions_t(); // failed - } + m = orc::ThreadSafeModule(); + return jl_llvm_functions_t(); // user error } //assert(jl_egal((jl_value_t*)jl_atomic_load_relaxed(&codeinst->debuginfo), (jl_value_t*)src->debuginfo) && "trying to generate code for a codeinst for an incompatible src"); jl_llvm_functions_t decls = jl_emit_code(m, jl_get_ci_mi(codeinst), src, get_ci_abi(codeinst), params); - JL_GC_POP(); return decls; } diff --git a/src/gf.c b/src/gf.c index 390ba72143f5d..48cb8003072ca 100644 --- a/src/gf.c +++ b/src/gf.c @@ -337,6 +337,59 @@ jl_datatype_t *jl_mk_builtin_func(jl_datatype_t *dt, const char *name, jl_fptr_a return dt; } +// only relevant for bootstrapping. otherwise fairly broken. +static int emit_codeinst_and_edges(jl_code_instance_t *codeinst) +{ + jl_value_t *code = jl_atomic_load_relaxed(&codeinst->inferred); + if (code) { + if (jl_atomic_load_relaxed(&codeinst->invoke) != NULL) + return 1; + if (code != jl_nothing) { + JL_GC_PUSH1(&code); + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); + jl_method_t *def = mi->def.method; + if (jl_is_string(code) && jl_is_method(def)) + code = (jl_value_t*)jl_uncompress_ir(def, codeinst, (jl_value_t*)code); + if (jl_is_code_info(code)) { + jl_emit_codeinst_to_jit(codeinst, (jl_code_info_t*)code); + if (0) { + // next emit all the invoke edges too (if this seems profitable) + jl_array_t *src = ((jl_code_info_t*)code)->code; + for (size_t i = 0; i < jl_array_dim0(src); i++) { + jl_value_t *stmt = jl_array_ptr_ref(src, i); + if (jl_is_expr(stmt) && ((jl_expr_t*)stmt)->head == jl_assign_sym) + stmt = jl_exprarg(stmt, 1); + if (jl_is_expr(stmt) && ((jl_expr_t*)stmt)->head == jl_invoke_sym) { + jl_value_t *invoke = jl_exprarg(stmt, 0); + if (jl_is_code_instance(invoke)) + emit_codeinst_and_edges((jl_code_instance_t*)invoke); + } + } + } + JL_GC_POP(); + return 1; + } + JL_GC_POP(); + } + } + return 0; +} + +// Opportunistic SOURCE_MODE_ABI cache lookup, only for bootstrapping. +static jl_code_instance_t *jl_method_inferred_with_abi(jl_method_instance_t *mi JL_PROPAGATES_ROOT, size_t world) +{ + jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&mi->cache); + for (; codeinst; codeinst = jl_atomic_load_relaxed(&codeinst->next)) { + if (codeinst->owner != jl_nothing) + continue; + if (jl_atomic_load_relaxed(&codeinst->min_world) <= world && world <= jl_atomic_load_relaxed(&codeinst->max_world)) { + if (emit_codeinst_and_edges(codeinst)) + return codeinst; + } + } + return NULL; +} + // run type inference on lambda "mi" for given argument types. // returns the inferred source, and may cache the result in mi // if successful, also updates the mi argument to describe the validity of this src @@ -2571,23 +2624,6 @@ jl_code_instance_t *jl_method_compiled(jl_method_instance_t *mi, size_t world) return NULL; } -// Opportunistic SOURCE_MODE_ABI cache lookup. -jl_code_instance_t *jl_method_inferred_with_abi(jl_method_instance_t *mi JL_PROPAGATES_ROOT, size_t world) -{ - jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&mi->cache); - for (; codeinst; codeinst = jl_atomic_load_relaxed(&codeinst->next)) { - if (codeinst->owner != jl_nothing) - continue; - - if (jl_atomic_load_relaxed(&codeinst->min_world) <= world && world <= jl_atomic_load_relaxed(&codeinst->max_world)) { - jl_value_t *code = jl_atomic_load_relaxed(&codeinst->inferred); - if (code && (code != jl_nothing || (jl_atomic_load_relaxed(&codeinst->invoke) != NULL))) - return codeinst; - } - } - return NULL; -} - jl_mutex_t precomp_statement_out_lock; _Atomic(uint8_t) jl_force_trace_compile_timing_enabled = 0; diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 395f60644c53e..6b19df93f2a56 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -794,47 +794,6 @@ void jl_emit_codeinst_to_jit_impl( emittedmodules[codeinst] = std::move(result_m); } -static void recursive_compile_graph( - jl_code_instance_t *codeinst, - jl_code_info_t *src) -{ - jl_emit_codeinst_to_jit(codeinst, src); - DenseSet Seen; - SmallVector workqueue; - workqueue.push_back(codeinst); - // if any edges were incomplete, try to complete them now - while (!workqueue.empty()) { - auto this_code = workqueue.pop_back_val(); - if (Seen.insert(this_code).second) { - jl_code_instance_t *compiled_ci = jl_method_compiled_egal(codeinst); - if (!compiled_ci) { - if (this_code != codeinst) { - JL_GC_PROMISE_ROOTED(this_code); // rooted transitively from following edges from original argument - jl_emit_codeinst_to_jit(this_code, nullptr); // contains safepoints - } - jl_unique_gcsafe_lock lock(engine_lock); - auto edges = complete_graph.find(this_code); - if (edges != complete_graph.end()) { - workqueue.append(edges->second); - } - } - } - } -} - -// this generates llvm code for the lambda info -// and adds the result to the jitlayers -// (and the shadow module), -// and generates code for it -static void _jl_compile_codeinst( - jl_code_instance_t *codeinst, - jl_code_info_t *src) -{ - recursive_compile_graph(codeinst, src); - jl_compile_codeinst_now(codeinst); - assert(jl_is_compiled_codeinst(codeinst)); -} - const char *jl_generate_ccallable(Module *llvmmod, void *sysimg_handle, jl_value_t *declrt, jl_value_t *sigt, jl_codegen_params_t ¶ms); @@ -858,7 +817,6 @@ int jl_compile_extern_c_impl(LLVMOrcThreadSafeModuleRef llvmmod, void *p, void * orc::ThreadSafeModule backing; bool success = true; const char *name = ""; - SmallVector dependencies; if (into == NULL) { ctx = pparams ? pparams->tsctx : jl_ExecutionEngine->makeContext(); backing = jl_create_ts_module("cextern", ctx, DL, TargetTriple); @@ -886,11 +844,16 @@ int jl_compile_extern_c_impl(LLVMOrcThreadSafeModuleRef llvmmod, void *p, void * } params.tsctx_lock = params.tsctx.getLock(); // re-acquire lock if (success && params.cache) { - for (auto &it : params.workqueue) { + size_t newest_world = jl_atomic_load_acquire(&jl_world_counter); + for (auto &it : params.workqueue) { // really just zero or one, and just the ABI not the rest of the metadata jl_code_instance_t *codeinst = it.first; JL_GC_PROMISE_ROOTED(codeinst); - dependencies.push_back(codeinst); - recursive_compile_graph(codeinst, nullptr); + jl_code_instance_t *newest_ci = jl_type_infer(jl_get_ci_mi(codeinst), newest_world, SOURCE_MODE_ABI); + if (newest_ci) { + if (jl_egal(codeinst->rettype, newest_ci->rettype)) + it.first = codeinst; + jl_compile_codeinst_now(newest_ci); + } } jl_analyze_workqueue(nullptr, params, true); assert(params.workqueue.empty()); @@ -903,8 +866,6 @@ int jl_compile_extern_c_impl(LLVMOrcThreadSafeModuleRef llvmmod, void *p, void * { // lock scope jl_unique_gcsafe_lock lock(extern_c_lock); if (!jl_ExecutionEngine->getGlobalValueAddress(name)) { - for (auto dep : dependencies) - jl_compile_codeinst_now(dep); { auto Lock = backing.getContext().getLock(); jl_ExecutionEngine->optimizeDLSyms(*backing.getModuleUnlocked()); // safepoint @@ -975,7 +936,7 @@ int jl_compile_codeinst_impl(jl_code_instance_t *ci) if (!jl_is_compiled_codeinst(ci)) { ++SpecFPtrCount; uint64_t start = jl_typeinf_timing_begin(); - _jl_compile_codeinst(ci, NULL); + jl_compile_codeinst_now(ci); jl_typeinf_timing_end(start, 0); newly_compiled = 1; } @@ -1006,8 +967,7 @@ void jl_generate_fptr_for_unspecialized_impl(jl_code_instance_t *unspec) } else { jl_method_instance_t *mi = jl_get_ci_mi(unspec); - jl_code_instance_t *uninferred = jl_cached_uninferred( - jl_atomic_load_relaxed(&mi->cache), 1); + jl_code_instance_t *uninferred = jl_cached_uninferred(jl_atomic_load_relaxed(&mi->cache), 1); assert(uninferred); src = (jl_code_info_t*)jl_atomic_load_relaxed(&uninferred->inferred); assert(src); @@ -1018,10 +978,16 @@ void jl_generate_fptr_for_unspecialized_impl(jl_code_instance_t *unspec) if (!jl_is_compiled_codeinst(unspec)) { assert(jl_is_code_info(src)); ++UnspecFPtrCount; + jl_svec_t *edges = (jl_svec_t*)src->edges; + if (jl_is_svec(edges)) { + jl_atomic_store_release(&unspec->edges, edges); // n.b. this assumes the field was always empty svec(), which is not entirely true + jl_gc_wb(unspec, edges); + } jl_debuginfo_t *debuginfo = src->debuginfo; jl_atomic_store_release(&unspec->debuginfo, debuginfo); // n.b. this assumes the field was previously NULL, which is not entirely true jl_gc_wb(unspec, debuginfo); - _jl_compile_codeinst(unspec, src); + jl_emit_codeinst_to_jit(unspec, src); + jl_compile_codeinst_now(unspec); } JL_UNLOCK(&jitlock); // Might GC } diff --git a/src/julia_internal.h b/src/julia_internal.h index 7e91e23b9087d..00103e9b00a48 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -673,7 +673,7 @@ JL_DLLEXPORT void jl_engine_fulfill(jl_code_instance_t *ci, jl_code_info_t *src) void jl_engine_sweep(jl_ptls_t *gc_all_tls_states) JL_NOTSAFEPOINT; int jl_engine_hasreserved(jl_method_instance_t *m, jl_value_t *owner) JL_NOTSAFEPOINT; -JL_DLLEXPORT jl_code_instance_t *jl_type_infer(jl_method_instance_t *li, size_t world, uint8_t source_mode); +JL_DLLEXPORT jl_code_instance_t *jl_type_infer(jl_method_instance_t *li JL_PROPAGATES_ROOT, size_t world, uint8_t source_mode); JL_DLLEXPORT jl_code_info_t *jl_gdbcodetyped1(jl_method_instance_t *mi, size_t world); JL_DLLEXPORT jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *meth JL_PROPAGATES_ROOT, size_t world); JL_DLLEXPORT jl_code_instance_t *jl_get_method_inferred( @@ -1210,7 +1210,6 @@ jl_method_instance_t *jl_get_specialized(jl_method_t *m, jl_value_t *types, jl_s JL_DLLEXPORT jl_value_t *jl_rettype_inferred(jl_value_t *owner, jl_method_instance_t *li JL_PROPAGATES_ROOT, size_t min_world, size_t max_world); JL_DLLEXPORT jl_value_t *jl_rettype_inferred_native(jl_method_instance_t *mi, size_t min_world, size_t max_world) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_code_instance_t *jl_method_compiled(jl_method_instance_t *mi JL_PROPAGATES_ROOT, size_t world) JL_NOTSAFEPOINT; -JL_DLLEXPORT jl_code_instance_t *jl_method_inferred_with_abi(jl_method_instance_t *mi JL_PROPAGATES_ROOT, size_t world) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_methtable_t *mt JL_PROPAGATES_ROOT, jl_value_t *type, size_t world); JL_DLLEXPORT jl_method_instance_t *jl_specializations_get_linfo( jl_method_t *m JL_PROPAGATES_ROOT, jl_value_t *type, jl_svec_t *sparams); diff --git a/src/opaque_closure.c b/src/opaque_closure.c index 8a9dcac30a4f8..2d11d763be662 100644 --- a/src/opaque_closure.c +++ b/src/opaque_closure.c @@ -118,8 +118,10 @@ static jl_opaque_closure_t *new_opaque_closure(jl_tupletype_t *argt, jl_value_t // OC wrapper methods are not world dependent and have no edges or other info ci = jl_get_method_inferred(mi_generic, selected_rt, 1, ~(size_t)0, NULL, NULL); - if (!jl_atomic_load_acquire(&ci->invoke)) - jl_compile_codeinst(ci); // confusing this actually calls jl_emit_oc_wrapper and never actually compiles ci (which would be impossible since it cannot have source) + if (!jl_atomic_load_acquire(&ci->invoke)) { + jl_emit_codeinst_to_jit(ci, NULL); // confusing this actually calls jl_emit_oc_wrapper and never actually compiles ci (which would be impossible since it cannot have source) + jl_compile_codeinst(ci); + } specptr = jl_atomic_load_relaxed(&ci->specptr.fptr); } jl_opaque_closure_t *oc = (jl_opaque_closure_t*)jl_gc_alloc(ct->ptls, sizeof(jl_opaque_closure_t), oc_type);