diff --git a/base/loading.jl b/base/loading.jl index c1972907b02bb..744d2274dff0c 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1043,7 +1043,7 @@ const TIMING_IMPORTS = Threads.Atomic{Int}(0) # these return either the array of modules loaded from the path / content given # or an Exception that describes why it couldn't be loaded # and it reconnects the Base.Docs.META -function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{Nothing, String}, depmods::Vector{Any}) +function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{Nothing, String}, depmods::Vector{Any}, skip_verify_edges::Int=0) assert_havelock(require_lock) timing_imports = TIMING_IMPORTS[] > 0 try @@ -1055,10 +1055,10 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No if ocachepath !== nothing @debug "Loading object cache file $ocachepath for $pkg" - sv = ccall(:jl_restore_package_image_from_file, Any, (Cstring, Any, Cint, Cstring), ocachepath, depmods, false, pkg.name) + sv = ccall(:jl_restore_package_image_from_file, Any, (Cstring, Any, Cint, Cstring, Cint), ocachepath, depmods, false, pkg.name, skip_verify_edges) else @debug "Loading cache file $path for $pkg" - sv = ccall(:jl_restore_incremental, Any, (Cstring, Any, Cint, Cstring), path, depmods, false, pkg.name) + sv = ccall(:jl_restore_incremental, Any, (Cstring, Any, Cint, Cstring, Cint), path, depmods, false, pkg.name, skip_verify_edges) end if isa(sv, Exception) return sv diff --git a/src/julia.h b/src/julia.h index 1f85f06dd35dc..90c96aeab77b2 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1876,7 +1876,7 @@ JL_DLLEXPORT void jl_set_sysimg_so(void *handle); JL_DLLEXPORT void jl_create_system_image(void **, jl_array_t *worklist, bool_t emit_split, ios_t **s, ios_t **z, jl_array_t **udeps, int64_t *srctextpos); JL_DLLEXPORT void jl_restore_system_image(const char *fname); JL_DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len); -JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *depmods, int complete, const char *pkgimage); +JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *depmods, int complete, const char *pkgimage, int skip_verify_edges); JL_DLLEXPORT void jl_set_newly_inferred(jl_value_t *newly_inferred); JL_DLLEXPORT void jl_push_newly_inferred(jl_value_t *ci); diff --git a/src/staticdata.c b/src/staticdata.c index c130c2e7569fe..70cb12b332f08 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -3393,7 +3393,7 @@ static jl_value_t *jl_validate_cache_file(ios_t *f, jl_array_t *depmods, uint64_ } // TODO?: refactor to make it easier to create the "package inspector" -static jl_value_t *jl_restore_package_image_from_stream(void* pkgimage_handle, ios_t *f, jl_image_t *image, jl_array_t *depmods, int completeinfo, const char *pkgname, bool needs_permalloc) +static jl_value_t *jl_restore_package_image_from_stream(void* pkgimage_handle, ios_t *f, jl_image_t *image, jl_array_t *depmods, int completeinfo, const char *pkgname, bool needs_permalloc, int skip_verify_edges) { JL_TIMING(LOAD_IMAGE, LOAD_Pkgimg); jl_timing_printf(JL_TIMING_DEFAULT_BLOCK, pkgname); @@ -3446,7 +3446,7 @@ static jl_value_t *jl_restore_package_image_from_stream(void* pkgimage_handle, i jl_copy_roots(method_roots_list, jl_worklist_key((jl_array_t*)restored)); // Handle edges size_t world = jl_atomic_load_acquire(&jl_world_counter); - jl_insert_backedges((jl_array_t*)edges, (jl_array_t*)ext_targets, (jl_array_t*)new_specializations, world); // restore external backedges (needs to be last) + jl_insert_backedges((jl_array_t*)edges, (jl_array_t*)ext_targets, (jl_array_t*)new_specializations, world, skip_verify_edges); // restore external backedges (needs to be last) // reinit ccallables jl_reinit_ccallable(&ccallable_list, base, pkgimage_handle); arraylist_free(&ccallable_list); @@ -3479,16 +3479,16 @@ static void jl_restore_system_image_from_stream(ios_t *f, jl_image_t *image, uin jl_restore_system_image_from_stream_(f, image, NULL, checksum | ((uint64_t)0xfdfcfbfa << 32), NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } -JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(void* pkgimage_handle, const char *buf, jl_image_t *image, size_t sz, jl_array_t *depmods, int completeinfo, const char *pkgname, bool needs_permalloc) +JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(void* pkgimage_handle, const char *buf, jl_image_t *image, size_t sz, jl_array_t *depmods, int completeinfo, const char *pkgname, bool needs_permalloc, int skip_verify_edges) { ios_t f; ios_static_buffer(&f, (char*)buf, sz); - jl_value_t *ret = jl_restore_package_image_from_stream(pkgimage_handle, &f, image, depmods, completeinfo, pkgname, needs_permalloc); + jl_value_t *ret = jl_restore_package_image_from_stream(pkgimage_handle, &f, image, depmods, completeinfo, pkgname, needs_permalloc, skip_verify_edges); ios_close(&f); return ret; } -JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *depmods, int completeinfo, const char *pkgname) +JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *depmods, int completeinfo, const char *pkgname, int skip_verify_edges) { ios_t f; if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) { @@ -3496,7 +3496,7 @@ JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *d "Cache file \"%s\" not found.\n", fname); } jl_image_t pkgimage = {}; - jl_value_t *ret = jl_restore_package_image_from_stream(NULL, &f, &pkgimage, depmods, completeinfo, pkgname, true); + jl_value_t *ret = jl_restore_package_image_from_stream(NULL, &f, &pkgimage, depmods, completeinfo, pkgname, true, skip_verify_edges); ios_close(&f); return ret; } @@ -3546,7 +3546,7 @@ JL_DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len) JL_SIGATOMIC_END(); } -JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, jl_array_t *depmods, int completeinfo, const char *pkgname) +JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, jl_array_t *depmods, int completeinfo, const char *pkgname, int skip_verify_edges) { void *pkgimg_handle = jl_dlopen(fname, JL_RTLD_LAZY); if (!pkgimg_handle) { @@ -3567,7 +3567,7 @@ JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, j jl_image_t pkgimage = jl_init_processor_pkgimg(pkgimg_handle); - jl_value_t* mod = jl_restore_incremental_from_buf(pkgimg_handle, pkgimg_data, &pkgimage, *plen, depmods, completeinfo, pkgname, false); + jl_value_t* mod = jl_restore_incremental_from_buf(pkgimg_handle, pkgimg_data, &pkgimage, *plen, depmods, completeinfo, pkgname, false, skip_verify_edges); return mod; } diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index b488934606671..cba2aa0914f59 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -1078,10 +1078,29 @@ static void jl_verify_graph(jl_array_t *edges, jl_array_t *maxvalids2) // Restore backedges to external targets // `edges` = [caller1, targets_indexes1, ...], the list of worklist-owned methods calling external methods. // `ext_targets` is [invokesig1, callee1, matches1, ...], the global set of non-worklist callees of worklist-owned methods. -static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_targets, jl_array_t *ci_list, size_t minworld) +static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_targets, jl_array_t *ci_list, size_t minworld, int skip_verify_edges) { - // determine which CodeInstance objects are still valid in our image - jl_array_t *valids = jl_verify_edges(ext_targets, minworld); + jl_array_t *valids; + if (skip_verify_edges == 1) { + // The RAI use case of caching JITted code via package images based on precompile + // statements from running engines was getting bottlenecked by the subtyping checks + // in jl_verify_edges. Since we're only loading new method instances and never introducing + // new methods in these packages, we want to avoid the expensive checks. + // The package image would be only created _and_ loaded by a Julia process with the same system + // image and same set of dependencies. + // Relevant issue: https://github.com/JuliaLang/julia/issues/48092 + size_t max_valid = ~(size_t)0; + static jl_value_t *ulong_array JL_ALWAYS_LEAFTYPE = NULL; + if (ulong_array == NULL) { + ulong_array = jl_apply_array_type((jl_value_t*)jl_ulong_type, 1); + } + size_t ntargets = jl_array_len(ext_targets) / 3; + valids = jl_alloc_array_1d(ulong_array, ntargets); + memset(jl_array_data(valids), max_valid, ntargets * sizeof(size_t)); + } else { + // determine which CodeInstance objects are still valid in our image + valids = jl_verify_edges(ext_targets, minworld); + } JL_GC_PUSH1(&valids); valids = jl_verify_methods(edges, valids); // consumes edges valids, initializes methods valids jl_verify_graph(edges, valids); // propagates methods valids for each edge diff --git a/test/precompile.jl b/test/precompile.jl index d317731b51370..27e14e161036b 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -1698,9 +1698,9 @@ precompile_test_harness("PkgCacheInspector") do load_path end if ocachefile !== nothing - sv = ccall(:jl_restore_package_image_from_file, Any, (Cstring, Any, Cint, Cstring), ocachefile, depmods, true, "PCI") + sv = ccall(:jl_restore_package_image_from_file, Any, (Cstring, Any, Cint, Cstring, Cint), ocachefile, depmods, true, "PCI", false) else - sv = ccall(:jl_restore_incremental, Any, (Cstring, Any, Cint, Cstring), cachefile, depmods, true, "PCI") + sv = ccall(:jl_restore_incremental, Any, (Cstring, Any, Cint, Cstring, Cint), cachefile, depmods, true, "PCI", false) end modules, init_order, external_methods, new_specializations, new_method_roots, external_targets, edges = sv