use a function in pkg precompile instead of writing code to stdin (#36028)

Author: JeffBezanson

base/loading.jl

@@ -1200,66 +1200,73 @@ function load_path_setup_code(load_path::Bool=true)
     return code
 end
 
+# this is called in the external process that generates precompiled package files
+function include_package_for_output(input::String, depot_path::Vector{String}, dl_load_path::Vector{String}, load_path::Vector{String}, concrete_deps::typeof(_concrete_dependencies), uuid_tuple::NTuple{2,UInt64}, source::Union{Nothing,String})
+    append!(empty!(Base.DEPOT_PATH), depot_path)
+    append!(empty!(Base.DL_LOAD_PATH), dl_load_path)
+    append!(empty!(Base.LOAD_PATH), load_path)
+    ENV["JULIA_LOAD_PATH"] = join(load_path, Sys.iswindows() ? ';' : ':')
+    Base.HOME_PROJECT[] = Base.ACTIVE_PROJECT[] = nothing
+    Base._track_dependencies[] = true
+    append!(empty!(Base._concrete_dependencies), concrete_deps)
+
+    ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, uuid_tuple)
+    if source !== nothing
+        task_local_storage()[:SOURCE_PATH] = source
+    end
+
+    try
+        Base.include(Base.__toplevel__, input)
+    catch ex
+        precompilableerror(ex) || rethrow()
+        @debug "Aborting `create_expr_cache'" exception=(ErrorException("Declaration of __precompile__(false) not allowed"), catch_backtrace())
+        exit(125) # we define status = 125 means PrecompileableError
+    end
+end
+
+@assert precompile(include_package_for_output, (String,Vector{String},Vector{String},Vector{String},typeof(_concrete_dependencies),NTuple{2,UInt64},Nothing))
+@assert precompile(include_package_for_output, (String,Vector{String},Vector{String},Vector{String},typeof(_concrete_dependencies),NTuple{2,UInt64},String))
+
 function create_expr_cache(input::String, output::String, concrete_deps::typeof(_concrete_dependencies), uuid::Union{Nothing,UUID})
     rm(output, force=true)   # Remove file if it exists
-    code_object = """
-        while !eof(stdin)
-            code = readuntil(stdin, '\\0')
-            eval(Meta.parse(code))
+    depot_path = map(abspath, DEPOT_PATH)
+    dl_load_path = map(abspath, DL_LOAD_PATH)
+    load_path = map(abspath, Base.load_path())
+    path_sep = Sys.iswindows() ? ';' : ':'
+    any(path -> path_sep in path, load_path) &&
+        error("LOAD_PATH entries cannot contain $(repr(path_sep))")
+
+    deps_strs = String[]
+    for (pkg, build_id) in concrete_deps
+        pkg_str = if pkg.uuid === nothing
+            "Base.PkgId($(repr(pkg.name)))"
+        else
+            "Base.PkgId(Base.UUID(\"$(pkg.uuid)\"), $(repr(pkg.name)))"
         end
-        """
+        push!(deps_strs, "$pkg_str => $(repr(build_id))")
+    end
+    deps = repr(eltype(concrete_deps)) * "[" * join(deps_strs, ",") * "]"
+
+    uuid_tuple = uuid === nothing ? (UInt64(0), UInt64(0)) : convert(NTuple{2, UInt64}, uuid)
 
     io = open(pipeline(`$(julia_cmd()) -O0
                        --output-ji $output --output-incremental=yes
                        --startup-file=no --history-file=no --warn-overwrite=yes
                        --color=$(have_color === nothing ? "auto" : have_color ? "yes" : "no")
-                       --eval $code_object`, stderr=stderr),
+                       --eval 'eval(Meta.parse(read(stdin,String)))'`, stderr=stderr),
               "w", stdout)
-    in = io.in
-    try
-        write(in, """
-            begin
-                $(Base.load_path_setup_code())
-                Base._track_dependencies[] = true
-                Base.empty!(Base._concrete_dependencies)
-            """)
-        for (pkg, build_id) in concrete_deps
-            pkg_str = if pkg.uuid === nothing
-                "Base.PkgId($(repr(pkg.name)))"
-            else
-                "Base.PkgId(Base.UUID(\"$(pkg.uuid)\"), $(repr(pkg.name)))"
-            end
-            write(in, "Base.push!(Base._concrete_dependencies, $pkg_str => $(repr(build_id)))\n")
-        end
-        write(io, "end\0")
-        uuid_tuple = uuid === nothing ? (0, 0) : convert(NTuple{2, UInt64}, uuid)
-        write(in, "ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, $uuid_tuple)\0")
-        source = source_path(nothing)
-        if source !== nothing
-            write(in, "task_local_storage()[:SOURCE_PATH] = $(repr(source))\0")
-        end
-        write(in, """
-            try
-                Base.include(Base.__toplevel__, $(repr(abspath(input))))
-            catch ex
-                Base.precompilableerror(ex) || Base.rethrow()
-                Base.@debug "Aborting `createexprcache'" exception=(Base.ErrorException("Declaration of __precompile__(false) not allowed"), Base.catch_backtrace())
-                Base.exit(125) # we define status = 125 means PrecompileableError
-            end\0""")
-        # TODO: cleanup is probably unnecessary here
-        if source !== nothing
-            write(in, "delete!(task_local_storage(), :SOURCE_PATH)\0")
-        end
-        write(in, "ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, (0, 0))\0")
-        close(in)
-    catch
-        close(in)
-        process_running(io) && Timer(t -> kill(io), 5.0) # wait a short time before killing the process to give it a chance to clean up on its own first
-        rethrow()
-    end
+    # write data over stdin to avoid the (unlikely) case of exceeding max command line size
+    write(io.in, """
+        Base.include_package_for_output($(repr(abspath(input))), $(repr(depot_path)), $(repr(dl_load_path)),
+            $(repr(load_path)), $deps, $(repr(uuid_tuple)), $(repr(source_path(nothing))))
+        """)
+    close(io.in)
     return io
 end
 
+@assert precompile(create_expr_cache, (String, String, typeof(_concrete_dependencies), Nothing))
+@assert precompile(create_expr_cache, (String, String, typeof(_concrete_dependencies), UUID))
+
 function compilecache_path(pkg::PkgId)::String
     entrypath, entryfile = cache_file_entry(pkg)
     cachepath = joinpath(DEPOT_PATH[1], entrypath)