Make Compiler a stdlib

This is a further extension to #56128 to make the compiler into a proper
stdlib, useable outside of `Base` as `using Compiler` in the same way
that `JuliaSyntax` works already. There's a few remaining questions
around how loading works, but mechanically, this PR is complete.
For those remaining questions, I'll probably put up a separate PR
that would migrate JuliaSyntax to it and then simply adopt that
here once we've figured out the correct semantics.

base/Base.jl

@@ -380,6 +380,7 @@ a_method_to_overwrite_in_test() = inferencebarrier(1)
 
 # Compatibility with when Compiler was in Core
 @eval Core const Compiler = Main.Base.Compiler
+@eval Compiler const fl_parse = Core.Main.Base.fl_parse
 
 # External libraries vendored into Base
 Core.println("JuliaSyntax/src/JuliaSyntax.jl")

base/Base_compiler.jl

@@ -252,15 +252,14 @@ include("namedtuple.jl")
 include("ordering.jl")
 using .Order
 
-include("compiler/compiler.jl")
+include("coreir.jl")
+
+include("../stdlib/Compiler/src/Compiler.jl")
 
 const _return_type = Compiler.return_type
 
 # Enable compiler
-Core.eval(Compiler, quote
-include("compiler/bootstrap.jl")
-ccall(:jl_set_typeinf_func, Cvoid, (Any,), typeinf_ext_toplevel)
+Compiler.bootstrap!()
 
-include("compiler/parsing.jl")
+include("flparse.jl")
 Core._setparser!(fl_parse)
-end)

base/boot.jl

@@ -512,7 +512,6 @@ eval(Core, quote
     UpsilonNode(@nospecialize(val)) = $(Expr(:new, :UpsilonNode, :val))
     UpsilonNode() = $(Expr(:new, :UpsilonNode))
     Const(@nospecialize(v)) = $(Expr(:new, :Const, :v))
-    # NOTE the main constructor is defined within `Core.Compiler`
     _PartialStruct(@nospecialize(typ), fields::Array{Any, 1}) = $(Expr(:new, :PartialStruct, :typ, :fields))
     PartialOpaque(@nospecialize(typ), @nospecialize(env), parent::MethodInstance, source) = $(Expr(:new, :PartialOpaque, :typ, :env, :parent, :source))
     InterConditional(slot::Int, @nospecialize(thentype), @nospecialize(elsetype)) = $(Expr(:new, :InterConditional, :slot, :thentype, :elsetype))

base/coreir.jl

@@ -0,0 +1,54 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+Core.PhiNode() = Core.PhiNode(Int32[], Any[])
+
+"""
+    struct Const
+        val
+    end
+
+The type representing a constant value.
+"""
+Core.Const
+
+"""
+    struct PartialStruct
+        typ
+        fields::Vector{Any} # elements are other type lattice members
+    end
+
+This extended lattice element is introduced when we have information about an object's
+fields beyond what can be obtained from the object type. E.g. it represents a tuple where
+some elements are known to be constants or a struct whose `Any`-typed field is initialized
+with `Int` values.
+
+- `typ` indicates the type of the object
+- `fields` holds the lattice elements corresponding to each field of the object
+
+If `typ` is a struct, `fields` represents the fields of the struct that are guaranteed to be
+initialized. For instance, if the length of `fields` of `PartialStruct` representing a
+struct with 4 fields is 3, the 4th field may not be initialized. If the length is 4, all
+fields are guaranteed to be initialized.
+
+If `typ` is a tuple, the last element of `fields` may be `Vararg`. In this case, it is
+guaranteed that the number of elements in the tuple is at least `length(fields)-1`, but the
+exact number of elements is unknown.
+"""
+Core.PartialStruct
+
+"""
+    struct InterConditional
+        slot::Int
+        thentype
+        elsetype
+    end
+
+Similar to `Conditional`, but conveys inter-procedural constraints imposed on call arguments.
+This is separate from `Conditional` to catch logic errors: the lattice element name is `InterConditional`
+while processing a call, then `Conditional` everywhere else. Thus `InterConditional` does not appear in
+`CompilerTypes`—these type's usages are disjoint—though we define the lattice for `InterConditional`.
+"""
+Core.InterConditional
+
+InterConditional(var::SlotNumber, @nospecialize(thentype), @nospecialize(elsetype)) =
+    InterConditional(slot_id(var), thentype, elsetype)

base/show.jl

@@ -2856,7 +2856,7 @@ module IRShow
         Effects, ALWAYS_TRUE, ALWAYS_FALSE, DebugInfoStream, getdebugidx,
         VarState, InvalidIRError, argextype, widenconst, singleton_type,
         sptypes_from_meth_instance, EMPTY_SPTYPES
-    include("compiler/ssair/show.jl")
+    include("../stdlib/Compiler/src/ssair/show.jl")
 
     const __debuginfo = Dict{Symbol, Any}(
         # :full => src -> statementidx_lineinfo_printer(src), # and add variable slot information

src/toplevel.c

@@ -1289,6 +1289,21 @@ JL_DLLEXPORT jl_value_t *jl_prepend_cwd(jl_value_t *str)
     return jl_cstr_to_string(path);
 }
 
+JL_DLLEXPORT jl_value_t *jl_prepend_string(jl_value_t *prefix, jl_value_t *str)
+{
+    char path[1024];
+    const char *pstr = (const char*)jl_string_data(prefix);
+    size_t sz = strlen(pstr);
+    const char *fstr = (const char*)jl_string_data(str);
+    if (strlen(fstr) + sz >= sizeof(path)) {
+        jl_errorf("use a bigger buffer for jl_fullpath");
+    }
+    strcpy(path, pstr);
+    strcpy(path + sz, fstr);
+    return jl_cstr_to_string(path);
+}
+
+
 #ifdef __cplusplus
 }
 #endif

stdlib/Compiler/Project.toml

@@ -0,0 +1,3 @@
+name = "Compiler"
+uuid = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1"
+version = "0.0.1"

base/compiler/compiler.jl → stdlib/Compiler/src/Compiler.jl

@@ -12,8 +12,8 @@ import Core: print, println, show, write, unsafe_write,
              memoryref_isassigned, memoryrefnew, memoryrefoffset, memoryrefget,
              memoryrefset!, typename
 
-using ..Base
-using ..Base: Ordering, vect, EffectsOverride, BitVector, @_gc_preserve_begin, @_gc_preserve_end, RefValue,
+using Base
+using Base: Ordering, vect, EffectsOverride, BitVector, @_gc_preserve_begin, @_gc_preserve_end, RefValue,
     @nospecializeinfer, @_foldable_meta, fieldindex, is_function_def, indexed_iterate, isexpr, methods,
     get_world_counter, JLOptions, _methods_by_ftype, unwrap_unionall, cconvert, unsafe_convert,
     issingletontype, isType, rewrap_unionall, has_free_typevars, isvarargtype, hasgenerator,
@@ -29,8 +29,8 @@ using ..Base: Ordering, vect, EffectsOverride, BitVector, @_gc_preserve_begin, @
     specialize_method, hasintersect, is_nospecializeinfer, is_nospecialized,
     get_nospecializeinfer_sig, tls_world_age, uniontype_layout, kwerr,
     moduleroot, is_file_tracked, decode_effects_override
-using ..Base.Order
-import ..Base: getindex, setindex!, length, iterate, push!, isempty, first, convert, ==,
+using Base.Order
+import Base: getindex, setindex!, length, iterate, push!, isempty, first, convert, ==,
     copy, popfirst!, in, haskey, resize!, copy!, append!, last, get!, size,
     get, iterate, findall
 
@@ -46,8 +46,21 @@ ccall(:jl_set_istopmod, Cvoid, (Any, Bool), Compiler, false)
 eval(x) = Core.eval(Compiler, x)
 eval(m, x) = Core.eval(m, x)
 
-include(x) = Base.include(Compiler, x)
-include(mod, x) = Base.include(mod, x)
+function include(x::String)
+    if !isdefined(Base, :end_base_include)
+        # During bootstrap, all includes are relative to `base/`
+        x = ccall(:jl_prepend_string, Ref{String}, (Any, Any), "../stdlib/Compiler/src/", x)
+    end
+    Base.include(Compiler, x)
+end
+
+function include(mod::Module, x::String)
+    if !isdefined(Base, :end_base_include)
+        x = ccall(:jl_prepend_string, Ref{String}, (Any, Any), "../stdlib/Compiler/src/", x)
+    end
+    Base.include(mod, x)
+end
+
 
 macro _boundscheck() Expr(:boundscheck) end
 
@@ -59,7 +72,7 @@ abstract type AbstractInterpreter end
 function return_type end # promotion.jl expects this to exist
 is_return_type(Core.@nospecialize(f)) = f === return_type
 
-include("compiler/sort.jl")
+include("sort.jl")
 
 # We don't include some.jl, but this definition is still useful.
 something(x::Nothing, y...) = something(y...)
@@ -94,30 +107,32 @@ else
     end
 end
 
-include("compiler/cicache.jl")
-include("compiler/methodtable.jl")
-include("compiler/effects.jl")
-include("compiler/types.jl")
-include("compiler/utilities.jl")
-include("compiler/validation.jl")
-
-include("compiler/ssair/basicblock.jl")
-include("compiler/ssair/domtree.jl")
-include("compiler/ssair/ir.jl")
-include("compiler/ssair/tarjan.jl")
-
-include("compiler/abstractlattice.jl")
-include("compiler/stmtinfo.jl")
-include("compiler/inferenceresult.jl")
-include("compiler/inferencestate.jl")
-
-include("compiler/typeutils.jl")
-include("compiler/typelimits.jl")
-include("compiler/typelattice.jl")
-include("compiler/tfuncs.jl")
-
-include("compiler/abstractinterpretation.jl")
-include("compiler/typeinfer.jl")
-include("compiler/optimize.jl")
+include("cicache.jl")
+include("methodtable.jl")
+include("effects.jl")
+include("types.jl")
+include("utilities.jl")
+include("validation.jl")
+
+include("ssair/basicblock.jl")
+include("ssair/domtree.jl")
+include("ssair/ir.jl")
+include("ssair/tarjan.jl")
+
+include("abstractlattice.jl")
+include("stmtinfo.jl")
+include("inferenceresult.jl")
+include("inferencestate.jl")
+
+include("typeutils.jl")
+include("typelimits.jl")
+include("typelattice.jl")
+include("tfuncs.jl")
+
+include("abstractinterpretation.jl")
+include("typeinfer.jl")
+include("optimize.jl")
+
+include("bootstrap.jl")
 
 end

base/compiler/abstractinterpretation.jl → stdlib/Compiler/src/abstractinterpretation.jl

@@ -2119,7 +2119,7 @@ function form_partially_defined_struct(@nospecialize(obj), @nospecialize(name))
     else
         fldidx > nminfld || return nothing
     end
-    return PartialStruct(objt0, Any[obj isa PartialStruct && i≤length(obj.fields) ?
+    return PartialStruct(fallback_lattice, objt0, Any[obj isa PartialStruct && i≤length(obj.fields) ?
         obj.fields[i] : fieldtype(objt0,i) for i = 1:fldidx])
 end
 
@@ -2955,7 +2955,7 @@ function abstract_eval_new(interp::AbstractInterpreter, e::Expr, vtypes::Union{V
                 # - any refinement information is available (`anyrefine`), or when
                 # - `nargs` is greater than `n_initialized` derived from the struct type
                 #   information alone
-                rt = PartialStruct(rt, ats)
+                rt = PartialStruct(𝕃ᵢ, rt, ats)
             end
         else
             rt = refine_partial_type(rt)
@@ -2990,7 +2990,7 @@ function abstract_eval_splatnew(interp::AbstractInterpreter, e::Expr, vtypes::Un
                     all(i::Int -> ⊑(𝕃ᵢ, (at.fields::Vector{Any})[i], fieldtype(t, i)), 1:n)
                 end))
             nothrow = isexact
-            rt = PartialStruct(rt, at.fields::Vector{Any})
+            rt = PartialStruct(𝕃ᵢ, rt, at.fields::Vector{Any})
         end
     else
         rt = refine_partial_type(rt)
@@ -3527,7 +3527,7 @@ end
             end
             fields[i] = a
         end
-        anyrefine && return PartialStruct(rt.typ, fields)
+        anyrefine && return PartialStruct(𝕃ᵢ, rt.typ, fields)
     end
     if isa(rt, PartialOpaque)
         return rt # XXX: this case was missed in #39512

stdlib/Compiler/src/bootstrap.jl

@@ -0,0 +1,57 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+# make sure that typeinf is executed before turning on typeinf_ext
+# this ensures that typeinf_ext doesn't recurse before it can add the item to the workq
+# especially try to make sure any recursive and leaf functions have concrete signatures,
+# since we won't be able to specialize & infer them at runtime
+
+activate!() = ccall(:jl_set_typeinf_func, Cvoid, (Any,), typeinf_ext_toplevel)
+
+function bootstrap!()
+    let time() = ccall(:jl_clock_now, Float64, ())
+        println("Compiling the compiler. This may take several minutes ...")
+        interp = NativeInterpreter()
+
+        # analyze_escapes_tt = Tuple{typeof(analyze_escapes), IRCode, Int, TODO}
+        optimize_tt = Tuple{typeof(optimize), NativeInterpreter, OptimizationState{NativeInterpreter}, InferenceResult}
+        fs = Any[
+            # we first create caches for the optimizer, because they contain many loop constructions
+            # and they're better to not run in interpreter even during bootstrapping
+            #=analyze_escapes_tt,=# optimize_tt,
+            # then we create caches for inference entries
+            typeinf_ext, typeinf, typeinf_edge,
+        ]
+        # tfuncs can't be inferred from the inference entries above, so here we infer them manually
+        for x in T_FFUNC_VAL
+            push!(fs, x[3])
+        end
+        for i = 1:length(T_IFUNC)
+            if isassigned(T_IFUNC, i)
+                x = T_IFUNC[i]
+                push!(fs, x[3])
+            else
+                println(stderr, "WARNING: tfunc missing for ", reinterpret(IntrinsicFunction, Int32(i)))
+            end
+        end
+        starttime = time()
+        for f in fs
+            if isa(f, DataType) && f.name === typename(Tuple)
+                tt = f
+            else
+                tt = Tuple{typeof(f), Vararg{Any}}
+            end
+            for m in _methods_by_ftype(tt, 10, get_world_counter())::Vector
+                # remove any TypeVars from the intersection
+                m = m::MethodMatch
+                typ = Any[m.spec_types.parameters...]
+                for i = 1:length(typ)
+                    typ[i] = unwraptv(typ[i])
+                end
+                typeinf_type(interp, m.method, Tuple{typ...}, m.sparams)
+            end
+        end
+        endtime = time()
+        println("Base.Compiler ──── ", sub_float(endtime,starttime), " seconds")
+    end
+    activate!()
+end