diff --git a/doc/make.jl b/doc/make.jl index 1ea3d50f9f5b0..0ca31f155f940 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -87,12 +87,13 @@ const PAGES = [ "devdocs/reflection.md", "Documentation of Julia's Internals" => [ "devdocs/init.md", - "devdocs/eval.md", "devdocs/ast.md", "devdocs/types.md", "devdocs/object.md", - "devdocs/functions.md", + "devdocs/eval.md", "devdocs/callconv.md", + "devdocs/compiler.md", + "devdocs/functions.md", "devdocs/cartesian.md", "devdocs/meta.md", "devdocs/subarrays.md", diff --git a/doc/src/devdocs/compiler.md b/doc/src/devdocs/compiler.md new file mode 100644 index 0000000000000..5b4f0fd7fe3bb --- /dev/null +++ b/doc/src/devdocs/compiler.md @@ -0,0 +1,119 @@ +# High-level Overview of the Native-Code Generation Process + + + + + +## Representation of Pointers + +When emitting code to an object file, pointers will be emitted as relocations. +The deserialization code will ensure any object that pointed to one of these constants +gets recreated and contains the right runtime pointer. + +Otherwise, they will be emitted as literal constants. + +To emit one of these objects, call `literal_pointer_val`. +It'll handle tracking the Julia value and the LLVM global, +ensuring they are valid both for the current runtime and after deserialization. + +When emitted into the object file, these globals are stored as references +in a large `gvals` table. This allows the deserializer to reference them by index, +and implement a custom manual GOT-like mechanism to restore them. + +Function pointers are handled similarly. +They are stored as values in a large `fvals` table. +Like globals, this allows the deserializer to reference them by index. + +Note that extern functions are handled separately, +with names, via the usual symbol resolution mechanism in the linker. + +Note too that ccall functions are also handled separately, +via a manual GOT + PLT. + + +## Representation of Intermediate Values + +Values are passed around in a `jl_cgval_t` struct. +This represents an R-value, and includes enough information to +determine how to assign or pass it somewhere. + +They are created via one of the helper constructors, usually: +`mark_julia_type` (for immediate values) and `mark_julia_slot` (for pointers to values). + +The function `convert_julia_type` can transform between any two types. +It returns an R-value with `cgval.typ` set to `typ`. +It'll cast the object to the requested representation, +making heap boxes, allocating stack copies, and computing tagged unions as +needed to change the representation. + +By contrast `update_julia_type` will change `cgval.typ` to `typ`, +only if it can be done at zero-cost (i.e. without emitting any code). + + +## Union representation + +Inferred union types may be stack allocated via a tagged type representation. + +The primitive routines that need to be able to handle tagged unions are: +- mark-type +- load-local +- store-local +- isa +- is +- emit_typeof +- emit_sizeof +- boxed +- unbox +- specialized cc-ret + +Everything else should be possible to handle in inference by using these +primitives to implement union-splitting. + +The representation of the tagged-union is as a pair +of `< void* union, byte selector >`. +The selector is fixed-size as `byte & 0x7f`, +and will union-tag the first 126 isbits. +It records the one-based depth-first count into the type-union of the +isbits objects inside. An index of zero indicates that the `union*` is +actually a tagged heap-allocated `jl_value_t*`, +and needs to be treated as normal for a boxed object rather than as a +tagged union. + +The high bit of the selector (`byte & 0x80`) can be tested to determine if the +`void*` is actually a heap-allocated (`jl_value_t*`) box, +thus avoiding the cost of re-allocating a box, +while maintaining the ability to efficiently handle union-splitting based on the low bits. + +It is guaranteed that `byte & 0x7f` is an exact test for the type, +if the value can be represented by a tag – it will never be marked `byte = 0x80`. +It is not necessary to also test the type-tag when testing `isa`. + +The `union*` memory region may be allocated at *any* size. +The only constraint is that it is big enough to contain the data +currently specified by `selector`. +It might not be big enough to contain the union of all types that +could be stored there according to the associated Union type field. +Use appropriate care when copying. + + +## Specialized Calling Convention Signature Representation + +A `jl_returninfo_t` object describes the calling convention details of any callable. + +If any of the arguments or return type of a method can be represented unboxed, +and the method is not varargs, it'll be given an optimized calling convention +signature based on its `specTypes` and `rettype` fields. + +The general principles are that: + +- Primitive types get passed in int/float registers. +- Tuples of VecElement types get passed in vector registers. +- Structs get passed on the stack. +- Return values are handle similarly to arguments, + with a size-cutoff at which they will instead be returned via a hidden sret argument. + +The total logic for this is implemented by `get_specsig_function` and `deserves_sret`. + +Additionally, if the return type is a union, it may be returned as a pair of values (a pointer and a tag). +If the union values can be stack-allocated, then sufficient space to store them will also be passed as a hidden first argument. +It is up to the callee whether the returned pointer will point to this space, a boxed object, or even other constant memory. diff --git a/doc/src/index.md b/doc/src/index.md index a08246094d703..ff862c7d1955e 100644 --- a/doc/src/index.md +++ b/doc/src/index.md @@ -71,12 +71,13 @@ * [Reflection and introspection](@ref) * Documentation of Julia's Internals * [Initialization of the Julia runtime](@ref) - * [Eval of Julia code](@ref) * [Julia ASTs](@ref) * [More about types](@ref) * [Memory layout of Julia Objects](@ref) - * [Julia Functions](@ref) + * [Eval of Julia code](@ref) * [Calling Conventions](@ref) + * [High-level Overview of the Native-Code Generation Process](@ref) + * [Julia Functions](@ref) * [Base.Cartesian](@ref) * [Talking to the compiler (the `:meta` mechanism)](@ref) * [SubArrays](@ref) diff --git a/src/ccall.cpp b/src/ccall.cpp index 1e672d198efdf..a38ab2c112ccf 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -672,7 +672,7 @@ static void interpret_symbol_arg(native_sym_arg_t &out, jl_value_t *arg, jl_code "cglobal: first argument not a pointer or valid constant expression", ctx); } - arg1 = remark_julia_type(arg1, (jl_value_t*)jl_voidpointer_type, ctx); + arg1 = update_julia_type(arg1, (jl_value_t*)jl_voidpointer_type, ctx); jl_ptr = emit_unbox(T_size, arg1, (jl_value_t*)jl_voidpointer_type); } else { diff --git a/src/cgutils.cpp b/src/cgutils.cpp index b2e3d2a4b84c5..bf41d5e6ed557 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -224,7 +224,7 @@ static DIType julia_type_to_di(jl_value_t *jt, DIBuilder *dbuilder, bool isboxed // --- emitting pointers directly into code --- -static Value *literal_static_pointer_val(const void *p, Type *t) +static Constant *literal_static_pointer_val(const void *p, Type *t) { // this function will emit a static pointer into the generated code // the generated code will only be valid during the current session, @@ -237,14 +237,13 @@ static Value *literal_static_pointer_val(const void *p, Type *t) } -static Value *julia_gv(const char *cname, void *addr) +static Value *julia_pgv(const char *cname, void *addr) { // emit a GlobalVariable for a jl_value_t named "cname" - GlobalVariable *gv = jl_get_global_for(cname, addr, jl_builderModule); - return tbaa_decorate(tbaa_const, builder.CreateLoad(gv)); + return jl_get_global_for(cname, addr, jl_builderModule); } -static Value *julia_gv(const char *prefix, jl_sym_t *name, jl_module_t *mod, void *addr) +static Value *julia_pgv(const char *prefix, jl_sym_t *name, jl_module_t *mod, void *addr) { // emit a GlobalVariable for a jl_value_t, using the prefix, name, and module to // to create a readable name of the form prefixModA.ModB.name @@ -269,55 +268,75 @@ static Value *julia_gv(const char *prefix, jl_sym_t *name, jl_module_t *mod, voi prev = parent; parent = parent->parent; } - return julia_gv(fullname, addr); + return julia_pgv(fullname, addr); } static GlobalVariable *julia_const_gv(jl_value_t *val); -static Value *literal_pointer_val(jl_value_t *p) +static Value *literal_pointer_val_slot(jl_value_t *p) { - // emit a pointer to any jl_value_t which will be valid across reloading code + // emit a pointer to a jl_value_t* which will allow it to be valid across reloading code // also, try to give it a nice name for gdb, for easy identification - if (p == NULL) - return ConstantPointerNull::get((PointerType*)T_pjlvalue); - if (!imaging_mode) - return literal_static_pointer_val(p, T_pjlvalue); - if (auto gv = julia_const_gv(p)) { - return tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(gv))); + if (!imaging_mode) { + Module *M = jl_builderModule; + GlobalVariable *gv = new GlobalVariable( + *M, T_pjlvalue, true, GlobalVariable::PrivateLinkage, + literal_static_pointer_val(p, T_pjlvalue)); +#if JL_LLVM_VERSION >= 30900 + gv->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); +#else + gv->setUnnamedAddr(true); +#endif + return gv; + } + if (GlobalVariable *gv = julia_const_gv(p)) { + // if this is a known object, use the existing GlobalValue + return prepare_global(gv, jl_builderModule); } if (jl_is_datatype(p)) { jl_datatype_t *addr = (jl_datatype_t*)p; // DataTypes are prefixed with a + - return julia_gv("+", addr->name->name, addr->name->module, p); + return julia_pgv("+", addr->name->name, addr->name->module, p); } if (jl_is_method(p)) { jl_method_t *m = (jl_method_t*)p; // functions are prefixed with a - - return julia_gv("-", m->name, m->module, p); + return julia_pgv("-", m->name, m->module, p); } if (jl_is_method_instance(p)) { jl_method_instance_t *linfo = (jl_method_instance_t*)p; // Type-inferred functions are also prefixed with a - if (linfo->def) - return julia_gv("-", linfo->def->name, linfo->def->module, p); + return julia_pgv("-", linfo->def->name, linfo->def->module, p); } if (jl_is_symbol(p)) { jl_sym_t *addr = (jl_sym_t*)p; // Symbols are prefixed with jl_sym# - return julia_gv("jl_sym#", addr, NULL, p); + return julia_pgv("jl_sym#", addr, NULL, p); } // something else gets just a generic name - return julia_gv("jl_global#", p); + return julia_pgv("jl_global#", p); +} + +static Value *literal_pointer_val(jl_value_t *p) +{ + if (p == NULL) + return V_null; + if (!imaging_mode) + return literal_static_pointer_val(p, T_pjlvalue); + Value *pgv = literal_pointer_val_slot(p); + return tbaa_decorate(tbaa_const, builder.CreateLoad(pgv)); } static Value *literal_pointer_val(jl_binding_t *p) { // emit a pointer to any jl_value_t which will be valid across reloading code if (p == NULL) - return ConstantPointerNull::get((PointerType*)T_pjlvalue); + return V_null; if (!imaging_mode) return literal_static_pointer_val(p, T_pjlvalue); // bindings are prefixed with jl_bnd# - return julia_gv("jl_bnd#", p->name, p->owner, p); + Value *pgv = julia_pgv("jl_bnd#", p->name, p->owner, p); + return tbaa_decorate(tbaa_const, builder.CreateLoad(pgv)); } // bitcast a value, but preserve its address space when dealing with pointer types @@ -338,18 +357,22 @@ static Value *emit_bitcast(Value *v, Type *jl_value) static Value *julia_binding_gv(Value *bv) { - return builder. - CreateGEP(bv,ConstantInt::get(T_size, - offsetof(jl_binding_t,value)/sizeof(size_t))); + Value *offset = ConstantInt::get(T_size, offsetof(jl_binding_t, value) / sizeof(size_t)); + return builder.CreateGEP(bv, offset); } static Value *julia_binding_gv(jl_binding_t *b) { // emit a literal_pointer_val to the value field of a jl_binding_t // binding->value are prefixed with * - Value *bv = imaging_mode ? - emit_bitcast(julia_gv("*", b->name, b->owner, b), T_ppjlvalue) : - literal_static_pointer_val(b,T_ppjlvalue); + Value *bv; + if (imaging_mode) + bv = emit_bitcast( + tbaa_decorate(tbaa_const, + builder.CreateLoad(julia_pgv("*", b->name, b->owner, b))), + T_ppjlvalue); + else + bv = literal_static_pointer_val(b, T_ppjlvalue); return julia_binding_gv(bv); } @@ -558,6 +581,43 @@ static bool deserves_sret(jl_value_t *dt, Type *T) return (size_t)jl_datatype_size(dt) > sizeof(void*) && !T->isFloatingPointTy() && !T->isVectorTy(); } +static bool for_each_uniontype_small( + std::function f, + jl_value_t *ty, + unsigned &counter) +{ + if (counter > 127) + return false; + if (jl_is_uniontype(ty)) { + bool allunbox = for_each_uniontype_small(f, ((jl_uniontype_t*)ty)->a, counter); + allunbox &= for_each_uniontype_small(f, ((jl_uniontype_t*)ty)->b, counter); + return allunbox; + } + else if (isbits_spec(ty)) { + f(++counter, (jl_datatype_t*)ty); + return true; + } + return false; +} + +static Value *emit_typeof_boxed(const jl_cgval_t &p, jl_codectx_t *ctx); + +static unsigned get_box_tindex(jl_datatype_t *jt, jl_value_t *ut) +{ + unsigned new_idx = 0; + unsigned new_counter = 0; + for_each_uniontype_small( + // find the corresponding index in the new union-type + [&](unsigned new_idx_, jl_datatype_t *new_jt) { + if (jt == new_jt) + new_idx = new_idx_; + }, + ut, + new_counter); + return new_idx; +} + + // --- generating various field accessors --- static Value *emit_nthptr_addr(Value *v, ssize_t n) @@ -602,24 +662,60 @@ static Value *emit_typeptr_addr(Value *p) static Value *boxed(const jl_cgval_t &v, jl_codectx_t *ctx, bool gcooted=true); static Value *boxed(const jl_cgval_t &v, jl_codectx_t *ctx, jl_value_t* type) = delete; // C++11 (temporary to prevent rebase error) +static Value* mask_gc_bits(Value *tag) +{ + return builder.CreateIntToPtr(builder.CreateAnd( + builder.CreatePtrToInt(tag, T_size), + ConstantInt::get(T_size, ~(uintptr_t)15)), + tag->getType()); +} + static Value *emit_typeof(Value *tt) { + assert(tt != NULL && !isa(tt) && "expected a conditionally boxed value"); // given p, a jl_value_t*, compute its type tag - assert(tt->getType() == T_pjlvalue); - tt = tbaa_decorate(tbaa_tag, builder.CreateLoad(emit_typeptr_addr(tt), false)); - tt = builder.CreateIntToPtr(builder.CreateAnd( - builder.CreatePtrToInt(tt, T_size), - ConstantInt::get(T_size,~(uintptr_t)15)), - T_pjlvalue); - return tt; + tt = tbaa_decorate(tbaa_tag, builder.CreateLoad(emit_typeptr_addr(tt))); + return mask_gc_bits(tt); } static jl_cgval_t emit_typeof(const jl_cgval_t &p, jl_codectx_t *ctx) { // given p, compute its type - if (!p.constant && p.isboxed && !jl_is_leaf_type(p.typ)) { + if (p.constant) + return mark_julia_const(jl_typeof(p.constant)); + if (p.isboxed && !jl_is_leaf_type(p.typ)) { return mark_julia_type(emit_typeof(p.V), true, jl_datatype_type, ctx, /*needsroot*/false); } + if (p.TIndex) { + Value *tindex = builder.CreateAnd(p.TIndex, ConstantInt::get(T_int8, 0x7f)); + Value *pdatatype; + unsigned counter; + counter = 0; + bool allunboxed = for_each_uniontype_small( + [&](unsigned idx, jl_datatype_t *jt) { }, + p.typ, + counter); + if (allunboxed) + pdatatype = Constant::getNullValue(T_ppjlvalue); + else + pdatatype = emit_typeptr_addr(p.V); + counter = 0; + for_each_uniontype_small( + [&](unsigned idx, jl_datatype_t *jt) { + Value *cmp = builder.CreateICmpEQ(tindex, ConstantInt::get(T_int8, idx)); + pdatatype = builder.CreateSelect(cmp, literal_pointer_val_slot((jl_value_t*)jt), pdatatype); + }, + p.typ, + counter); + Value *datatype; + if (allunboxed) { + datatype = tbaa_decorate(tbaa_const, builder.CreateLoad(pdatatype)); + } + else { + datatype = mask_gc_bits(tbaa_decorate(tbaa_tag, builder.CreateLoad(pdatatype))); + } + return mark_julia_type(datatype, true, jl_datatype_type, ctx, /*needsroot*/false); + } jl_value_t *aty = p.typ; if (jl_is_type_type(aty)) { // convert Int::Type{Int} ==> typeof(Int) ==> DataType @@ -668,6 +764,55 @@ static Value *emit_datatype_size(Value *dt) return size; } +/* this is valid code, it's simply unused +static Value *emit_sizeof(const jl_cgval_t &p, jl_codectx_t *ctx) +{ + if (p.TIndex) { + Value *tindex = builder.CreateAnd(p.TIndex, ConstantInt::get(T_int8, 0x7f)); + Value *size = ConstantInt::get(T_int32, -1); + unsigned counter = 0; + bool allunboxed = for_each_uniontype_small( + [&](unsigned idx, jl_datatype_t *jt) { + Value *cmp = builder.CreateICmpEQ(tindex, ConstantInt::get(T_int8, idx)); + size = builder.CreateSelect(cmp, ConstantInt::get(T_int32, jl_datatype_size(jt)), size); + }, + p.typ, + counter); + if (!allunboxed && p.ispointer() && p.V && !isa(p.V)) { + BasicBlock *currBB = builder.GetInsertBlock(); + BasicBlock *dynloadBB = BasicBlock::Create(jl_LLVMContext, "dyn_sizeof", ctx->f); + BasicBlock *postBB = BasicBlock::Create(jl_LLVMContext, "post_sizeof", ctx->f); + Value *isboxed = builder.CreateICmpNE( + builder.CreateAnd(p.TIndex, ConstantInt::get(T_int8, 0x80)), + ConstantInt::get(T_int8, 0)); + builder.CreateCondBr(isboxed, dynloadBB, postBB); + builder.SetInsertPoint(dynloadBB); + Value *datatype = emit_typeof(p.V); + Value *dyn_size = emit_datatype_size(datatype); + builder.CreateBr(postBB); + builder.SetInsertPoint(postBB); + PHINode *sizeof_merge = builder.CreatePHI(T_int32, 2); + sizeof_merge->addIncoming(dyn_size, dynloadBB); + sizeof_merge->addIncoming(size, currBB); + size = sizeof_merge; + } +#ifndef NDEBUG + // try to catch codegen errors early, before it uses this to memcpy over the entire stack + CreateConditionalAbort(builder, builder.CreateICmpEQ(size, ConstantInt::get(T_int32, -1))); +#endif + return size; + } + else if (jl_is_leaf_type(p.typ)) { + return ConstantInt::get(T_int32, jl_datatype_size(p.typ)); + } + else { + Value *datatype = emit_typeof_boxed(p, ctx); + Value *dyn_size = emit_datatype_size(datatype); + return dyn_size; + } +} +*/ + static Value *emit_datatype_mutabl(Value *dt) { Value *mutabl = tbaa_decorate(tbaa_const, builder. @@ -798,18 +943,22 @@ static void emit_type_error(const jl_cgval_t &x, Value *type, const std::string static Value *emit_isa(const jl_cgval_t &x, jl_value_t *type, const std::string *msg, jl_codectx_t *ctx) { - Value *istype; - if (jl_type_intersection(x.typ, type) == (jl_value_t*)jl_bottom_type) { - if (msg) { - emit_type_error(x, literal_pointer_val(type), *msg, ctx); - builder.CreateUnreachable(); - BasicBlock *failBB = BasicBlock::Create(jl_LLVMContext, "fail", ctx->f); - builder.SetInsertPoint(failBB); - } - return ConstantInt::get(T_int1, 0); + bool maybe_isa = true; + if (x.constant) + maybe_isa = jl_isa(x.constant, type); + else if (jl_type_intersection(x.typ, type) == (jl_value_t*)jl_bottom_type) + maybe_isa = false; + if (!maybe_isa && msg) { + emit_type_error(x, literal_pointer_val(type), *msg, ctx); + builder.CreateUnreachable(); + BasicBlock *failBB = BasicBlock::Create(jl_LLVMContext, "fail", ctx->f); + builder.SetInsertPoint(failBB); } - else if (jl_has_intersect_type_not_kind(x.typ) || - jl_has_intersect_type_not_kind(type)) { + if (!maybe_isa || x.constant) + return ConstantInt::get(T_int1, maybe_isa); + + // intersection with Type needs to be handled specially + if (jl_has_intersect_type_not_kind(type)) { Value *vx = boxed(x, ctx); if (msg && *msg == "typeassert") { #if JL_LLVM_VERSION >= 30700 @@ -819,7 +968,7 @@ static Value *emit_isa(const jl_cgval_t &x, jl_value_t *type, const std::string #endif return ConstantInt::get(T_int1, 1); } - istype = builder.CreateICmpNE( + return builder.CreateICmpNE( #if JL_LLVM_VERSION >= 30700 builder.CreateCall(prepare_call(jlisa_func), { vx, literal_pointer_val(type) }), #else @@ -827,20 +976,43 @@ static Value *emit_isa(const jl_cgval_t &x, jl_value_t *type, const std::string #endif ConstantInt::get(T_int32, 0)); } - else if (jl_is_leaf_type(type)) { - istype = builder.CreateICmpEQ(emit_typeof_boxed(x, ctx), literal_pointer_val(type)); + // tests for isa leaftype can be handled with pointer comparisons + if (jl_is_leaf_type(type)) { + if (x.TIndex) { + unsigned tindex = get_box_tindex((jl_datatype_t*)type, x.typ); + if (tindex > 0) { + // optimize more when we know that this is a split union-type where tindex = 0 is invalid + Value *xtindex = builder.CreateAnd(x.TIndex, ConstantInt::get(T_int8, 0x7f)); + return builder.CreateICmpEQ(xtindex, ConstantInt::get(T_int8, tindex)); + } + else { + // test for (x.TIndex == 0x80 && typeof(x.V) == type) + Value *isboxed = builder.CreateICmpEQ(x.TIndex, ConstantInt::get(T_int8, 0x80)); + BasicBlock *currBB = builder.GetInsertBlock(); + BasicBlock *isaBB = BasicBlock::Create(jl_LLVMContext, "isa", ctx->f); + BasicBlock *postBB = BasicBlock::Create(jl_LLVMContext, "post_isa", ctx->f); + builder.CreateCondBr(isboxed, isaBB, postBB); + builder.SetInsertPoint(isaBB); + Value *istype_boxed = builder.CreateICmpEQ(emit_typeof(x.V), literal_pointer_val(type)); + builder.CreateBr(postBB); + builder.SetInsertPoint(postBB); + PHINode *istype = builder.CreatePHI(T_int1, 2); + istype->addIncoming(ConstantInt::get(T_int1, 0), currBB); + istype->addIncoming(istype_boxed, isaBB); + return istype; + } + } + return builder.CreateICmpEQ(emit_typeof_boxed(x, ctx), literal_pointer_val(type)); } - else { - Value *vxt = emit_typeof_boxed(x, ctx); - istype = builder.CreateICmpNE( + // everything else can be handled via subtype tests + Value *vxt = emit_typeof_boxed(x, ctx); + return builder.CreateICmpNE( #if JL_LLVM_VERSION >= 30700 - builder.CreateCall(prepare_call(jlsubtype_func), { vxt, literal_pointer_val(type) }), + builder.CreateCall(prepare_call(jlsubtype_func), { vxt, literal_pointer_val(type) }), #else - builder.CreateCall2(prepare_call(jlsubtype_func), vxt, literal_pointer_val(type)), + builder.CreateCall2(prepare_call(jlsubtype_func), vxt, literal_pointer_val(type)), #endif - ConstantInt::get(T_int32, 0)); - } - return istype; + ConstantInt::get(T_int32, 0)); } static void emit_typecheck(const jl_cgval_t &x, jl_value_t *type, const std::string &msg, @@ -1130,7 +1302,7 @@ static bool emit_getfield_unknownidx(jl_cgval_t *ret, const jl_cgval_t &strct, // just compute the pointer and let user load it when necessary Type *fty = julia_type_to_llvm(jt); Value *addr = builder.CreateGEP(builder.CreatePointerCast(ptr, PointerType::get(fty,0)), idx); - *ret = mark_julia_slot(addr, jt, strct.tbaa); + *ret = mark_julia_slot(addr, jt, NULL, strct.tbaa); ret->gcroot = strct.gcroot; ret->isimmutable = strct.isimmutable; return true; @@ -1198,7 +1370,7 @@ static jl_cgval_t emit_getfield_knownidx(const jl_cgval_t &strct, unsigned idx, } else if (!jt->mutabl) { // just compute the pointer and let user load it when necessary - jl_cgval_t fieldval = mark_julia_slot(addr, jfty, strct.tbaa); + jl_cgval_t fieldval = mark_julia_slot(addr, jfty, NULL, strct.tbaa); fieldval.isimmutable = strct.isimmutable; fieldval.gcroot = strct.gcroot; return fieldval; @@ -1244,6 +1416,29 @@ static bool arraytype_constshape(jl_value_t *ty) jl_is_long(jl_tparam1(ty)) && jl_unbox_long(jl_tparam1(ty)) != 1); } +static void maybe_alloc_arrayvar(int s, jl_codectx_t *ctx) +{ + jl_value_t *jt = ctx->slots[s].value.typ; + if (arraytype_constshape(jt)) { + // TODO: this optimization does not yet work with 1-d arrays, since the + // length and data pointer can change at any time via push! + // we could make it work by reloading the metadata when the array is + // passed to an external function (ideally only impure functions) + jl_arrayvar_t av; + int ndims = jl_unbox_long(jl_tparam1(jt)); + Type *elt = julia_type_to_llvm(jl_tparam0(jt)); + if (type_is_ghost(elt)) + return; + // CreateAlloca is OK here because maybe_alloc_arrayvar is only called in the prologue setup + av.dataptr = builder.CreateAlloca(PointerType::get(elt,0)); + av.len = builder.CreateAlloca(T_size); + for (int i = 0; i < ndims - 1; i++) + av.sizes.push_back(builder.CreateAlloca(T_size)); + av.ty = jt; + (*ctx->arrayvars)[s] = av; + } +} + static Value *emit_arraysize(const jl_cgval_t &tinfo, Value *dim, jl_codectx_t *ctx) { Value *t = boxed(tinfo, ctx); @@ -1484,25 +1679,22 @@ static Value *emit_array_nd_index(const jl_cgval_t &ainfo, jl_value_t *ex, ssize // --- boxing --- static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size, Value *jt); -static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size, const jl_cgval_t &v); -static Value *init_bits_value(Value *newv, Value *v, MDNode *tbaa, unsigned alignment = sizeof(void*)) // min alignment in julia's gc is pointer-aligned +static void init_bits_value(Value *newv, Value *v, MDNode *tbaa, unsigned alignment = sizeof(void*)) // min alignment in julia's gc is pointer-aligned { // newv should already be tagged tbaa_decorate(tbaa, builder.CreateAlignedStore(v, emit_bitcast(newv, PointerType::get(v->getType(), 0)), alignment)); - return newv; } -static Value *init_bits_cgval(Value *newv, const jl_cgval_t& v, MDNode *tbaa, jl_codectx_t *ctx) +static void init_bits_cgval(Value *newv, const jl_cgval_t& v, MDNode *tbaa, jl_codectx_t *ctx) { // newv should already be tagged if (v.ispointer()) { builder.CreateMemCpy(newv, data_pointer(v, ctx, T_pint8), jl_datatype_size(v.typ), sizeof(void*)); - return newv; } else { - return init_bits_value(newv, v.V, tbaa); + init_bits_value(newv, v.V, tbaa); } } @@ -1652,6 +1844,79 @@ static Value *_boxed_special(const jl_cgval_t &vinfo, Type *t, jl_codectx_t *ctx return box; } + + +static Value *box_union(const jl_cgval_t &vinfo, jl_codectx_t *ctx, const SmallBitVector &skip) +{ + // given vinfo::Union{T, S}, emit IR of the form: + // ... + // switch , label [ 1, label + // 2, label ] + // box_union_1: + // box1 = create_box(T) + // br post_box_union + // box_union_2: + // box2 = create_box(S) + // br post_box_union + // box_union_isboxed: + // br post_box_union + // post_box_union: + // box = phi [ box1, box_union_1 ], [ box2, box_union_2 ], [ vinfo, post_box_union ] + // ... + Value *tindex = vinfo.TIndex; + BasicBlock *defaultBB = BasicBlock::Create(jl_LLVMContext, "box_union_isboxed", ctx->f); + SwitchInst *switchInst = builder.CreateSwitch(tindex, defaultBB); + BasicBlock *postBB = BasicBlock::Create(jl_LLVMContext, "post_box_union", ctx->f); + builder.SetInsertPoint(postBB); + PHINode *box_merge = builder.CreatePHI(T_pjlvalue, 2); + unsigned counter = 0; + for_each_uniontype_small( + [&](unsigned idx, jl_datatype_t *jt) { + if (idx < skip.size() && skip[idx]) + return; + Type *t = julia_type_to_llvm((jl_value_t*)jt); + BasicBlock *tempBB = BasicBlock::Create(jl_LLVMContext, "box_union", ctx->f); + builder.SetInsertPoint(tempBB); + switchInst->addCase(ConstantInt::get(T_int8, idx), tempBB); + Value *box; + if (type_is_ghost(t)) { + box = literal_pointer_val(jt->instance); + } + else { + jl_cgval_t vinfo_r = jl_cgval_t(vinfo, (jl_value_t*)jt, NULL); + box = _boxed_special(vinfo_r, t, ctx); + if (!box) { + box = emit_allocobj(ctx, jl_datatype_size(jt), literal_pointer_val((jl_value_t*)jt)); + init_bits_cgval(box, vinfo_r, jl_is_mutable(jt) ? tbaa_mutab : tbaa_immut, ctx); + } + } + box_merge->addIncoming(box, tempBB); + builder.CreateBr(postBB); + }, + vinfo.typ, + counter); + builder.SetInsertPoint(defaultBB); + if (skip.size() > 0 && skip[0]) { + // skip[0] specifies where to return NULL or the original pointer + // if the value was not handled above + box_merge->addIncoming(V_null, defaultBB); + builder.CreateBr(postBB); + } + else if (vinfo.V == NULL || isa(vinfo.V)) { + Function *trap_func = Intrinsic::getDeclaration( + ctx->f->getParent(), + Intrinsic::trap); + builder.CreateCall(trap_func); + builder.CreateUnreachable(); + } + else { + box_merge->addIncoming(emit_bitcast(vinfo.V, T_pjlvalue), defaultBB); + builder.CreateBr(postBB); + } + builder.SetInsertPoint(postBB); + return box_merge; +} + // this is used to wrap values for generic contexts, where a // dynamically-typed value is required (e.g. argument to unknown function). // if it's already a pointer it's left alone. @@ -1663,17 +1928,26 @@ static Value *boxed(const jl_cgval_t &vinfo, jl_codectx_t *ctx, bool gcrooted) return UndefValue::get(T_pjlvalue); if (vinfo.constant) return literal_pointer_val(vinfo.constant); - assert(vinfo.V); - if (vinfo.isboxed) + if (vinfo.isboxed) { + assert(vinfo.V && "Missing value for box."); return vinfo.V; + } - assert(jl_isbits(jt) && jl_is_leaf_type(jt) && "This type shouldn't have been unboxed."); - Type *t = julia_type_to_llvm(jt); - assert(!type_is_ghost(t)); // ghost values should have been handled by vinfo.constant above! - Value *box = _boxed_special(vinfo, t, ctx); - if (!box) { - box = init_bits_cgval(emit_allocobj(ctx, jl_datatype_size(jt), vinfo), - vinfo, jl_is_mutable(jt) ? tbaa_mutab : tbaa_immut, ctx); + Value *box; + if (vinfo.TIndex) { + SmallBitVector skip_none; + box = box_union(vinfo, ctx, skip_none); + } + else { + assert(vinfo.V && "Missing data for unboxed value."); + assert(jl_isbits(jt) && jl_is_leaf_type(jt) && "This type shouldn't have been unboxed."); + Type *t = julia_type_to_llvm(jt); + assert(!type_is_ghost(t)); // ghost values should have been handled by vinfo.constant above! + box = _boxed_special(vinfo, t, ctx); + if (!box) { + box = emit_allocobj(ctx, jl_datatype_size(jt), literal_pointer_val((jl_value_t*)jt)); + init_bits_cgval(box, vinfo, jl_is_mutable(jt) ? tbaa_mutab : tbaa_immut, ctx); + } } if (gcrooted) { // make a gcroot for the new box @@ -1684,6 +1958,70 @@ static Value *boxed(const jl_cgval_t &vinfo, jl_codectx_t *ctx, bool gcrooted) return box; } +// copy src to dest, if src is isbits. if skip is true, the value of dest is undefined +static void emit_unionmove(Value *dest, const jl_cgval_t &src, Value *skip, bool isVolatile, MDNode *tbaa, jl_codectx_t *ctx) +{ + if (jl_is_leaf_type(src.typ) || src.constant) { + jl_value_t *typ = src.constant ? jl_typeof(src.constant) : src.typ; + Type *store_ty = julia_type_to_llvm(typ); + assert(skip || jl_isbits(typ)); + if (jl_isbits(typ)) { + if (!src.ispointer() || src.constant) { + emit_unbox(store_ty, src, typ, dest, isVolatile); + } + else { + Value *src_ptr = data_pointer(src, ctx, T_pint8); + if (dest->getType() != T_pint8) + dest = emit_bitcast(dest, T_pint8); + if (skip) // copy dest -> dest to simulate an undef value / conditional copy + src_ptr = builder.CreateSelect(skip, dest, src_ptr); + unsigned nb = jl_datatype_size(typ); + unsigned alignment = 0; + builder.CreateMemCpy(dest, src_ptr, nb, alignment, tbaa); + } + } + } + else if (src.TIndex) { + Value *tindex = builder.CreateAnd(src.TIndex, ConstantInt::get(T_int8, 0x7f)); + Value *copy_bytes = ConstantInt::get(T_int32, -1); + unsigned counter = 0; + bool allunboxed = for_each_uniontype_small( + [&](unsigned idx, jl_datatype_t *jt) { + Value *cmp = builder.CreateICmpEQ(tindex, ConstantInt::get(T_int8, idx)); + copy_bytes = builder.CreateSelect(cmp, ConstantInt::get(T_int32, jl_datatype_size(jt)), copy_bytes); + }, + src.typ, + counter); + Value *src_ptr = data_pointer(src, ctx, T_pint8); + if (dest->getType() != T_pint8) + dest = emit_bitcast(dest, T_pint8); + if (skip) { + if (allunboxed) // copy dest -> dest to simulate an undef value / conditional copy + src_ptr = builder.CreateSelect(skip, dest, src_ptr); + else + copy_bytes = builder.CreateSelect(skip, ConstantInt::get(copy_bytes->getType(), 0), copy_bytes); + } +#ifndef NDEBUG + // try to catch codegen errors early, before it uses this to memcpy over the entire stack + CreateConditionalAbort(builder, builder.CreateICmpEQ(copy_bytes, ConstantInt::get(T_int32, -1))); +#endif + builder.CreateMemCpy(dest, + src_ptr, + copy_bytes, + /*TODO: min-align*/1); + } + else { + Value *datatype = emit_typeof_boxed(src, ctx); + Value *copy_bytes = emit_datatype_size(datatype); + if (skip) + copy_bytes = builder.CreateSelect(skip, ConstantInt::get(copy_bytes->getType(), 0), copy_bytes); + builder.CreateMemCpy(dest, + data_pointer(src, ctx, T_pint8), + copy_bytes, + /*TODO: min-align*/1); + } +} + static void emit_cpointercheck(const jl_cgval_t &x, const std::string &msg, jl_codectx_t *ctx) { @@ -1730,11 +2068,6 @@ static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size, Value *jt) tbaa_decorate(tbaa_tag, builder.CreateStore(jt, emit_typeptr_addr(v))); return v; } -static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size, - const jl_cgval_t &v) -{ - return emit_allocobj(ctx, static_size, literal_pointer_val(v.typ)); -} // if ptr is NULL this emits a write barrier _back_ static void emit_write_barrier(jl_codectx_t *ctx, Value *parent, Value *ptr) @@ -1870,7 +2203,7 @@ static jl_cgval_t emit_new_struct(jl_value_t *ty, size_t nargs, jl_value_t **arg if (init_as_value) return mark_julia_type(strct, false, ty, ctx); else - return mark_julia_slot(strct, ty, tbaa_stack); + return mark_julia_slot(strct, ty, NULL, tbaa_stack); } Value *f1 = NULL; size_t j = 0; diff --git a/src/codegen.cpp b/src/codegen.cpp index 6961fb1a53abc..26c67da603b17 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -72,6 +72,7 @@ #endif // support +#include #include #include #include // for llvmcall @@ -311,7 +312,7 @@ int32_t jl_jlcall_api(const void *function) // constants -static Value *V_null; +static Constant *V_null; static Type *NoopType; static Value *literal_pointer_val(jl_value_t *p); extern "C" { @@ -431,22 +432,36 @@ extern "C" { int globalUnique = 0; } +static bool isbits_spec(jl_value_t *jt, bool allow_singleton = true) +{ + return jl_isbits(jt) && jl_is_leaf_type(jt) && + (allow_singleton || (jl_datatype_size(jt) > 0) || (jl_datatype_nfields(jt) > 0)); +} + // metadata tracking for a llvm Value* during codegen struct jl_cgval_t { Value *V; // may be of type T* or T, or set to NULL if ghost (or if the value has not been initialized yet, for a variable definition) + Value *TIndex; // if `V` is an unboxed (tagged) Union described by `typ`, this gives the DataType index (1-based, small int) as an i8 jl_value_t *constant; // constant value (rooted in linfo.def.roots) Value *gcroot; // the gcroot associated with V (if it has one) jl_value_t *typ; // the original type of V, never NULL bool isboxed; // whether this value is a jl_value_t* allocated on the heap with the right type tag bool isghost; // whether this value is "ghost" bool isimmutable; // V points to something that is definitely immutable (e.g. single-assignment, but including memory) - MDNode *tbaa; // The related tbaa node. Non-NULL iff this is not a pointer. + MDNode *tbaa; // The related tbaa node. Non-NULL iff this holds an address. bool ispointer() const { + // whether this value is compatible with `data_pointer` return tbaa != nullptr; } - jl_cgval_t(Value *V, Value *gcroot, bool isboxed, jl_value_t *typ) : // general constructor (with pointer type auto-detect) + //bool isvalue() const + //{ + // // whether this value is compatible with loading into registers (`emit_unbox` without an explicit type) + // return isbits_spec(typ) && (!ispointer() || constant); + //} + jl_cgval_t(Value *V, Value *gcroot, bool isboxed, jl_value_t *typ, Value *tindex) : // general constructor (with pointer type auto-detect) V(V), // V is allowed to be NULL in a jl_varinfo_t context, but not during codegen contexts + TIndex(tindex), constant(NULL), gcroot(gcroot), typ(typ), @@ -457,9 +472,12 @@ struct jl_cgval_t { (jl_is_mutable(typ) ? tbaa_mutab : tbaa_immut) : tbaa_value) : nullptr) { + assert(!(isboxed && TIndex != NULL)); + assert(TIndex == NULL || TIndex->getType() == T_int8); } jl_cgval_t(jl_value_t *typ) : // ghost value constructor V(NULL), + TIndex(NULL), constant(((jl_datatype_t*)typ)->instance), gcroot(NULL), typ(typ), @@ -471,8 +489,9 @@ struct jl_cgval_t { assert(jl_is_datatype(typ)); assert(constant); } - jl_cgval_t(const jl_cgval_t &v, jl_value_t *typ) : // copy constructor with new type + jl_cgval_t(const jl_cgval_t &v, jl_value_t *typ, Value *tindex) : // copy constructor with new type V(v.V), + TIndex(tindex), constant(v.constant), gcroot(v.gcroot), typ(typ), @@ -481,10 +500,18 @@ struct jl_cgval_t { isimmutable(v.isimmutable), tbaa(v.tbaa) { - assert(isboxed || v.typ == typ); // expect a badly or equivalently typed version + // this constructor expects we had a badly or equivalently typed version + // make sure we aren't discarding the actual type information + if (v.TIndex) { + assert((TIndex == NULL) == jl_is_leaf_type(typ)); + } + else { + assert(isboxed || v.typ == typ || tindex); + } } jl_cgval_t() : // undef / unreachable / default constructor V(UndefValue::get(T_void)), + TIndex(NULL), constant(NULL), gcroot(NULL), typ(jl_bottom_type), @@ -498,8 +525,9 @@ struct jl_cgval_t { // per-local-variable information struct jl_varinfo_t { - Value *boxroot; // an address, if the var is in a jl_value_t* gc stack slot or jl_box_t* Box object (marked tbaa_const, if appropriate) - jl_cgval_t value; // a value, if the var is unboxed or SSA (and thus boxroot == NULL) + Instruction *boxroot; // an address, if the var might be in a jl_value_t** stack slot (marked tbaa_const, if appropriate) + jl_cgval_t value; // a stack slot or constant value + Value *pTIndex; // i8* stack slot for the value.TIndex tag describing `value.V` #if JL_LLVM_VERSION >= 30700 DILocalVariable *dinfo; #else @@ -517,6 +545,7 @@ struct jl_varinfo_t { jl_varinfo_t() : boxroot(NULL), value(jl_cgval_t()), + pTIndex(NULL), #if JL_LLVM_VERSION >= 30700 dinfo(NULL), #else @@ -541,6 +570,22 @@ typedef struct { jl_value_t *ty; } jl_arrayvar_t; +struct jl_returninfo_t { + Function *decl; + enum CallingConv { + Boxed = 0, + Register, + SRet, + Union, + Ghosts + } cc; + size_t union_bytes; + size_t union_align; + size_t union_minalign; +}; + +static jl_returninfo_t get_specsig_function(Module *M, const std::string &name, jl_value_t *sig, jl_value_t *jlrettype); + // information about the context of a piece of code: its enclosing // function and module, and visible local variables and labels. class jl_codectx_t { @@ -566,7 +611,7 @@ class jl_codectx_t { std::string funcName; int vaSlot; // name of vararg argument bool vaStack; // varargs stack-allocated - bool sret; + bool has_sret; int nReqArgs; int nargs; @@ -598,7 +643,7 @@ static void allocate_gc_frame(BasicBlock *b0, jl_codectx_t *ctx); static GlobalVariable *prepare_global(GlobalVariable *G, Module *M = jl_builderModule); static Value *prepare_call(Value *Callee); static Value *prepare_call(IRBuilder<> &builder, Value *Callee); - +static void CreateTrap(IRBuilder<> &builder); template static void push_gc_use(T &&vec, const jl_cgval_t &v) { @@ -642,11 +687,11 @@ static inline jl_cgval_t ghostValue(jl_datatype_t *typ) return ghostValue((jl_value_t*)typ); } -static inline jl_cgval_t mark_julia_slot(Value *v, jl_value_t *typ, MDNode *tbaa) +static inline jl_cgval_t mark_julia_slot(Value *v, jl_value_t *typ, Value *tindex, MDNode *tbaa) { // this enables lazy-copying of immutable values and stack or argument slots assert(tbaa); - jl_cgval_t tagval(v, NULL, false, typ); + jl_cgval_t tagval(v, NULL, false, typ, tindex); tagval.tbaa = tbaa; tagval.isimmutable = true; return tagval; @@ -660,7 +705,7 @@ static inline jl_cgval_t mark_julia_type(Value *v, bool isboxed, jl_value_t *typ } if (jl_is_type_type(typ) && jl_is_leaf_type(jl_tparam0(typ))) { // replace T::Type{T} with T - jl_cgval_t constant(NULL, NULL, true, typ); + jl_cgval_t constant(NULL, NULL, true, typ, NULL); constant.constant = jl_tparam0(typ); return constant; } @@ -673,14 +718,14 @@ static inline jl_cgval_t mark_julia_type(Value *v, bool isboxed, jl_value_t *typ // llvm mem2reg pass will remove this if unneeded Value *loc = emit_static_alloca(T); builder.CreateStore(v, loc); - return mark_julia_slot(loc, typ, tbaa_stack); + return mark_julia_slot(loc, typ, NULL, tbaa_stack); } Value *froot = NULL; if (needsroot && isboxed) { froot = emit_local_root(ctx); builder.CreateStore(v, froot); } - return jl_cgval_t(v, froot, isboxed, typ); + return jl_cgval_t(v, froot, isboxed, typ, NULL); } static inline jl_cgval_t mark_julia_type(Value *v, bool isboxed, jl_datatype_t *typ, jl_codectx_t *ctx, bool needsroot = true) @@ -688,14 +733,38 @@ static inline jl_cgval_t mark_julia_type(Value *v, bool isboxed, jl_datatype_t * return mark_julia_type(v, isboxed, (jl_value_t*)typ, ctx, needsroot); } -static inline jl_cgval_t remark_julia_type(const jl_cgval_t &v, jl_value_t *typ, jl_codectx_t *ctx) +// see if it might be profitable (and cheap) to change the type of v to typ +static inline jl_cgval_t update_julia_type(const jl_cgval_t &v, jl_value_t *typ, jl_codectx_t *ctx) { - if (v.typ == typ) + if (v.typ == typ || v.typ == jl_bottom_type || jl_egal(v.typ, typ) || typ == (jl_value_t*)jl_any_type) return v; // fast-path + if (jl_is_leaf_type(v.typ) && !jl_is_kind(v.typ)) { + if (jl_is_leaf_type(typ) && !jl_is_kind(typ) && !((jl_datatype_t*)typ)->abstract && !((jl_datatype_t*)v.typ)->abstract) { + // type mismatch: changing from one leaftype to another + CreateTrap(builder); + return jl_cgval_t(); + } + return v; // doesn't improve type info + } + if (v.TIndex) { + if (!jl_is_leaf_type(typ)) + return v; // not worth trying to improve type info + if (!isbits_spec(typ)) { + // discovered that this union-split type must actually be isboxed + if (v.V) { + return jl_cgval_t(v.V, v.gcroot, true, typ, NULL); + } + else { + // type mismatch (there wasn't any boxed values in the union) + CreateTrap(builder); + return jl_cgval_t(); + } + } + } Type *T = julia_type_to_llvm(typ); if (type_is_ghost(T)) return ghostValue(typ); - return jl_cgval_t(v, typ); + return jl_cgval_t(v, typ, NULL); } static inline jl_cgval_t mark_julia_const(jl_value_t *jv) @@ -708,35 +777,37 @@ static inline jl_cgval_t mark_julia_const(jl_value_t *jv) if (type_is_ghost(julia_type_to_llvm(typ))) { return ghostValue(typ); } - jl_cgval_t constant(NULL, NULL, true, typ); + jl_cgval_t constant(NULL, NULL, true, typ, NULL); constant.constant = jv; return constant; } -// --- utilities --- +// --- allocating local variables --- -static void emit_write_barrier(jl_codectx_t*, Value*, Value*); +static jl_sym_t *slot_symbol(int s, jl_codectx_t *ctx) +{ + return (jl_sym_t*)jl_array_ptr_ref(ctx->source->slotnames, s); +} -#include "cgutils.cpp" +static void store_def_flag(const jl_varinfo_t &vi, bool val) +{ + assert((!vi.boxroot || vi.pTIndex) && "undef check is null pointer for boxed things"); + assert(vi.usedUndef && vi.defFlag && "undef flag codegen corrupted"); + builder.CreateStore(ConstantInt::get(T_int1, val), vi.defFlag, vi.isVolatile); +} -static void jl_rethrow_with_add(const char *fmt, ...) +static void alloc_def_flag(jl_varinfo_t& vi, jl_codectx_t* ctx) { - jl_ptls_t ptls = jl_get_ptls_states(); - if (jl_typeis(ptls->exception_in_transit, jl_errorexception_type)) { - char *str = jl_string_data(jl_fieldref(ptls->exception_in_transit,0)); - char buf[1024]; - va_list args; - va_start(args, fmt); - int nc = vsnprintf(buf, sizeof(buf), fmt, args); - va_end(args); - nc += snprintf(buf+nc, sizeof(buf)-nc, ": %s", str); - jl_value_t *msg = jl_pchar_to_string(buf, nc); - JL_GC_PUSH1(&msg); - jl_throw(jl_new_struct(jl_errorexception_type, msg)); + assert((!vi.boxroot || vi.pTIndex) && "undef check is null pointer for boxed things"); + if (vi.usedUndef) { + vi.defFlag = emit_static_alloca(T_int1, ctx); + store_def_flag(vi, false); } - jl_rethrow(); } + +// --- utilities --- + static void CreateTrap(IRBuilder<> &builder) { Function *f = builder.GetInsertBlock()->getParent(); @@ -749,56 +820,259 @@ static void CreateTrap(IRBuilder<> &builder) builder.SetInsertPoint(newBB); } -// --- allocating local variables --- - -static bool isbits_spec(jl_value_t *jt, bool allow_singleton = true) +#ifndef NDEBUG +static void CreateConditionalAbort(IRBuilder<> &builder, Value *test) { - return jl_isbits(jt) && jl_is_leaf_type(jt) && - (allow_singleton || (jl_datatype_size(jt) > 0) || (jl_datatype_nfields(jt) > 0)); + Function *f = builder.GetInsertBlock()->getParent(); + BasicBlock *abortBB = BasicBlock::Create(jl_LLVMContext, "debug_abort", f); + BasicBlock *postBB = BasicBlock::Create(jl_LLVMContext, "post_abort", f); + builder.CreateCondBr(test, abortBB, postBB); + builder.SetInsertPoint(abortBB); + Function *trap_func = Intrinsic::getDeclaration( + f->getParent(), + Intrinsic::trap); + builder.CreateCall(trap_func); + builder.CreateUnreachable(); + builder.SetInsertPoint(postBB); } +#endif -static jl_sym_t *slot_symbol(int s, jl_codectx_t *ctx) -{ - return (jl_sym_t*)jl_array_ptr_ref(ctx->source->slotnames, s); -} +static void emit_write_barrier(jl_codectx_t*, Value*, Value*); -static void store_def_flag(const jl_varinfo_t &vi, bool val) -{ - assert(!vi.boxroot && "undef check is null pointer for boxed things"); - assert(vi.usedUndef && vi.defFlag && "undef flag codegen corrupted"); - builder.CreateStore(ConstantInt::get(T_int1, val), vi.defFlag, vi.isVolatile); -} +#include "cgutils.cpp" -static void alloc_def_flag(jl_varinfo_t& vi, jl_codectx_t* ctx) +static void jl_rethrow_with_add(const char *fmt, ...) { - assert(!vi.boxroot && "undef check is null pointer for boxed things"); - if (vi.usedUndef) { - vi.defFlag = emit_static_alloca(T_int1, ctx); - store_def_flag(vi, false); + jl_ptls_t ptls = jl_get_ptls_states(); + if (jl_typeis(ptls->exception_in_transit, jl_errorexception_type)) { + char *str = jl_string_data(jl_fieldref(ptls->exception_in_transit,0)); + char buf[1024]; + va_list args; + va_start(args, fmt); + int nc = vsnprintf(buf, sizeof(buf), fmt, args); + va_end(args); + nc += snprintf(buf+nc, sizeof(buf)-nc, ": %s", str); + jl_value_t *msg = jl_pchar_to_string(buf, nc); + JL_GC_PUSH1(&msg); + jl_throw(jl_new_struct(jl_errorexception_type, msg)); } + jl_rethrow(); } -static void maybe_alloc_arrayvar(int s, jl_codectx_t *ctx) +// given a value marked with type `v.typ`, compute the mapping and/or boxing to return a value of type `typ` +static jl_cgval_t convert_julia_type(const jl_cgval_t &v, jl_value_t *typ, jl_codectx_t *ctx, bool needsroot = true) { - jl_value_t *jt = ctx->slots[s].value.typ; - if (arraytype_constshape(jt)) { - // TODO: this optimization does not yet work with 1-d arrays, since the - // length and data pointer can change at any time via push! - // we could make it work by reloading the metadata when the array is - // passed to an external function (ideally only impure functions) - jl_arrayvar_t av; - int ndims = jl_unbox_long(jl_tparam1(jt)); - Type *elt = julia_type_to_llvm(jl_tparam0(jt)); - if (type_is_ghost(elt)) - return; - // CreateAlloca is OK here because maybe_alloc_arrayvar is only called in the prologue setup - av.dataptr = builder.CreateAlloca(PointerType::get(elt,0)); - av.len = builder.CreateAlloca(T_size); - for (int i = 0; i < ndims - 1; i++) - av.sizes.push_back(builder.CreateAlloca(T_size)); - av.ty = jt; - (*ctx->arrayvars)[s] = av; + if (v.typ == typ || v.typ == jl_bottom_type || jl_egal(v.typ, typ)) + return v; // fast-path + Type *T = julia_type_to_llvm(typ); + if (type_is_ghost(T)) + return ghostValue(typ); + Value *new_tindex = NULL; + if (jl_is_leaf_type(typ)) { + if (v.TIndex && !isbits_spec(typ)) { + // discovered that this union-split type must actually be isboxed + if (v.V) { + return jl_cgval_t(v.V, v.gcroot, true, typ, NULL); + } + else { + // type mismatch: there wasn't any boxed values in the union + CreateTrap(builder); + return jl_cgval_t(); + } + } + if (jl_is_leaf_type(v.typ) && !jl_is_kind(v.typ) && !((jl_datatype_t*)v.typ)->abstract) { + if (jl_is_leaf_type(typ) && !jl_is_kind(typ) && !((jl_datatype_t*)typ)->abstract) { + // type mismatch: changing from one leaftype to another + CreateTrap(builder); + return jl_cgval_t(); + } + } } + else { + bool makeboxed = false; + if (v.TIndex) { + // previous value was a split union, compute new index, or box + new_tindex = ConstantInt::get(T_int8, 0x80); + SmallBitVector skip_box(1, true); + Value *tindex = builder.CreateAnd(v.TIndex, ConstantInt::get(T_int8, 0x7f)); + if (jl_is_uniontype(typ)) { + // compute the TIndex mapping from v.typ -> typ + unsigned counter = 0; + for_each_uniontype_small( + // for each old union-split value + [&](unsigned idx, jl_datatype_t *jt) { + unsigned new_idx = get_box_tindex(jt, typ); + bool t; + if (new_idx) { + // found a matching element, + // match it against either the unboxed index + Value *cmp = builder.CreateICmpEQ(tindex, ConstantInt::get(T_int8, idx)); + new_tindex = builder.CreateSelect(cmp, ConstantInt::get(T_int8, new_idx), new_tindex); + t = true; + } + else if (!jl_subtype((jl_value_t*)jt, typ)) { + // new value doesn't need to be boxed + // since it isn't part of the new union + t = true; + } + else { + // will actually need to box this element + // since it appeared as a leaftype in the original type + // but not in the remark type + t = false; + } + skip_box.resize(idx + 1, t); + }, + v.typ, + counter); + } + + // some of the values are still unboxed + if (!isa(new_tindex)) { + Value *wasboxed = NULL; + // check if some of the old values might have been boxed + // and copy that information over into the new tindex + if (v.ispointer() && v.V && !isa(v.V)) { + wasboxed = builder.CreateAnd(v.TIndex, ConstantInt::get(T_int8, 0x80)); + new_tindex = builder.CreateOr(wasboxed, new_tindex); + wasboxed = builder.CreateICmpNE(wasboxed, ConstantInt::get(T_int8, 0)); + + // may need to handle compute_box_tindex for some of the values + BasicBlock *currBB = builder.GetInsertBlock(); + Value *union_box_dt = NULL; + Value *union_box_tindex = ConstantInt::get(T_int8, 0x80); + unsigned counter = 0; + for_each_uniontype_small( + // for each new union-split value + [&](unsigned idx, jl_datatype_t *jt) { + unsigned old_idx = get_box_tindex(jt, v.typ); + if (old_idx == 0) { + if (!union_box_dt) { + BasicBlock *isaBB = BasicBlock::Create(jl_LLVMContext, "union_isa", ctx->f); + builder.SetInsertPoint(isaBB); + union_box_dt = emit_typeof(v.V); + } + // didn't handle this item before, select its new union index + Value *cmp = builder.CreateICmpEQ(literal_pointer_val((jl_value_t*)jt), union_box_dt); + union_box_tindex = builder.CreateSelect(cmp, ConstantInt::get(T_int8, 0x80 | idx), union_box_tindex); + } + }, + typ, + counter); + if (union_box_dt) { + BasicBlock *isaBB = builder.GetInsertBlock(); + BasicBlock *postBB = BasicBlock::Create(jl_LLVMContext, "post_union_isa", ctx->f); + builder.CreateBr(postBB); + builder.SetInsertPoint(currBB); + Value *wasunknown = builder.CreateICmpEQ(v.TIndex, ConstantInt::get(T_int8, 0x80)); + builder.CreateCondBr(wasunknown, isaBB, postBB); + builder.SetInsertPoint(postBB); + PHINode *tindex_phi = builder.CreatePHI(T_int8, 2); + tindex_phi->addIncoming(new_tindex, currBB); + tindex_phi->addIncoming(union_box_tindex, isaBB); + new_tindex = tindex_phi; + } + + } + + if (!skip_box.all()) { + // some values weren't unboxed in the new union + // box them now (tindex above already selected 0x80 = box for them) + // root the result, and return a new mark_julia_slot over the result + Value *boxv = box_union(v, ctx, skip_box); + Value *froot = NULL; + if (needsroot) { + // build a new gc-root, as needed + froot = emit_local_root(ctx); + Value *newroot = boxv; + if (wasboxed || v.gcroot) { // oldbox might be all ghost values (which don't need roots) + // store either the old box or the new box into the gc-root (skip_box ensures these are mutually-exclusive) + // need to clone the value from `v.gcroot` if this isn't a new box + Value *oldroot; + if (v.gcroot) + oldroot = builder.CreateLoad(v.gcroot); + else + oldroot = v.V; + newroot = builder.CreateSelect(wasboxed, emit_bitcast(oldroot, boxv->getType()), newroot); + } + builder.CreateStore(newroot, froot); + } + else { + mark_gc_use(v); + } + if (v.V == NULL) { + // v.V might be NULL if it was all ghost objects before + return jl_cgval_t(boxv, froot, false, typ, new_tindex); + } + else { + Value *isboxv = builder.CreateIsNotNull(boxv); + Value *slotv; + MDNode *tbaa; + bool isimmutable; + if (v.ispointer()) { + slotv = v.V; + tbaa = v.tbaa; + isimmutable = v.isimmutable; + } + else { + slotv = emit_static_alloca(v.V->getType()); + builder.CreateStore(v.V, slotv); + tbaa = tbaa_stack; + isimmutable = true; + } + slotv = builder.CreateSelect(isboxv, boxv, emit_bitcast(slotv, boxv->getType())); + jl_cgval_t newv = jl_cgval_t(slotv, froot, false, typ, new_tindex); + newv.tbaa = tbaa; + newv.isimmutable = isimmutable; + return newv; + } + } + } + else { + new_tindex = NULL; + makeboxed = true; + } + } + else if (!v.isboxed && jl_is_uniontype(typ)) { + // previous value was unboxed (leaftype), statically compute union tindex + assert(jl_is_leaf_type(v.typ)); + unsigned new_idx = get_box_tindex((jl_datatype_t*)v.typ, typ); + if (new_idx) { + new_tindex = ConstantInt::get(T_int8, new_idx); + if (v.V && !v.ispointer()) { + // TODO: remove this branch once all consumers of v.TIndex understand how to handle a non-ispointer value + Value *slotv = emit_static_alloca(v.V->getType()); + builder.CreateStore(v.V, slotv); + jl_cgval_t newv = jl_cgval_t(slotv, NULL, false, typ, new_tindex); + newv.tbaa = tbaa_stack; + newv.isimmutable = true; + return newv; + } + } + else if (jl_subtype(v.typ, typ)) { + makeboxed = true; + } + else { + // unreachable + CreateTrap(builder); + return jl_cgval_t(); + } + } + else if (!v.isboxed) { + makeboxed = true; + } + if (makeboxed) { + // convert to a simple isboxed value + Value *boxv = boxed(v, ctx); + Value *froot = NULL; + if (needsroot) { + froot = emit_local_root(ctx); + builder.CreateStore(boxv, froot); + } + return jl_cgval_t(boxv, froot, true, typ, NULL); + } + } + return jl_cgval_t(v, typ, new_tindex); } // Snooping on which functions are being compiled, and how long it takes @@ -933,7 +1207,6 @@ jl_llvm_functions_t jl_compile_linfo(jl_method_instance_t **pli, jl_code_info_t Function *f = (Function*)decls.functionObject; Function *specf = (Function*)decls.specFunctionObject; - if (JL_HOOK_TEST(params, module_activation)) { JL_HOOK_CALL(params, module_activation, 1, jl_box_voidpointer(wrap(m.release()))); } else { @@ -2013,6 +2286,7 @@ static Value *emit_local_root(jl_codectx_t *ctx, jl_varinfo_t *vi) if (vi) { vi->boxroot->replaceAllUsesWith(newroot); newroot->takeName(vi->boxroot); + vi->boxroot->eraseFromParent(); vi->boxroot = newroot; } return newroot; @@ -2195,8 +2469,8 @@ static Value *emit_bits_compare(const jl_cgval_t &arg1, const jl_cgval_t &arg2, if (type_is_ghost(fld1->getType()->getPointerElementType())) continue; subAns = emit_bits_compare( - mark_julia_slot(fld1, fldty, arg1.tbaa), - mark_julia_slot(fld2, fldty, arg2.tbaa), + mark_julia_slot(fld1, fldty, NULL, arg1.tbaa), + mark_julia_slot(fld2, fldty, NULL, arg2.tbaa), ctx); answer = builder.CreateAnd(answer, subAns); } @@ -2211,17 +2485,17 @@ static Value *emit_bits_compare(const jl_cgval_t &arg1, const jl_cgval_t &arg2, static Value *emit_f_is(const jl_cgval_t &arg1, const jl_cgval_t &arg2, jl_codectx_t *ctx) { jl_value_t *rt1 = arg1.typ, *rt2 = arg2.typ; - bool isleaf = jl_is_leaf_type(rt1) && jl_is_leaf_type(rt2); - if (isleaf && rt1 != rt2 && !jl_is_type_type(rt1) && !jl_is_type_type(rt2)) - // disjoint leaf types are never equal (quick test) + if (jl_is_leaf_type(rt1) && jl_is_leaf_type(rt2) && rt1 != rt2 + && !jl_is_type_type(rt1) && !jl_is_type_type(rt2)) + // disjoint concrete leaf types are never equal (quick test) return ConstantInt::get(T_int1, 0); - bool ghost1 = arg1.isghost || (isleaf && jl_is_datatype_singleton((jl_datatype_t*)rt1)); - bool ghost2 = arg2.isghost || (isleaf && jl_is_datatype_singleton((jl_datatype_t*)rt2)); - if (ghost1 || ghost2) { - // comparing singleton objects - if (ghost1 && ghost2) { - return ConstantInt::get(T_int1, rt1 == rt2); - } + + if (arg1.isghost || arg2.isghost) { + // comparing to a singleton object + if (arg1.TIndex) + return emit_isa(arg1, rt2, NULL, ctx); // rt2 is a singleton type + if (arg2.TIndex) + return emit_isa(arg2, rt1, NULL, ctx); // rt1 is a singleton type // mark_gc_use isn't needed since we won't load this pointer // and we know at least one of them is a unique Singleton // which is already enough to ensure pointer uniqueness for this test @@ -2232,24 +2506,46 @@ static Value *emit_f_is(const jl_cgval_t &arg1, const jl_cgval_t &arg2, jl_codec if (jl_type_intersection(rt1, rt2) == (jl_value_t*)jl_bottom_type) // types are disjoint (exhaustive test) return ConstantInt::get(T_int1, 0); - bool isbits = isleaf && jl_isbits(rt1) && jl_types_equal(rt1, rt2); + bool isbits = jl_isbits(rt1) || jl_isbits(rt2); if (isbits) { // whether this type is unique'd by value - return emit_bits_compare(arg1, arg2, ctx); + jl_value_t *typ = jl_isbits(rt1) ? rt1 : rt2; + if (rt1 == rt2) + return emit_bits_compare(arg1, arg2, ctx); + Value *same_type = (typ == rt2) ? emit_isa(arg1, typ, NULL, ctx) : emit_isa(arg2, typ, NULL, ctx); + BasicBlock *currBB = builder.GetInsertBlock(); + BasicBlock *isaBB = BasicBlock::Create(jl_LLVMContext, "is", ctx->f); + BasicBlock *postBB = BasicBlock::Create(jl_LLVMContext, "post_is", ctx->f); + builder.CreateCondBr(same_type, isaBB, postBB); + builder.SetInsertPoint(isaBB); + Value *bitcmp = emit_bits_compare( + jl_cgval_t(arg1, typ, NULL), + jl_cgval_t(arg2, typ, NULL), + ctx); + builder.CreateBr(postBB); + builder.SetInsertPoint(postBB); + PHINode *cmp = builder.CreatePHI(T_int1, 2); + cmp->addIncoming(ConstantInt::get(T_int1, 0), currBB); + cmp->addIncoming(bitcmp, isaBB); + return cmp; } int ptr_comparable = 0; // whether this type is unique'd by pointer - if (rt1==(jl_value_t*)jl_sym_type || rt2==(jl_value_t*)jl_sym_type || - jl_is_mutable_datatype(rt1) || jl_is_mutable_datatype(rt2)) // excludes abstract types + if (rt1 == (jl_value_t*)jl_sym_type || rt2 == (jl_value_t*)jl_sym_type) + ptr_comparable = 1; + if (jl_is_mutable_datatype(rt1) || jl_is_mutable_datatype(rt2)) // excludes abstract types ptr_comparable = 1; if (jl_subtype(rt1, (jl_value_t*)jl_type_type) || - jl_subtype(rt2, (jl_value_t*)jl_type_type)) // use typeseq for datatypes + jl_subtype(rt2, (jl_value_t*)jl_type_type)) // need to use typeseq for datatypes ptr_comparable = 0; if ((jl_is_type_type(rt1) && jl_is_leaf_type(jl_tparam0(rt1))) || (jl_is_type_type(rt2) && jl_is_leaf_type(jl_tparam0(rt2)))) // can compare leaf types by pointer ptr_comparable = 1; if (ptr_comparable) { - assert(arg1.isboxed && arg2.isboxed); // only boxed types are valid for pointer comparison - return builder.CreateICmpEQ(boxed(arg1, ctx), boxed(arg2, ctx)); + Value *varg1 = arg1.constant ? literal_pointer_val(arg1.constant) : arg1.V; + Value *varg2 = arg2.constant ? literal_pointer_val(arg2.constant) : arg2.V; + assert(varg1 && varg2 && (arg1.isboxed || arg1.TIndex) && (arg2.isboxed || arg2.TIndex) && + "Only boxed types are valid for pointer comparison."); + return builder.CreateICmpEQ(varg1, varg2); } JL_FEAT_REQUIRE(ctx, runtime); @@ -2283,9 +2579,9 @@ static bool emit_builtin_call(jl_cgval_t *ret, jl_value_t *f, jl_value_t **args, } // FIXME: v.typ is roughly equiv. to expr_type, but with typeof(T) == Type{T} instead of DataType in a few cases if (v1.typ == (jl_value_t*)jl_datatype_type) - v1 = remark_julia_type(v1, expr_type(args[1], ctx), ctx); // patch up typ if necessary + v1 = update_julia_type(v1, expr_type(args[1], ctx), ctx); // patch up typ if necessary if (v2.typ == (jl_value_t*)jl_datatype_type) - v2 = remark_julia_type(v2, expr_type(args[2], ctx), ctx); // patch up typ if necessary + v2 = update_julia_type(v2, expr_type(args[2], ctx), ctx); // patch up typ if necessary // emit comparison test Value *ans = emit_f_is(v1, v2, ctx); mark_gc_use(v1); @@ -2309,13 +2605,10 @@ static bool emit_builtin_call(jl_cgval_t *ret, jl_value_t *f, jl_value_t **args, jl_value_t *tp0 = jl_tparam0(ty); *ret = emit_expr(args[1], ctx); emit_expr(args[2], ctx); - if (jl_subtype(arg, tp0)) { - JL_GC_POP(); - return true; - } - emit_typecheck(*ret, tp0, "typeassert", ctx); - if (ret->isboxed) - *ret = remark_julia_type(*ret, expr_type(expr, ctx), ctx); + if (!jl_subtype(arg, tp0)) + emit_typecheck(*ret, tp0, "typeassert", ctx); + ty = expr_type(expr, ctx); rt2 = ty; + *ret = update_julia_type(*ret, ty, ctx); JL_GC_POP(); return true; } @@ -2349,11 +2642,11 @@ static bool emit_builtin_call(jl_cgval_t *ret, jl_value_t *f, jl_value_t **args, emit_expr(args[2], ctx); if (jl_subtype(arg, tp0)) { *ret = mark_julia_type(ConstantInt::get(T_int8, 1), false, jl_bool_type, ctx); - JL_GC_POP(); - return true; } - Value *isa = emit_isa(rt_arg, tp0, NULL, ctx); - *ret = mark_julia_type(builder.CreateZExt(isa, T_int8), false, jl_bool_type, ctx); + else { + Value *isa = emit_isa(rt_arg, tp0, NULL, ctx); + *ret = mark_julia_type(builder.CreateZExt(isa, T_int8), false, jl_bool_type, ctx); + } JL_GC_POP(); return true; } @@ -2597,7 +2890,7 @@ static bool emit_builtin_call(jl_cgval_t *ret, jl_value_t *f, jl_value_t **args, *ret = emit_getfield(args[1], (jl_sym_t*)jl_fieldref(args[2],0), ctx); if (ret->typ == (jl_value_t*)jl_any_type) // improve the type, if known from the expr - *ret = remark_julia_type(*ret, expr_type(expr, ctx), ctx); + *ret = update_julia_type(*ret, expr_type(expr, ctx), ctx); JL_GC_POP(); return true; } @@ -2609,7 +2902,7 @@ static bool emit_builtin_call(jl_cgval_t *ret, jl_value_t *f, jl_value_t **args, Value *valen = emit_n_varargs(ctx); Value *idx = emit_unbox(T_size, emit_expr(args[2], ctx), fldt); idx = emit_bounds_check( - jl_cgval_t(builder.CreateGEP(ctx->argArray, ConstantInt::get(T_size, ctx->nReqArgs)), NULL, false, NULL), + jl_cgval_t(builder.CreateGEP(ctx->argArray, ConstantInt::get(T_size, ctx->nReqArgs)), NULL, false, NULL, NULL), NULL, idx, valen, ctx); idx = builder.CreateAdd(idx, ConstantInt::get(T_size, ctx->nReqArgs)); *ret = mark_julia_type( @@ -2854,24 +3147,45 @@ static jl_cgval_t emit_call_function_object(jl_method_instance_t *li, const jl_c if (decls.specFunctionObject != NULL) { // emit specialized call site jl_value_t *jlretty = li->rettype; - bool retboxed; - (void)julia_type_to_llvm(jlretty, &retboxed); - Function *cf = cast(prepare_call((Function*)decls.specFunctionObject)); - FunctionType *cft = cf->getFunctionType(); + Function *proto = (Function*)decls.specFunctionObject; + jl_returninfo_t returninfo = get_specsig_function(jl_Module, proto->getName(), li->specTypes, jlretty); + FunctionType *cft = returninfo.decl->getFunctionType(); + assert(proto->getFunctionType() == cft); + + proto = cast(prepare_call(proto)); + if (proto != returninfo.decl) { + assert(proto->getFunctionType() == cft); + returninfo.decl->replaceAllUsesWith(proto); + returninfo.decl->eraseFromParent(); + returninfo.decl = proto; + } + size_t nfargs = cft->getNumParams(); - Value **argvals = (Value**) alloca(nfargs*sizeof(Value*)); - bool sret = cf->hasStructRetAttr(); + Value **argvals = (Value**)alloca(nfargs * sizeof(Value*)); unsigned idx = 0; - Value *result; - if (sret) { - assert(!retboxed); + AllocaInst *result; + switch (returninfo.cc) { + case jl_returninfo_t::Boxed: + case jl_returninfo_t::Register: + case jl_returninfo_t::Ghosts: + break; + case jl_returninfo_t::SRet: result = emit_static_alloca(cft->getParamType(0)->getContainedType(0), ctx); argvals[idx] = result; idx++; + break; + case jl_returninfo_t::Union: + result = emit_static_alloca(ArrayType::get(T_int8, returninfo.union_bytes), ctx); + if (returninfo.union_align > 1) + result->setAlignment(returninfo.union_align); + argvals[idx] = result; + idx++; + break; } + SmallVector gc_uses; for (size_t i = 0; i < nargs + 1; i++) { - jl_value_t *jt = jl_nth_slot_type(li->specTypes,i); + jl_value_t *jt = jl_nth_slot_type(li->specTypes, i); bool isboxed; Type *et = julia_type_to_llvm(jt, &isboxed); if (type_is_ghost(et)) { @@ -2906,16 +3220,45 @@ static jl_cgval_t emit_call_function_object(jl_method_instance_t *li, const jl_c } assert(idx == nfargs); mark_gc_uses(gc_uses); - CallInst *call = builder.CreateCall(prepare_call(cf), ArrayRef(&argvals[0], nfargs)); - call->setAttributes(cf->getAttributes()); + CallInst *call = builder.CreateCall(returninfo.decl, ArrayRef(&argvals[0], nfargs)); + call->setAttributes(returninfo.decl->getAttributes()); mark_gc_uses(gc_uses); - if (sret) - return mark_julia_slot(result, jlretty, tbaa_stack); - // see if codegen has a better type for the call than inference had at the time - if (!retboxed && jlretty != inferred_retty) { - inferred_retty = jlretty; + + jl_cgval_t retval; + switch (returninfo.cc) { + case jl_returninfo_t::Boxed: + retval = mark_julia_type(call, true, inferred_retty, ctx); + break; + case jl_returninfo_t::Register: + retval = mark_julia_type(call, false, jlretty, ctx); + break; + case jl_returninfo_t::SRet: + retval = mark_julia_slot(result, jlretty, NULL, tbaa_stack); + break; + case jl_returninfo_t::Union: + retval = mark_julia_slot(builder.CreateExtractValue(call, 0), + jlretty, + builder.CreateExtractValue(call, 1), + tbaa_stack); + // root this, if the return value was a box (tindex & 0x80) != 0 + retval.gcroot = emit_local_root(ctx); + builder.CreateStore( + builder.CreateSelect( + builder.CreateICmpEQ( + builder.CreateAnd(retval.TIndex, ConstantInt::get(T_int8, 0x80)), + ConstantInt::get(T_int8, 0)), + V_null, + retval.V), + retval.gcroot); + break; + case jl_returninfo_t::Ghosts: + retval = mark_julia_slot(NULL, jlretty, call, tbaa_stack); + break; } - return mark_julia_type(call, retboxed, inferred_retty, ctx); + // see if inference has a different / better type for the call than the lambda + if (inferred_retty != retval.typ) + retval = update_julia_type(retval, inferred_retty, ctx); + return retval; } Value *ret = emit_jlcall(theFptr, boxed(theF, ctx), &args[1], nargs, ctx); return mark_julia_type(ret, true, inferred_retty, ctx); @@ -2974,7 +3317,7 @@ static jl_cgval_t emit_call(jl_expr_t *ex, jl_codectx_t *ctx) if (jl_typeis(f, jl_intrinsic_type)) { result = emit_intrinsic((intrinsic)*(uint32_t*)jl_data_ptr(f), args, nargs, ctx); if (result.typ == (jl_value_t*)jl_any_type) // the select_value intrinsic may be missing type information - result = remark_julia_type(result, expr_type(expr, ctx), ctx); + result = update_julia_type(result, expr_type(expr, ctx), ctx); JL_GC_POP(); return result; } @@ -3063,7 +3406,7 @@ static Value *global_binding_pointer(jl_module_t *m, jl_sym_t *s, JL_FEAT_REQUIRE(ctx, runtime); std::stringstream name; name << "delayedvar" << globalUnique++; - Constant *initnul = ConstantPointerNull::get((PointerType*)T_pjlvalue); + Constant *initnul = V_null; GlobalVariable *bindinggv = new GlobalVariable(*ctx->f->getParent(), T_pjlvalue, false, GlobalVariable::InternalLinkage, initnul, name.str()); @@ -3158,9 +3501,11 @@ static jl_cgval_t emit_local(jl_value_t *slotload, jl_codectx_t *ctx) jl_cgval_t v; Value *isnull = NULL; - if (vi.boxroot == NULL) { - if (!vi.isVolatile || vi.value.constant) { + if (vi.boxroot == NULL || vi.pTIndex != NULL) { + if (!vi.isVolatile || vi.value.constant || !vi.value.V) { v = vi.value; + if (vi.pTIndex) + v.TIndex = builder.CreateLoad(vi.pTIndex); } else { // copy value to a non-volatile location @@ -3171,19 +3516,43 @@ static jl_cgval_t emit_local(jl_value_t *slotload, jl_codectx_t *ctx) // TODO: emit memcpy instead Value *unbox = builder.CreateLoad(vi.value.V, /*volatile*/true); builder.CreateStore(unbox, slot); - v = mark_julia_slot(slot, vi.value.typ, tbaa_stack); + Value *tindex = NULL; + if (vi.pTIndex) + tindex = builder.CreateLoad(vi.pTIndex, /*volatile*/true); + v = mark_julia_slot(slot, vi.value.typ, tindex, tbaa_stack); } + if (vi.boxroot == NULL) + v = update_julia_type(v, typ, ctx); if (vi.usedUndef) { assert(vi.defFlag); isnull = builder.CreateLoad(vi.defFlag, vi.isVolatile); } } - else { + if (vi.boxroot != NULL) { Value *boxed = builder.CreateLoad(vi.boxroot, vi.isVolatile); - v = mark_julia_type(boxed, true, typ, ctx, - /*gc-root*/!vi.isArgument); // if an argument, doesn't need an additional root + Value *box_isnull; if (vi.usedUndef) - isnull = builder.CreateICmpNE(boxed, V_null); + box_isnull = builder.CreateICmpNE(boxed, V_null); + if (vi.pTIndex) { + // value is either boxed in the stack slot, or unboxed in value + // as indicated by testing (pTIndex & 0x80) + Value *load_unbox = builder.CreateICmpEQ( + builder.CreateAnd(v.TIndex, ConstantInt::get(T_int8, 0x80)), + ConstantInt::get(T_int8, 0)); + if (vi.usedUndef) + isnull = builder.CreateSelect(load_unbox, isnull, box_isnull); + if (v.V) // v.V will be null if it is a union of all ghost values + v.V = builder.CreateSelect(load_unbox, emit_bitcast(v.V, boxed->getType()), boxed); + else + v.V = boxed; + v = update_julia_type(v, typ, ctx); + } + else { + v = mark_julia_type(boxed, true, typ, ctx, + /*gc-root*/!vi.isArgument); // if an argument, doesn't need an additional root + if (vi.usedUndef) + isnull = box_isnull; + } } if (isnull) undef_var_error_ifnot(isnull, sym, ctx); @@ -3191,6 +3560,73 @@ static jl_cgval_t emit_local(jl_value_t *slotload, jl_codectx_t *ctx) } +static void union_alloca_type(jl_uniontype_t *ut, + bool &allunbox, size_t &nbytes, size_t &align, size_t &min_align) +{ + nbytes = 0; + align = 0; + min_align = MAX_ALIGN; + // compute the size of the union alloca that could hold this type + unsigned counter = 0; + allunbox = for_each_uniontype_small( + [&](unsigned idx, jl_datatype_t *jt) { + if (!jl_is_datatype_singleton(jt)) { + size_t nb1 = jl_datatype_size(jt); + size_t align1 = jt->layout->alignment; + if (nb1 > nbytes) + nbytes = nb1; + if (align1 > align) + align = align1; + if (align1 < min_align) + min_align = align1; + } + }, + (jl_value_t*)ut, + counter); +} + +static Value *try_emit_union_alloca(jl_uniontype_t *ut, bool &allunbox, size_t &min_align, jl_codectx_t *ctx) +{ + size_t nbytes, align; + union_alloca_type(ut, allunbox, nbytes, align, min_align); + if (nbytes > 0) { + // at least some of the values can live on the stack + Type *AT = ArrayType::get(T_int8, nbytes); + AllocaInst *lv = emit_static_alloca(AT, ctx); + if (align > 1) + lv->setAlignment(align); + return lv; + } + return NULL; +} + +static Value *compute_box_tindex(Value *datatype, jl_value_t *supertype, jl_value_t *ut, jl_codectx_t *ctx) +{ + Value *tindex = ConstantInt::get(T_int8, 0); + unsigned counter = 0; + for_each_uniontype_small( + [&](unsigned idx, jl_datatype_t *jt) { + if (jl_subtype((jl_value_t*)jt, supertype)) { + Value *cmp = builder.CreateICmpEQ(literal_pointer_val((jl_value_t*)jt), datatype); + tindex = builder.CreateSelect(cmp, ConstantInt::get(T_int8, idx), tindex); + } + }, + ut, + counter); + return tindex; +} + +// get the runtime tindex value +static Value *compute_tindex_unboxed(const jl_cgval_t &val, jl_value_t *typ, jl_codectx_t *ctx) +{ + if (val.constant) + return ConstantInt::get(T_int8, get_box_tindex((jl_datatype_t*)jl_typeof(val.constant), typ)); + if (val.isboxed) + return compute_box_tindex(emit_typeof_boxed(val, ctx), val.typ, typ, ctx); + assert(val.TIndex); + return builder.CreateAnd(val.TIndex, ConstantInt::get(T_int8, 0x7f)); +} + static void emit_assignment(jl_value_t *l, jl_value_t *r, jl_codectx_t *ctx) { if (jl_is_ssavalue(l)) { @@ -3198,24 +3634,61 @@ static void emit_assignment(jl_value_t *l, jl_value_t *r, jl_codectx_t *ctx) assert(idx >= 0); assert(!ctx->ssavalue_assigned.at(idx)); jl_cgval_t slot = emit_expr(r, ctx); // slot could be a jl_value_t (unboxed) or jl_value_t* (ispointer) - if (slot.isboxed) { - // see if inference had a better type for the ssavalue than the expression (after inlining getfield on a Tuple) + if (slot.isboxed || slot.TIndex) { + // see if inference suggested a different type for the ssavalue than the expression + // e.g. sometimes the information is inconsistent after inlining getfield on a Tuple jl_value_t *ssavalue_types = (jl_value_t*)ctx->source->ssavaluetypes; if (jl_is_array(ssavalue_types)) { jl_value_t *declType = jl_array_ptr_ref(ssavalue_types, idx); if (declType != slot.typ) { - slot = remark_julia_type(slot, declType, ctx); + slot = update_julia_type(slot, declType, ctx); } } } if (!slot.isboxed && !slot.isimmutable) { // emit a copy of values stored in mutable slots - bool isboxed; - Type *vtype = julia_type_to_llvm(slot.typ, &isboxed); - assert(!isboxed); - Value *dest = emit_static_alloca(vtype); - emit_unbox(vtype, slot, slot.typ, dest); - slot = mark_julia_slot(dest, slot.typ, tbaa_stack); + Value *dest; + jl_value_t *jt = slot.typ; + if (jl_is_uniontype(jt)) { + assert(slot.TIndex && "Unboxed union must have a type-index."); + bool allunbox; + size_t min_align; + dest = try_emit_union_alloca(((jl_uniontype_t*)jt), allunbox, min_align, ctx); + Value *isboxed = NULL; + if (slot.ispointer() && slot.V != NULL && !isa(slot.V)) { + isboxed = builder.CreateICmpNE( + builder.CreateAnd(slot.TIndex, ConstantInt::get(T_int8, 0x80)), + ConstantInt::get(T_int8, 0)); + } + if (dest) + emit_unionmove(dest, slot, isboxed, false, NULL, ctx); + Value *gcroot = NULL; + if (isboxed) { + if (slot.gcroot) + gcroot = emit_local_root(ctx); + else + gcroot = emit_static_alloca(T_pjlvalue); + Value *box = builder.CreateSelect(isboxed, emit_bitcast(slot.V, T_pjlvalue), V_null); + builder.CreateStore(box, gcroot); + if (dest) // might be all ghost values + dest = builder.CreateSelect(isboxed, box, emit_bitcast(dest, box->getType())); + else + dest = box; + } + else { + assert(allunbox && "Failed to allocate correct union-type storage."); + } + slot = mark_julia_slot(dest, slot.typ, slot.TIndex, tbaa_stack); + slot.gcroot = gcroot; + } + else { + bool isboxed; + Type *vtype = julia_type_to_llvm(slot.typ, &isboxed); + assert(!isboxed); + dest = emit_static_alloca(vtype); + emit_unbox(vtype, slot, slot.typ, dest); + slot = mark_julia_slot(dest, slot.typ, NULL, tbaa_stack); + } } ctx->SAvalues.at(idx) = slot; // now SAvalues[idx] contains the SAvalue ctx->ssavalue_assigned.at(idx) = true; @@ -3258,6 +3731,17 @@ static void emit_assignment(jl_value_t *l, jl_value_t *r, jl_codectx_t *ctx) if (!vi.used) return; + bool needs_root = false; + if ((!vi.isSA && rval_info.gcroot) || !rval_info.isboxed) + // rval needed a gcroot, so lval will need one too + needs_root = true; + + // convert rval-type to lval-type + jl_value_t *slot_type = vi.value.typ; + rval_info = convert_julia_type(rval_info, slot_type, ctx, /*needs-root*/false); + if (rval_info.typ == jl_bottom_type) + return; + // add info to arrayvar list if (rval_info.isboxed) { // check isboxed in case rval isn't the right type (for example, on a dead branch), @@ -3267,26 +3751,106 @@ static void emit_assignment(jl_value_t *l, jl_value_t *r, jl_codectx_t *ctx) assign_arrayvar(*av, rval_info, ctx); } + // compute / store tindex info + if (vi.pTIndex) { + Value *tindex; + if (rval_info.TIndex) { + tindex = rval_info.TIndex; + if (!vi.boxroot) + tindex = builder.CreateAnd(tindex, ConstantInt::get(T_int8, 0x7f)); + } + else { + assert(rval_info.isboxed || rval_info.constant); + tindex = compute_tindex_unboxed(rval_info, vi.value.typ, ctx); + if (vi.boxroot) + tindex = builder.CreateOr(tindex, ConstantInt::get(T_int8, 0x80)); + if (!vi.boxroot) + rval_info.TIndex = tindex; + } + builder.CreateStore(tindex, vi.pTIndex, vi.isVolatile); + } + + // store boxed variables + Value *isboxed = NULL; if (vi.boxroot) { - // boxed variables - if (((!vi.isSA && rval_info.gcroot) || !rval_info.isboxed) && isa(vi.boxroot)) { - // rval had a gcroot, so lval needs one too: promote variable slot to a gcroot - emit_local_root(ctx, &vi); + if (isa(vi.boxroot) && needs_root) + emit_local_root(ctx, &vi); // promote variable slot to a gcroot + Value *rval; + if (vi.pTIndex && rval_info.TIndex) { + builder.CreateStore(rval_info.TIndex, vi.pTIndex, vi.isVolatile); + isboxed = builder.CreateICmpNE( + builder.CreateAnd(rval_info.TIndex, ConstantInt::get(T_int8, 0x80)), + ConstantInt::get(T_int8, 0)); + rval = V_null; + if (rval_info.ispointer() && rval_info.V != NULL && !isa(rval_info.V)) // might be all ghost values or otherwise definitely not boxed + rval = builder.CreateSelect(isboxed, emit_bitcast(rval_info.V, rval->getType()), rval); + assert(!vi.value.constant); + } + else { + assert(!vi.pTIndex || rval_info.isboxed || rval_info.constant); + rval = boxed(rval_info, ctx, false); } - Value *rval = boxed(rval_info, ctx, false); // no root needed on the temporary since it is about to be assigned to the variable slot builder.CreateStore(rval, vi.boxroot, vi.isVolatile); } - else { + + // store unboxed variables + if (!vi.boxroot || (vi.pTIndex && rval_info.TIndex)) { if (vi.usedUndef) store_def_flag(vi, true); - if (vi.value.constant) { - // virtual store + if (!vi.value.constant) { // check that this is not a virtual store + assert(vi.value.ispointer() || (vi.pTIndex && vi.value.V == NULL)); + // store value + if (vi.value.V == NULL) { + // all ghost values in destination - nothing to copy or store + } + else if (rval_info.constant || !rval_info.ispointer()) { + if (rval_info.isghost) { + // all ghost values in source - nothing to copy or store + } + else { + if (rval_info.typ != vi.value.typ && !vi.pTIndex && !rval_info.TIndex) { + // isbits cast-on-assignment is invalid. this branch should be dead-code. + CreateTrap(builder); + } + else { + Value *dest = vi.value.V; + Type *store_ty = julia_type_to_llvm(rval_info.constant ? jl_typeof(rval_info.constant) : rval_info.typ); + Type *dest_ty = store_ty->getPointerTo(); + if (dest_ty != dest->getType()) + dest = emit_bitcast(dest, dest_ty); + tbaa_decorate(tbaa_stack, builder.CreateStore( + emit_unbox(store_ty, rval_info, rval_info.typ), + dest, + vi.isVolatile)); + } + } + } + else { + MDNode *tbaa = rval_info.tbaa; + // the memcpy intrinsic does not allow to specify different alias tags + // for the load part (x.tbaa) and the store part (tbaa_stack). + // since the tbaa lattice has to be a tree we have unfortunately + // x.tbaa ∪ tbaa_stack = tbaa_root if x.tbaa != tbaa_stack + if (tbaa != tbaa_stack) + tbaa = NULL; + if (vi.pTIndex == NULL) { + assert(jl_is_leaf_type(vi.value.typ)); + Value *copy_bytes = ConstantInt::get(T_int32, jl_datatype_size(vi.value.typ)); + builder.CreateMemCpy(vi.value.V, + data_pointer(rval_info, ctx, T_pint8), + copy_bytes, + /*TODO: min_align*/1, + vi.isVolatile, + tbaa); + } + else { + emit_unionmove(vi.value.V, rval_info, isboxed, vi.isVolatile, tbaa, ctx); + } + } } else { - // store unboxed - assert(vi.value.ispointer()); - emit_unbox(julia_type_to_llvm(vi.value.typ), rval_info, vi.value.typ, vi.value.V, vi.isVolatile); + assert(vi.pTIndex == NULL); } } } @@ -3298,6 +3862,10 @@ static Value *emit_condition(const jl_cgval_t &condV, const std::string &msg, { bool isbool = (condV.typ == (jl_value_t*)jl_bool_type); if (!isbool) { + if (condV.TIndex) { + // check whether this might be bool + isbool = jl_subtype((jl_value_t*)jl_bool_type, condV.typ); + } emit_typecheck(condV, (jl_value_t*)jl_bool_type, msg, ctx); } if (isbool) { @@ -3339,7 +3907,7 @@ static void emit_stmtpos(jl_value_t *expr, jl_codectx_t *ctx) Value *lv = vi.boxroot; if (lv != NULL) builder.CreateStore(V_null, lv); - else + if (lv == NULL || vi.pTIndex != NULL) store_def_flag(vi, false); } return; @@ -3453,9 +4021,7 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx) // some intrinsics (e.g. typeassert) can return a wider type // than what's actually possible jl_value_t *expr_t = expr_type((jl_value_t*)ex, ctx); - if (res.typ != expr_t && res.isboxed && !jl_is_leaf_type(res.typ)) { - res = remark_julia_type(res, expr_t, ctx); - } + res = update_julia_type(res, expr_t, ctx); if (res.typ == jl_bottom_type || expr_t == jl_bottom_type) { CreateTrap(builder); } @@ -3694,7 +4260,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t ctx.linfo = lam; ctx.code = NULL; ctx.world = world; - ctx.sret = false; + ctx.has_sret = false; ctx.spvals_ptr = NULL; ctx.params = &jl_default_cgparams; allocate_gc_frame(b0, &ctx); @@ -3730,6 +4296,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t // See whether this function is specsig or jlcall or generic (unknown) bool specsig, jlfunc_sret; + jl_returninfo_t::CallingConv cc = jl_returninfo_t::Boxed; Function *theFptr; Value *result; Value *myargs; @@ -3745,15 +4312,27 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t theFptr = NULL; } else if (lam && lam->functionObjectsDecls.specFunctionObject != NULL) { - theFptr = (Function*)lam->functionObjectsDecls.specFunctionObject; + Function *proto = (Function*)lam->functionObjectsDecls.specFunctionObject; + jl_returninfo_t returninfo = get_specsig_function(M, proto->getName(), lam->specTypes, lam->rettype); + FunctionType *cft = returninfo.decl->getFunctionType(); + assert(proto->getFunctionType() == cft); + proto = cast(prepare_call(proto)); + if (proto != returninfo.decl) { + assert(proto->getFunctionType() == cft); + returninfo.decl->replaceAllUsesWith(proto); + returninfo.decl->eraseFromParent(); + returninfo.decl = proto; + } + theFptr = returninfo.decl; specsig = true; - jlfunc_sret = theFptr->hasStructRetAttr(); - if (jlfunc_sret) { + cc = returninfo.cc; + jlfunc_sret = cc == jl_returninfo_t::SRet; + if (jlfunc_sret || cc == jl_returninfo_t::Union) { // fuse the two sret together, or emit an alloca to hold it - if (sig.sret) - result = emit_bitcast(sretPtr, theFptr->getFunctionType()->getParamType(0)); + if (sig.sret && jlfunc_sret) + result = emit_bitcast(sretPtr, cft->getParamType(0)); else - result = builder.CreateAlloca(theFptr->getFunctionType()->getParamType(0)->getContainedType(0)); + result = builder.CreateAlloca(cft->getParamType(0)->getContainedType(0)); args.push_back(result); FParamIndex++; } @@ -3883,7 +4462,6 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t } else { assert(theFptr); - bool retboxed; Value *call_v = prepare_call(theFptr); if (age_ok) { funcName << "_gfthunk"; @@ -3899,8 +4477,27 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t } CallInst *call = builder.CreateCall(call_v, ArrayRef(args)); call->setAttributes(theFptr->getAttributes()); - (void)julia_type_to_llvm(astrt, &retboxed); - retval = mark_julia_type(jlfunc_sret ? (Value*)builder.CreateLoad(result) : (Value*)call, retboxed, astrt, &ctx); + switch (cc) { + case jl_returninfo_t::Boxed: + retval = mark_julia_type(call, true, astrt, &ctx); + break; + case jl_returninfo_t::Register: + retval = mark_julia_type(call, false, astrt, &ctx); + break; + case jl_returninfo_t::SRet: + retval = mark_julia_slot(result, astrt, NULL, tbaa_stack); + break; + case jl_returninfo_t::Union: + retval = mark_julia_slot(builder.CreateExtractValue(call, 0), + astrt, + builder.CreateExtractValue(call, 1), + tbaa_stack); + // note that the value may not be rooted here (on the return path) + break; + case jl_returninfo_t::Ghosts: + retval = mark_julia_slot(NULL, astrt, call, tbaa_stack); + break; + } } } else { @@ -3963,6 +4560,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t } else if (sig.sret && jlfunc_sret) { // nothing to do + r = NULL; } else if (!type_is_ghost(sig.lrt)) { Type *prt = sig.prt; @@ -3972,21 +4570,23 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t Value *v = julia_to_native(sig.lrt, toboxed, declrt, NULL, retval, false, 0, &ctx, NULL); r = llvm_type_rewrite(v, prt, issigned, &ctx); - if (sig.sret) + if (sig.sret) { builder.CreateStore(r, sretPtr); + r = NULL; + } } else { assert(type_is_ghost(sig.lrt)); sig.sret = true; + r = NULL; } builder.CreateStore(last_age, ctx.world_age_field); - if (sig.sret) - builder.CreateRetVoid(); - else - builder.CreateRet(r); + builder.CreateRet(r); // also need to finish emission of our specsig -> jl_apply_generic converter thunk + // this builds a method that calls jl_apply_generic (as a closure over a singleton function pointer), + // but which has the signature of a specsig if (gf_thunk) { assert(lam && specsig); BasicBlock *b0 = BasicBlock::Create(jl_LLVMContext, "top", gf_thunk); @@ -3998,7 +4598,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t ctx2.linfo = NULL; ctx2.code = NULL; ctx2.world = world; - ctx2.sret = false; + ctx2.has_sret = false; ctx2.spvals_ptr = NULL; ctx2.params = &jl_default_cgparams; allocate_gc_frame(b0, &ctx2); @@ -4008,7 +4608,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t ConstantInt::get(T_int32, nargs + 1), "", /*InsertBefore*/ctx2.ptlsStates); - if (jlfunc_sret) + if (cc == jl_returninfo_t::SRet || cc == jl_returninfo_t::Union) ++AI; for (size_t i = 0; i < nargs + 1; i++) { jl_value_t *jt = jl_nth_slot_type(lam->specTypes, i); @@ -4028,7 +4628,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t arg_box = arg_v; } else if (et->isAggregateType()) { - arg_box = boxed(mark_julia_slot(arg_v, jt, tbaa_const), &ctx2, false); + arg_box = boxed(mark_julia_slot(arg_v, jt, NULL, tbaa_const), &ctx2, false); } else { assert(at == et); @@ -4046,28 +4646,47 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t #else Value *gf_ret = builder.CreateCall2(prepare_call(jlapplygeneric_func), myargs, nargs_v); #endif - bool retboxed; - (void)julia_type_to_llvm(astrt, &retboxed); - if (retboxed) { - assert(!jlfunc_sret); - builder.CreateRet(gf_ret); - } - else { - jl_cgval_t gf_retbox = mark_julia_type(gf_ret, true, jl_any_type, &ctx2); + jl_cgval_t gf_retbox = mark_julia_type(gf_ret, true, jl_any_type, &ctx, /*needsroot*/false); + if (cc != jl_returninfo_t::Boxed) { emit_typecheck(gf_retbox, astrt, "cfunction", &ctx2); + } + + switch (cc) { + case jl_returninfo_t::Boxed: + builder.CreateRet(gf_ret); + break; + case jl_returninfo_t::Register: { Type *gfrt = gf_thunk->getReturnType(); - if (jlfunc_sret) { - unsigned sret_nbytes = jl_datatype_size(astrt); - builder.CreateMemCpy(&*gf_thunk->arg_begin(), gf_ret, sret_nbytes, jl_alignment(sret_nbytes)); - builder.CreateRetVoid(); - } - else if (gfrt->isVoidTy()) { + if (gfrt->isVoidTy()) { builder.CreateRetVoid(); } else { gf_ret = emit_bitcast(gf_ret, gfrt->getPointerTo()); builder.CreateRet(builder.CreateLoad(gf_ret)); } + break; + } + case jl_returninfo_t::SRet: { + unsigned sret_nbytes = jl_datatype_size(astrt); + builder.CreateMemCpy(&*gf_thunk->arg_begin(), gf_ret, sret_nbytes, jl_alignment(sret_nbytes)); + builder.CreateRetVoid(); + break; + } + case jl_returninfo_t::Union: { + Type *retty = theFptr->getReturnType(); + Value *gf_retval = UndefValue::get(retty); + Value *tindex = compute_box_tindex(gf_ret, (jl_value_t*)jl_any_type, astrt, &ctx); + tindex = builder.CreateOr(tindex, ConstantInt::get(T_int8, 0x80)); + gf_retval = builder.CreateInsertValue(gf_retval, gf_ret, 0); + gf_retval = builder.CreateInsertValue(gf_retval, tindex, 1); + builder.CreateRet(gf_retval); + break; + } + case jl_returninfo_t::Ghosts: { + Value *gf_retval = compute_tindex_unboxed(gf_retbox, astrt, &ctx); + builder.CreateRet(gf_retval); + break; + } } } @@ -4181,7 +4800,7 @@ static Function *jl_cfunction_object(jl_function_t *ff, jl_value_t *declrt, jl_t } // generate a julia-callable function that calls f (AKA lam) -static Function *gen_jlcall_wrapper(jl_method_instance_t *lam, Function *f, const std::string &funcName, bool sret, Module *M) +static Function *gen_jlcall_wrapper(jl_method_instance_t *lam, const jl_returninfo_t &f, const std::string &funcName, Module *M) { Function *w = Function::Create(jl_func_sig, GlobalVariable::ExternalLinkage, funcName, M); @@ -4204,20 +4823,34 @@ static Function *gen_jlcall_wrapper(jl_method_instance_t *lam, Function *f, cons ctx.linfo = lam; ctx.code = NULL; ctx.world = 0; - ctx.sret = false; + ctx.has_sret = false; ctx.spvals_ptr = NULL; ctx.params = &jl_default_cgparams; allocate_gc_frame(b0, &ctx); + FunctionType *ftype = f.decl->getFunctionType(); size_t nargs = lam->def->nargs; - size_t nfargs = f->getFunctionType()->getNumParams(); + size_t nfargs = ftype->getNumParams(); Value **args = (Value**) alloca(nfargs*sizeof(Value*)); unsigned idx = 0; - Value *result; - if (sret) { - result = builder.CreateAlloca(f->getFunctionType()->getParamType(0)->getContainedType(0)); + AllocaInst *result; + switch (f.cc) { + case jl_returninfo_t::Boxed: + case jl_returninfo_t::Register: + case jl_returninfo_t::Ghosts: + break; + case jl_returninfo_t::SRet: + result = builder.CreateAlloca(ftype->getParamType(0)->getContainedType(0)); args[idx] = result; idx++; + break; + case jl_returninfo_t::Union: + result = builder.CreateAlloca(ArrayType::get(T_int8, f.union_bytes)); + if (f.union_align > 1) + result->setAlignment(f.union_align); + args[idx] = result; + idx++; + break; } for (size_t i = 0; i < nargs; i++) { jl_value_t *ty = jl_nth_slot_type(lam->specTypes, i); @@ -4242,41 +4875,76 @@ static Function *gen_jlcall_wrapper(jl_method_instance_t *lam, Function *f, cons args[idx] = theArg; idx++; } - CallInst *call = builder.CreateCall(prepare_call(f), ArrayRef(&args[0], nfargs)); - call->setAttributes(f->getAttributes()); + CallInst *call = builder.CreateCall(f.decl, ArrayRef(&args[0], nfargs)); + call->setAttributes(f.decl->getAttributes()); jl_value_t *jlretty = lam->rettype; - bool retboxed; - (void)julia_type_to_llvm(jlretty, &retboxed); - if (sret) { assert(!retboxed); } - jl_cgval_t retval = sret - ? mark_julia_slot(result, jlretty, tbaa_stack) - : mark_julia_type(call, retboxed, jlretty, &ctx, /*needsroot*/false); + jl_cgval_t retval; + switch (f.cc) { + case jl_returninfo_t::Boxed: + retval = mark_julia_type(call, true, jlretty, &ctx, /*needsroot*/false); + break; + case jl_returninfo_t::Register: + retval = mark_julia_type(call, false, jlretty, &ctx, /*needsroot*/false); + break; + case jl_returninfo_t::SRet: + retval = mark_julia_slot(result, jlretty, NULL, tbaa_stack); + break; + case jl_returninfo_t::Union: + retval = mark_julia_slot(builder.CreateExtractValue(call, 0), + jlretty, + builder.CreateExtractValue(call, 1), + tbaa_stack); + break; + case jl_returninfo_t::Ghosts: + retval = mark_julia_slot(NULL, jlretty, call, tbaa_stack); + break; + } builder.CreateRet(boxed(retval, &ctx, false)); // no gcroot needed since this on the return path - assert(!ctx.roots); return w; } -static Function *get_specsig_function(Module *M, const std::string &name, jl_value_t *sig, jl_value_t *jlrettype, bool &sret) +static jl_returninfo_t get_specsig_function(Module *M, const std::string &name, jl_value_t *sig, jl_value_t *jlrettype) { + jl_returninfo_t props = {}; SmallVector fsig; Type *rt; - bool retboxed; if (jlrettype == (jl_value_t*)jl_void_type) { rt = T_void; - retboxed = false; + props.cc = jl_returninfo_t::Register; + } + else if (jl_is_uniontype(jlrettype)) { + bool allunbox; + union_alloca_type((jl_uniontype_t*)jlrettype, allunbox, props.union_bytes, props.union_align, props.union_minalign); + if (props.union_bytes) { + props.cc = jl_returninfo_t::Union; + Type *AT = ArrayType::get(T_int8, props.union_bytes); + fsig.push_back(AT->getPointerTo()); + Type *pair[] = { T_pjlvalue, T_int8 }; + rt = StructType::get(jl_LLVMContext, makeArrayRef(pair)); + } + else if (allunbox) { + props.cc = jl_returninfo_t::Ghosts; + rt = T_int8; + } + else { + rt = T_pjlvalue; + } } else { + bool retboxed; rt = julia_type_to_llvm(jlrettype, &retboxed); - } - if (!retboxed && rt != T_void && deserves_sret(jlrettype, rt)) { - sret = true; - fsig.push_back(rt->getPointerTo()); - rt = T_void; - } - else { - sret = false; + if (!retboxed) { + if (rt != T_void && deserves_sret(jlrettype, rt)) { + props.cc = jl_returninfo_t::SRet; + fsig.push_back(rt->getPointerTo()); + rt = T_void; + } + else { + props.cc = jl_returninfo_t::Register; + } + } } for (size_t i = 0; i < jl_nparams(sig); i++) { jl_value_t *jt = jl_tparam(sig, i); @@ -4289,11 +4957,17 @@ static Function *get_specsig_function(Module *M, const std::string &name, jl_val } FunctionType *ftype = FunctionType::get(rt, fsig, false); Function *f = Function::Create(ftype, GlobalVariable::ExternalLinkage, name, M); - if (sret) { + if (props.cc == jl_returninfo_t::SRet) { f->addAttribute(1, Attribute::StructRet); f->addAttribute(1, Attribute::NoAlias); + f->addAttribute(1, Attribute::NoCapture); } - return f; + if (props.cc == jl_returninfo_t::Union) { + f->addAttribute(1, Attribute::NoAlias); + f->addAttribute(1, Attribute::NoCapture); + } + props.decl = f; + return props; } #if JL_LLVM_VERSION >= 30700 @@ -4503,16 +5177,18 @@ static std::unique_ptr emit_function( // allocate Function declarations and wrapper objects Module *M = new Module(ctx.name, jl_LLVMContext); jl_setup_module(M); + jl_returninfo_t returninfo = {}; Function *f = NULL; Function *fwrap = NULL; - ctx.sret = false; if (specsig) { // assumes !va and !needsparams std::stringstream specName; specName << "julia_" << unadorned_name << "_" << globalUnique; - f = get_specsig_function(M, specName.str(), lam->specTypes, jlrettype, ctx.sret); + returninfo = get_specsig_function(M, specName.str(), lam->specTypes, jlrettype); + f = returninfo.decl; + ctx.has_sret = (returninfo.cc == jl_returninfo_t::SRet || returninfo.cc == jl_returninfo_t::Union); jl_init_function(f); - fwrap = gen_jlcall_wrapper(lam, f, funcName.str(), ctx.sret, M); + fwrap = gen_jlcall_wrapper(lam, returninfo, funcName.str(), M); declarations->functionObject = function_proto(fwrap); declarations->specFunctionObject = function_proto(f); } @@ -4520,6 +5196,7 @@ static std::unique_ptr emit_function( f = Function::Create(needsparams ? jl_func_sig_sparams : jl_func_sig, GlobalVariable::ExternalLinkage, funcName.str(), M); + returninfo.decl = f; jl_init_function(f); declarations->functionObject = function_proto(f); declarations->specFunctionObject = NULL; @@ -4633,7 +5310,7 @@ static std::unique_ptr emit_function( varinfo.dinfo = dbuilder.createParameterVariable( SP, // Scope (current function will be fill in later) jl_symbol_name(argname), // Variable name - ctx.sret + i + 1, // Argument number (1-based) + ctx.has_sret + i + 1, // Argument number (1-based) topfile, // File toplineno == -1 ? 0 : toplineno, // Line // Variable type @@ -4643,38 +5320,38 @@ static std::unique_ptr emit_function( #else varinfo.dinfo = dbuilder.createLocalVariable( llvm::dwarf::DW_TAG_arg_variable, // Tag - SP, // Scope (current function will be fill in later) - jl_symbol_name(argname), // Variable name - topfile, // File + SP, // Scope (current function will be fill in later) + jl_symbol_name(argname), // Variable name + topfile, // File toplineno == -1 ? 0 : toplineno, // Line (for now, use lineno of the function) julia_type_to_di(varinfo.value.typ, &dbuilder, false), // Variable type - AlwaysPreserve, // May be deleted if optimized out - 0, // Flags (TODO: Do we need any) - ctx.sret + i + 1); // Argument number (1-based) + AlwaysPreserve, // May be deleted if optimized out + 0, // Flags (TODO: Do we need any) + ctx.has_sret + i + 1); // Argument number (1-based) #endif } if (va && ctx.vaSlot != -1) { #if JL_LLVM_VERSION >= 30800 ctx.slots[ctx.vaSlot].dinfo = dbuilder.createParameterVariable( - SP, // Scope (current function will be fill in later) + SP, // Scope (current function will be fill in later) std::string(jl_symbol_name(slot_symbol(ctx.vaSlot, &ctx))) + "...", // Variable name - ctx.sret + nreq + 1, // Argument number (1-based) - topfile, // File - toplineno == -1 ? 0 : toplineno, // Line (for now, use lineno of the function) + ctx.has_sret + nreq + 1, // Argument number (1-based) + topfile, // File + toplineno == -1 ? 0 : toplineno, // Line (for now, use lineno of the function) julia_type_to_di(ctx.slots[ctx.vaSlot].value.typ, &dbuilder, false), - AlwaysPreserve, // May be deleted if optimized out - DIFlagZero); // Flags (TODO: Do we need any) + AlwaysPreserve, // May be deleted if optimized out + DIFlagZero); // Flags (TODO: Do we need any) #else ctx.slots[ctx.vaSlot].dinfo = dbuilder.createLocalVariable( llvm::dwarf::DW_TAG_arg_variable, // Tag SP, // Scope (current function will be fill in later) std::string(jl_symbol_name(slot_symbol(ctx.vaSlot, &ctx))) + "...", // Variable name topfile, // File - toplineno == -1 ? 0 : toplineno, // Line (for now, use lineno of the function) + toplineno == -1 ? 0 : toplineno, // Line (for now, use lineno of the function) julia_type_to_di(ctx.slots[ctx.vaSlot].value.typ, &dbuilder, false), // Variable type - AlwaysPreserve, // May be deleted if optimized out - 0, // Flags (TODO: Do we need any) - ctx.sret + nreq + 1); // Argument number (1-based) + AlwaysPreserve, // May be deleted if optimized out + 0, // Flags (TODO: Do we need any) + ctx.has_sret + nreq + 1); // Argument number (1-based) #endif } for (i = 0; i < vinfoslen; i++) { @@ -4768,6 +5445,31 @@ static std::unique_ptr emit_function( if (i != (size_t)ctx.vaSlot && isbits_spec(jt, false)) continue; } + else if (jl_is_uniontype(jt)) { + bool allunbox; + size_t align; + Value *lv = try_emit_union_alloca((jl_uniontype_t*)jt, allunbox, align, &ctx); + if (lv) { + lv->setName(jl_symbol_name(s)); + varinfo.value = mark_julia_slot(lv, jt, NULL, tbaa_stack); + varinfo.pTIndex = emit_static_alloca(T_int8, &ctx); + // the slot is not immutable if there are multiple assignments + varinfo.value.isimmutable &= varinfo.isSA; + } + else if (allunbox) { + // all ghost values just need a selector allocated + AllocaInst *lv = emit_static_alloca(T_int8, &ctx); + lv->setName(jl_symbol_name(s)); + varinfo.pTIndex = lv; + varinfo.value.tbaa = NULL; + varinfo.value.isboxed = false; + varinfo.value.isimmutable = true; + } + if (lv || allunbox) + alloc_def_flag(varinfo, &ctx); + if (allunbox) + continue; + } else if (isbits_spec(jt, false)) { bool isboxed; Type *vtype = julia_type_to_llvm(jt, &isboxed); @@ -4775,7 +5477,7 @@ static std::unique_ptr emit_function( assert(!type_is_ghost(vtype) && "constants should already be handled"); // CreateAlloca is OK during prologue setup Value *lv = builder.CreateAlloca(vtype, NULL, jl_symbol_name(s)); - varinfo.value = mark_julia_slot(lv, jt, tbaa_stack); + varinfo.value = mark_julia_slot(lv, jt, NULL, tbaa_stack); // slot is not immutable if there are multiple assignments varinfo.value.isimmutable &= varinfo.isSA; alloc_def_flag(varinfo, &ctx); @@ -4821,7 +5523,7 @@ static std::unique_ptr emit_function( // step 9. move args into local variables Function::arg_iterator AI = f->arg_begin(); - if (ctx.sret) + if (ctx.has_sret) AI++; // skip sret slot for (i = 0; i < nreq; i++) { jl_sym_t *s = (jl_sym_t*)jl_array_ptr_ref(src->slotnames, i); @@ -4846,7 +5548,7 @@ static std::unique_ptr emit_function( theArg = ghostValue(argType); } else if (llvmArgType->isAggregateType()) { - theArg = mark_julia_slot(&*AI++, argType, tbaa_const); // this argument is by-pointer + theArg = mark_julia_slot(&*AI++, argType, NULL, tbaa_const); // this argument is by-pointer theArg.isimmutable = true; } else { @@ -5246,36 +5948,101 @@ static std::unique_ptr emit_function( continue; } if (expr && expr->head == return_sym) { - bool retboxed = false; - Type *retty; - if (specsig) { - retty = julia_type_to_llvm(jlrettype, &retboxed); - } - else { - retty = T_pjlvalue; - retboxed = true; - } + // this is basically a copy of emit_assignment, + // but where the assignment slot is the retval jl_cgval_t retvalinfo = emit_expr(jl_exprarg(expr, 0), &ctx); + retvalinfo = convert_julia_type(retvalinfo, jlrettype, &ctx, /*needs-root*/false); + if (retvalinfo.typ == jl_bottom_type) { + builder.CreateUnreachable(); + find_next_stmt(-1); + continue; + } + + Value *isboxed_union = NULL; Value *retval; - if (retboxed) { + Value *sret = ctx.has_sret ? &*f->arg_begin() : NULL; + Type *retty = f->getReturnType(); + switch (returninfo.cc) { + case jl_returninfo_t::Boxed: retval = boxed(retvalinfo, &ctx, false); // skip the gcroot on the return path - assert(!ctx.sret); + break; + case jl_returninfo_t::Register: + if (type_is_ghost(retty)) + retval = NULL; + else + retval = emit_unbox(retty, retvalinfo, jlrettype); + break; + case jl_returninfo_t::SRet: + retval = NULL; + break; + case jl_returninfo_t::Union: { + Value *data, *tindex; + if (retvalinfo.TIndex) { + tindex = retvalinfo.TIndex; + if (retvalinfo.V == NULL) { + // treat this as a simple Ghosts + data = V_null; + sret = NULL; + } + else { + data = emit_bitcast(sret, T_pjlvalue); + if (retvalinfo.ispointer() && !isa(retvalinfo.V)) { + // also need to account for the possibility the return object is boxed + // and avoid / skip copying it to the stack + isboxed_union = builder.CreateICmpNE( + builder.CreateAnd(tindex, ConstantInt::get(T_int8, 0x80)), + ConstantInt::get(T_int8, 0)); + data = builder.CreateSelect(isboxed_union, emit_bitcast(retvalinfo.V, T_pjlvalue), data); + } + } + } + else { + // treat this as a simple boxed returninfo + //assert(retvalinfo.isboxed); + tindex = compute_tindex_unboxed(retvalinfo, jlrettype, &ctx); + tindex = builder.CreateOr(tindex, ConstantInt::get(T_int8, 0x80)); + data = boxed(retvalinfo, &ctx, false); // skip the gcroot on the return path + sret = NULL; + } + retval = UndefValue::get(retty); + retval = builder.CreateInsertValue(retval, data, 0); + retval = builder.CreateInsertValue(retval, tindex, 1); + break; } - else if (!type_is_ghost(retty)) { - retval = emit_unbox(retty, retvalinfo, jlrettype, - ctx.sret ? &*ctx.f->arg_begin() : NULL); + case jl_returninfo_t::Ghosts: + retval = compute_tindex_unboxed(retvalinfo, jlrettype, &ctx); + break; } - else { // undef return type - retval = NULL; + if (sret) { + if (retvalinfo.ispointer()) { + if (returninfo.cc == jl_returninfo_t::SRet) { + assert(jl_is_leaf_type(jlrettype)); + Value *copy_bytes = ConstantInt::get(T_int32, jl_datatype_size(jlrettype)); + builder.CreateMemCpy(sret, + data_pointer(retvalinfo, &ctx, T_pint8), + copy_bytes, + returninfo.union_minalign); + } + else { + emit_unionmove(sret, retvalinfo, isboxed_union, false, NULL, &ctx); + } + } + else { + Type *store_ty = julia_type_to_llvm(retvalinfo.typ); + Type *dest_ty = store_ty->getPointerTo(); + if (dest_ty != sret->getType()) + sret = emit_bitcast(sret, dest_ty); + builder.CreateStore(emit_unbox(store_ty, retvalinfo, retvalinfo.typ), sret); + } } + if (do_malloc_log(props.in_user_code) && props.line != -1) mallocVisitLine(props.file, props.line); if (toplevel) builder.CreateStore(last_age, ctx.world_age_field); - if (type_is_ghost(retty) || ctx.sret) - builder.CreateRetVoid(); - else - builder.CreateRet(retval); + assert(type_is_ghost(retty) || returninfo.cc == jl_returninfo_t::SRet || + retval->getType() == ctx.f->getReturnType()); + builder.CreateRet(retval); find_next_stmt(-1); continue; } @@ -5455,8 +6222,7 @@ extern "C" void jl_fptr_to_llvm(jl_fptr_t fptr, jl_method_instance_t *lam, int s #endif funcName << unadorned_name << "_" << globalUnique++; if (specsig) { // assumes !va - bool sret; - Function *f = get_specsig_function(shadow_output, funcName.str(), lam->specTypes, lam->rettype, sret); + Function *f = get_specsig_function(shadow_output, funcName.str(), lam->specTypes, lam->rettype).decl; if (lam->functionObjectsDecls.specFunctionObject == NULL) { lam->functionObjectsDecls.specFunctionObject = (void*)f; } diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index f865217b7c69b..78e4eca282bcb 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -301,7 +301,7 @@ static Value *emit_unbox(Type *to, const jl_cgval_t &x, jl_value_t *jt, Value *d // bools may be stored internally as int8 unboxed = builder.CreateZExt(unboxed, T_int8); } - else { + else if (ty != to) { unboxed = builder.CreateBitCast(unboxed, to); } if (!dest) @@ -459,13 +459,14 @@ static jl_cgval_t generic_bitcast(const jl_cgval_t *argv, jl_codectx_t *ctx) vx = emit_bitcast(vx, llvmt); } - if (jl_is_leaf_type(bt)) + if (jl_is_leaf_type(bt)) { return mark_julia_type(vx, false, bt, ctx); - else - return mark_julia_type( - init_bits_value(emit_allocobj(ctx, nb, boxed(bt_value, ctx)), - vx, tbaa_immut), - true, bt, ctx); + } + else { + Value *box = emit_allocobj(ctx, nb, boxed(bt_value, ctx)); + init_bits_value(box, vx, tbaa_immut); + return mark_julia_type(box, true, bt, ctx); + } } static jl_cgval_t generic_cast(