Skip to content

Commit 4109326

Browse files
vtjnashvtjnash
committed
better optimize code for copying a value to an unboxed union
1 parent 76edf67 commit 4109326

File tree

2 files changed

+89
-34
lines changed

2 files changed

+89
-34
lines changed

‎src/cgutils.cpp

+68-2
Original file line numberDiff line numberDiff line change
@@ -320,7 +320,7 @@ static Value *literal_pointer_val_slot(jl_value_t *p)
320320
static Value *literal_pointer_val(jl_value_t *p)
321321
{
322322
if (p == NULL)
323-
return ConstantPointerNull::get((PointerType*)T_pjlvalue);
323+
return V_null;
324324
if (!imaging_mode)
325325
return literal_static_pointer_val(p, T_pjlvalue);
326326
Value *pgv = literal_pointer_val_slot(p);
@@ -331,7 +331,7 @@ static Value *literal_pointer_val(jl_binding_t *p)
331331
{
332332
// emit a pointer to any jl_value_t which will be valid across reloading code
333333
if (p == NULL)
334-
return ConstantPointerNull::get((PointerType*)T_pjlvalue);
334+
return V_null;
335335
if (!imaging_mode)
336336
return literal_static_pointer_val(p, T_pjlvalue);
337337
// bindings are prefixed with jl_bnd#
@@ -764,6 +764,7 @@ static Value *emit_datatype_size(Value *dt)
764764
return size;
765765
}
766766

767+
/* this is valid code, it's simply unused
767768
static Value *emit_sizeof(const jl_cgval_t &p, jl_codectx_t *ctx)
768769
{
769770
if (p.TIndex) {
@@ -810,6 +811,7 @@ static Value *emit_sizeof(const jl_cgval_t &p, jl_codectx_t *ctx)
810811
return dyn_size;
811812
}
812813
}
814+
*/
813815

814816
static Value *emit_datatype_mutabl(Value *dt)
815817
{
@@ -1956,6 +1958,70 @@ static Value *boxed(const jl_cgval_t &vinfo, jl_codectx_t *ctx, bool gcrooted)
19561958
return box;
19571959
}
19581960

1961+
// copy src to dest, if src is isbits. if skip is true, the value of dest is undefined
1962+
static void emit_unionmove(Value *dest, const jl_cgval_t &src, Value *skip, bool isVolatile, MDNode *tbaa, jl_codectx_t *ctx)
1963+
{
1964+
if (jl_is_leaf_type(src.typ) || src.constant) {
1965+
jl_value_t *typ = src.constant ? jl_typeof(src.constant) : src.typ;
1966+
Type *store_ty = julia_type_to_llvm(typ);
1967+
assert(skip || jl_isbits(typ));
1968+
if (jl_isbits(typ)) {
1969+
if (!src.ispointer() || src.constant) {
1970+
emit_unbox(store_ty, src, typ, dest, isVolatile);
1971+
}
1972+
else {
1973+
Value *src_ptr = data_pointer(src, ctx, T_pint8);
1974+
if (dest->getType() != T_pint8)
1975+
dest = emit_bitcast(dest, T_pint8);
1976+
if (skip) // copy dest -> dest to simulate an undef value / conditional copy
1977+
src_ptr = builder.CreateSelect(skip, dest, src_ptr);
1978+
unsigned nb = jl_datatype_size(typ);
1979+
unsigned alignment = 0;
1980+
builder.CreateMemCpy(dest, src_ptr, nb, alignment, tbaa);
1981+
}
1982+
}
1983+
}
1984+
else if (src.TIndex) {
1985+
Value *tindex = builder.CreateAnd(src.TIndex, ConstantInt::get(T_int8, 0x7f));
1986+
Value *copy_bytes = ConstantInt::get(T_int32, -1);
1987+
unsigned counter = 0;
1988+
bool allunboxed = for_each_uniontype_small(
1989+
[&](unsigned idx, jl_datatype_t *jt) {
1990+
Value *cmp = builder.CreateICmpEQ(tindex, ConstantInt::get(T_int8, idx));
1991+
copy_bytes = builder.CreateSelect(cmp, ConstantInt::get(T_int32, jl_datatype_size(jt)), copy_bytes);
1992+
},
1993+
src.typ,
1994+
counter);
1995+
Value *src_ptr = data_pointer(src, ctx, T_pint8);
1996+
if (dest->getType() != T_pint8)
1997+
dest = emit_bitcast(dest, T_pint8);
1998+
if (skip) {
1999+
if (allunboxed) // copy dest -> dest to simulate an undef value / conditional copy
2000+
src_ptr = builder.CreateSelect(skip, dest, src_ptr);
2001+
else
2002+
copy_bytes = builder.CreateSelect(skip, ConstantInt::get(copy_bytes->getType(), 0), copy_bytes);
2003+
}
2004+
#ifndef NDEBUG
2005+
// try to catch codegen errors early, before it uses this to memcpy over the entire stack
2006+
CreateConditionalAbort(builder, builder.CreateICmpEQ(copy_bytes, ConstantInt::get(T_int32, -1)));
2007+
#endif
2008+
builder.CreateMemCpy(dest,
2009+
src_ptr,
2010+
copy_bytes,
2011+
/*TODO: min-align*/1);
2012+
}
2013+
else {
2014+
Value *datatype = emit_typeof_boxed(src, ctx);
2015+
Value *copy_bytes = emit_datatype_size(datatype);
2016+
if (skip)
2017+
copy_bytes = builder.CreateSelect(skip, ConstantInt::get(copy_bytes->getType(), 0), copy_bytes);
2018+
builder.CreateMemCpy(dest,
2019+
data_pointer(src, ctx, T_pint8),
2020+
copy_bytes,
2021+
/*TODO: min-align*/1);
2022+
}
2023+
}
2024+
19592025

19602026
static void emit_cpointercheck(const jl_cgval_t &x, const std::string &msg, jl_codectx_t *ctx)
19612027
{

‎src/codegen.cpp

+21-32
Original file line numberDiff line numberDiff line change
@@ -3659,15 +3659,8 @@ static void emit_assignment(jl_value_t *l, jl_value_t *r, jl_codectx_t *ctx)
36593659
builder.CreateAnd(slot.TIndex, ConstantInt::get(T_int8, 0x80)),
36603660
ConstantInt::get(T_int8, 0));
36613661
}
3662-
if (dest) {
3663-
Value *copy_bytes = emit_sizeof(slot, ctx);
3664-
if (isboxed)
3665-
copy_bytes = builder.CreateSelect(isboxed, ConstantInt::get(copy_bytes->getType(), 0), copy_bytes);
3666-
builder.CreateMemCpy(dest,
3667-
data_pointer(slot, ctx, T_pint8),
3668-
copy_bytes,
3669-
min_align);
3670-
}
3662+
if (dest)
3663+
emit_unionmove(dest, slot, isboxed, false, NULL, ctx);
36713664
Value *gcroot = NULL;
36723665
if (isboxed) {
36733666
if (slot.gcroot)
@@ -3770,6 +3763,8 @@ static void emit_assignment(jl_value_t *l, jl_value_t *r, jl_codectx_t *ctx)
37703763
tindex = compute_tindex_unboxed(rval_info, vi.value.typ, ctx);
37713764
if (vi.boxroot)
37723765
tindex = builder.CreateOr(tindex, ConstantInt::get(T_int8, 0x80));
3766+
if (!vi.boxroot)
3767+
rval_info.TIndex = tindex;
37733768
}
37743769
builder.CreateStore(tindex, vi.pTIndex, vi.isVolatile);
37753770
}
@@ -3838,22 +3833,19 @@ static void emit_assignment(jl_value_t *l, jl_value_t *r, jl_codectx_t *ctx)
38383833
// x.tbaa ∪ tbaa_stack = tbaa_root if x.tbaa != tbaa_stack
38393834
if (tbaa != tbaa_stack)
38403835
tbaa = NULL;
3841-
Value *copy_bytes;
38423836
if (vi.pTIndex == NULL) {
38433837
assert(jl_is_leaf_type(vi.value.typ));
3844-
copy_bytes = ConstantInt::get(T_int32, jl_datatype_size(vi.value.typ));
3838+
Value *copy_bytes = ConstantInt::get(T_int32, jl_datatype_size(vi.value.typ));
3839+
builder.CreateMemCpy(vi.value.V,
3840+
data_pointer(rval_info, ctx, T_pint8),
3841+
copy_bytes,
3842+
/*TODO: min_align*/1,
3843+
vi.isVolatile,
3844+
tbaa);
38453845
}
38463846
else {
3847-
copy_bytes = emit_sizeof(rval_info, ctx);
3848-
if (isboxed)
3849-
copy_bytes = builder.CreateSelect(isboxed, ConstantInt::get(copy_bytes->getType(), 0), copy_bytes);
3847+
emit_unionmove(vi.value.V, rval_info, isboxed, vi.isVolatile, tbaa, ctx);
38503848
}
3851-
builder.CreateMemCpy(vi.value.V,
3852-
data_pointer(rval_info, ctx, T_pint8),
3853-
copy_bytes,
3854-
/*TODO: min_align*/1,
3855-
vi.isVolatile,
3856-
tbaa);
38573849
}
38583850
}
38593851
else {
@@ -5965,7 +5957,7 @@ static std::unique_ptr<Module> emit_function(
59655957
continue;
59665958
}
59675959

5968-
Value *isunboxed_union = NULL;
5960+
Value *isboxed_union = NULL;
59695961
Value *retval;
59705962
Value *sret = ctx.has_sret ? &*f->arg_begin() : NULL;
59715963
Type *retty = f->getReturnType();
@@ -5996,10 +5988,10 @@ static std::unique_ptr<Module> emit_function(
59965988
if (retvalinfo.ispointer() && !isa<AllocaInst>(retvalinfo.V)) {
59975989
// also need to account for the possibility the return object is boxed
59985990
// and avoid / skip copying it to the stack
5999-
isunboxed_union = builder.CreateICmpEQ(
5991+
isboxed_union = builder.CreateICmpNE(
60005992
builder.CreateAnd(tindex, ConstantInt::get(T_int8, 0x80)),
60015993
ConstantInt::get(T_int8, 0));
6002-
data = builder.CreateSelect(isunboxed_union, data, emit_bitcast(retvalinfo.V, T_pjlvalue));
5994+
data = builder.CreateSelect(isboxed_union, emit_bitcast(retvalinfo.V, T_pjlvalue), data);
60035995
}
60045996
}
60055997
}
@@ -6022,20 +6014,17 @@ static std::unique_ptr<Module> emit_function(
60226014
}
60236015
if (sret) {
60246016
if (retvalinfo.ispointer()) {
6025-
Value *copy_bytes;
60266017
if (returninfo.cc == jl_returninfo_t::SRet) {
60276018
assert(jl_is_leaf_type(jlrettype));
6028-
copy_bytes = ConstantInt::get(T_int32, jl_datatype_size(jlrettype));
6019+
Value *copy_bytes = ConstantInt::get(T_int32, jl_datatype_size(jlrettype));
6020+
builder.CreateMemCpy(sret,
6021+
data_pointer(retvalinfo, &ctx, T_pint8),
6022+
copy_bytes,
6023+
returninfo.union_minalign);
60296024
}
60306025
else {
6031-
copy_bytes = emit_sizeof(retvalinfo, &ctx);
6032-
if (isunboxed_union)
6033-
copy_bytes = builder.CreateSelect(isunboxed_union, copy_bytes, ConstantInt::get(copy_bytes->getType(), 0));
6026+
emit_unionmove(sret, retvalinfo, isboxed_union, false, NULL, &ctx);
60346027
}
6035-
builder.CreateMemCpy(sret,
6036-
data_pointer(retvalinfo, &ctx, T_pint8),
6037-
copy_bytes,
6038-
returninfo.union_minalign);
60396028
}
60406029
else {
60416030
Type *store_ty = julia_type_to_llvm(retvalinfo.typ);

0 commit comments

Comments
 (0)