Skip to content

Commit 298fffb

Browse files
vchuravyvchuravy
authored
Merge pull request #21980 from JuliaLang/vc/vec_alignment
limit maximum vector alignment to heap alignment
2 parents 62e8227 + b9671b0 commit 298fffb

14 files changed

+95
-28
lines changed

‎src/abi_arm.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -270,7 +270,7 @@ Type *preferred_llvm_type(jl_datatype_t *dt, bool isret) const override
270270
// For a Composite Type, the alignment of the copy will have 4-byte
271271
// alignment if its natural alignment is <= 4 and 8-byte alignment if
272272
// its natural alignment is >= 8
273-
size_t align = dt->layout->alignment;
273+
size_t align = jl_datatype_align(dt);
274274
if (align < 4)
275275
align = 4;
276276
if (align > 8)

‎src/abi_ppc64le.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ Type *preferred_llvm_type(jl_datatype_t *dt, bool isret) const override
144144
// rewrite integer-sized (non-HFA) struct to an array
145145
// the bitsize of the integer gives the desired alignment
146146
if (size > 8) {
147-
if (dt->layout->alignment <= 8) {
147+
if (jl_datatype_align(dt) <= 8) {
148148
return ArrayType::get(T_int64, (size + 7) / 8);
149149
}
150150
else {

‎src/array.c

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -193,10 +193,10 @@ JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data,
193193
assert(store_unboxed(el_type) == !data->flags.ptrarray);
194194
if (!data->flags.ptrarray) {
195195
a->elsize = jl_datatype_size(el_type);
196-
unsigned align = ((jl_datatype_t*)el_type)->layout->alignment;
196+
unsigned align = jl_datatype_align(el_type);
197197
jl_value_t *ownerty = jl_typeof(owner);
198198
unsigned oldalign = (ownerty == (jl_value_t*)jl_string_type ? 1 :
199-
((jl_datatype_t*)jl_tparam0(ownerty))->layout->alignment);
199+
jl_datatype_align(jl_tparam0(ownerty)));
200200
if (oldalign < align)
201201
jl_exceptionf(jl_argumenterror_type,
202202
"reinterpret from alignment %u bytes to alignment %u bytes not allowed",
@@ -283,7 +283,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data,
283283
unsigned align;
284284
if (isunboxed) {
285285
elsz = jl_datatype_size(el_type);
286-
align = ((jl_datatype_t*)el_type)->layout->alignment;
286+
align = jl_datatype_align(el_type);
287287
}
288288
else {
289289
align = elsz = sizeof(void*);
@@ -346,7 +346,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data,
346346
unsigned align;
347347
if (isunboxed) {
348348
elsz = jl_datatype_size(el_type);
349-
align = ((jl_datatype_t*)el_type)->layout->alignment;
349+
align = jl_datatype_align(el_type);
350350
}
351351
else {
352352
align = elsz = sizeof(void*);

‎src/ccall.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -618,7 +618,7 @@ static Value *julia_to_address(Type *to, jl_value_t *jlto, jl_unionall_t *jlto_e
618618
builder.CreateMemCpy(slot,
619619
data_pointer(jvinfo, ctx, slot->getType()),
620620
(uint64_t)jl_datatype_size(ety),
621-
(uint64_t)((jl_datatype_t*)ety)->layout->alignment);
621+
(uint64_t)jl_datatype_align(ety));
622622
mark_gc_use(jvinfo);
623623
}
624624
if (slot->getType() != to)
@@ -657,7 +657,7 @@ static Value *julia_to_native(Type *to, bool toboxed, jl_value_t *jlto, jl_union
657657
builder.CreateMemCpy(slot,
658658
data_pointer(jvinfo, ctx, slot->getType()),
659659
(uint64_t)jl_datatype_size(jlto),
660-
(uint64_t)((jl_datatype_t*)jlto)->layout->alignment);
660+
(uint64_t)jl_datatype_align(jlto));
661661
mark_gc_use(jvinfo);
662662
}
663663
return slot;

‎src/cgutils.cpp

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -157,15 +157,15 @@ static DIType julia_type_to_di(jl_value_t *jt, DIBuilder *dbuilder, bool isboxed
157157
llvm::DIType *t = dbuilder->createBasicType(
158158
jl_symbol_name(jdt->name->name),
159159
SizeInBits,
160-
8 * jdt->layout->alignment,
160+
8 * jl_datatype_align(jdt),
161161
llvm::dwarf::DW_ATE_unsigned);
162162
jdt->ditype = t;
163163
return t;
164164
#else
165165
DIType t = dbuilder->createBasicType(
166166
jl_symbol_name(jdt->name->name),
167167
SizeInBits,
168-
8 * jdt->layout->alignment,
168+
8 * jl_datatype_align(jdt),
169169
llvm::dwarf::DW_ATE_unsigned);
170170
MDNode *M = t;
171171
jdt->ditype = M;
@@ -189,7 +189,7 @@ static DIType julia_type_to_di(jl_value_t *jt, DIBuilder *dbuilder, bool isboxed
189189
NULL, // File
190190
0, // LineNumber
191191
jl_datatype_nbits(jdt), // SizeInBits
192-
8 * jdt->layout->alignment, // AlignInBits
192+
8 * jl_datatype_align(jdt), // AlignInBits
193193
DIFlagZero, // Flags
194194
NULL, // DerivedFrom
195195
DINodeArray(), // Elements
@@ -523,8 +523,12 @@ static Type *julia_struct_to_llvm(jl_value_t *jt, jl_unionall_t *ua, bool *isbox
523523
*jl_ExecutionEngine->getDataLayout();
524524
#endif
525525
unsigned llvm_alignment = DL.getABITypeAlignment((Type*)jst->struct_decl);
526-
unsigned julia_alignment = jst->layout->alignment;
527-
assert(llvm_alignment == julia_alignment);
526+
unsigned julia_alignment = jl_datatype_align(jst);
527+
// Check that the alignment adheres to the heap alignment.
528+
assert(julia_alignment <= JL_HEAP_ALIGNMENT);
529+
// TODO: Fix alignment calculation in LLVM, as well as in the GC and the struct declaration
530+
if (llvm_alignment <= JL_HEAP_ALIGNMENT)
531+
assert(julia_alignment == llvm_alignment);
528532
}
529533
#endif
530534
}
@@ -1165,9 +1169,10 @@ static Value *emit_bounds_check(const jl_cgval_t &ainfo, jl_value_t *ty, Value *
11651169
// It is currently unused, but might be used in the future for a more precise answer.
11661170
static unsigned julia_alignment(Value* /*ptr*/, jl_value_t *jltype, unsigned alignment)
11671171
{
1168-
if (!alignment && ((jl_datatype_t*)jltype)->layout->alignment > MAX_ALIGN) {
1169-
// Type's natural alignment exceeds strictest alignment promised in heap, so return the heap alignment.
1170-
return MAX_ALIGN;
1172+
if (!alignment) {
1173+
alignment = jl_datatype_align(jltype);
1174+
assert(alignment <= JL_HEAP_ALIGNMENT);
1175+
assert(JL_HEAP_ALIGNMENT % alignment == 0);
11711176
}
11721177
return alignment;
11731178
}

‎src/codegen.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3688,7 +3688,7 @@ static void union_alloca_type(jl_uniontype_t *ut,
36883688
[&](unsigned idx, jl_datatype_t *jt) {
36893689
if (!jl_is_datatype_singleton(jt)) {
36903690
size_t nb1 = jl_datatype_size(jt);
3691-
size_t align1 = jt->layout->alignment;
3691+
size_t align1 = jl_datatype_align(jt);
36923692
if (nb1 > nbytes)
36933693
nbytes = nb1;
36943694
if (align1 > align)
@@ -3959,7 +3959,7 @@ static void emit_assignment(jl_value_t *l, jl_value_t *r, jl_codectx_t *ctx)
39593959
builder.CreateMemCpy(vi.value.V,
39603960
data_pointer(rval_info, ctx, T_pint8),
39613961
copy_bytes,
3962-
((jl_datatype_t*)rval_info.typ)->layout->alignment,
3962+
jl_datatype_align(rval_info.typ),
39633963
vi.isVolatile,
39643964
tbaa);
39653965
}

‎src/datatype.c

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,7 @@ void jl_compute_field_offsets(jl_datatype_t *st)
278278
// Should never happen
279279
if (__unlikely(fsz > max_size))
280280
goto throw_ovf;
281-
al = ((jl_datatype_t*)ty)->layout->alignment;
281+
al = jl_datatype_align(ty);
282282
desc[i].isptr = 0;
283283
if (((jl_datatype_t*)ty)->layout->haspadding)
284284
haspadding = 1;
@@ -290,6 +290,7 @@ void jl_compute_field_offsets(jl_datatype_t *st)
290290
al = fsz;
291291
desc[i].isptr = 1;
292292
}
293+
assert(al <= JL_HEAP_ALIGNMENT && (JL_HEAP_ALIGNMENT % al) == 0);
293294
if (al != 0) {
294295
size_t alsz = LLT_ALIGN(sz, al);
295296
if (sz & (al - 1))
@@ -310,7 +311,10 @@ void jl_compute_field_offsets(jl_datatype_t *st)
310311
// Some tuples become LLVM vectors with stronger alignment than what was calculated above.
311312
unsigned al = jl_special_vector_alignment(nfields, lastty);
312313
assert(al % alignm == 0);
313-
if (al)
314+
// JL_HEAP_ALIGNMENT is the biggest alignment we can guarantee on the heap.
315+
if (al > JL_HEAP_ALIGNMENT)
316+
alignm = JL_HEAP_ALIGNMENT;
317+
else if (al)
314318
alignm = al;
315319
}
316320
st->size = LLT_ALIGN(sz, alignm);
@@ -438,7 +442,7 @@ static jl_value_t *jl_new_bits_internal(jl_value_t *dt, void *data, size_t *len)
438442
size_t nb = jl_datatype_size(bt);
439443
if (nb == 0)
440444
return jl_new_struct_uninit(bt);
441-
*len = LLT_ALIGN(*len, bt->layout->alignment);
445+
*len = LLT_ALIGN(*len, jl_datatype_align(bt));
442446
data = (char*)data + (*len);
443447
*len += nb;
444448
if (bt == jl_uint8_type) return jl_box_uint8(*(uint8_t*)data);
@@ -759,7 +763,7 @@ JL_DLLEXPORT size_t jl_get_alignment(jl_datatype_t *ty)
759763
{
760764
if (ty->layout == NULL)
761765
jl_error("non-leaf type doesn't have an alignment");
762-
return ty->layout->alignment;
766+
return jl_datatype_align(ty);
763767
}
764768

765769
#ifdef __cplusplus

‎src/gc.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -706,6 +706,7 @@ JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t sz)
706706
{
707707
maybe_collect(ptls);
708708
size_t offs = offsetof(bigval_t, header);
709+
static_assert(sizeof(bigval_t) % JL_HEAP_ALIGNMENT == 0, "");
709710
size_t allocsz = LLT_ALIGN(sz + offs, JL_CACHE_BYTE_ALIGNMENT);
710711
if (allocsz < sz) // overflow in adding offs, size was "negative"
711712
jl_throw(jl_memory_exception);

‎src/gc.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ extern "C" {
3232

3333
#define GC_PAGE_LG2 14 // log2(size of a page)
3434
#define GC_PAGE_SZ (1 << GC_PAGE_LG2) // 16k
35-
#define GC_PAGE_OFFSET (JL_SMALL_BYTE_ALIGNMENT - (sizeof(jl_taggedvalue_t) % JL_SMALL_BYTE_ALIGNMENT))
35+
#define GC_PAGE_OFFSET (JL_HEAP_ALIGNMENT - (sizeof(jl_taggedvalue_t) % JL_HEAP_ALIGNMENT))
3636

3737
#define jl_malloc_tag ((void*)0xdeadaa01)
3838
#define jl_singleton_tag ((void*)0xdeadaa02)

‎src/intrinsics.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -331,7 +331,7 @@ static Value *emit_unbox(Type *to, const jl_cgval_t &x, jl_value_t *jt, Value *d
331331

332332
int alignment;
333333
if (x.isboxed) {
334-
// julia's gc gives 16-byte aligned addresses
334+
// julia's gc gives 16-byte aligned addresses
335335
alignment = 16;
336336
}
337337
else if (jt) {
@@ -616,7 +616,7 @@ static jl_cgval_t emit_pointerref(jl_cgval_t *argv, jl_codectx_t *ctx)
616616
Value *strct = emit_allocobj(ctx, size,
617617
literal_pointer_val((jl_value_t*)ety));
618618
im1 = builder.CreateMul(im1, ConstantInt::get(T_size,
619-
LLT_ALIGN(size, ((jl_datatype_t*)ety)->layout->alignment)));
619+
LLT_ALIGN(size, jl_datatype_align(ety))));
620620
Value *thePtr = emit_unbox(T_pint8, e, e.typ);
621621
thePtr = builder.CreateGEP(emit_bitcast(thePtr, T_pint8), im1);
622622
builder.CreateMemCpy(emit_bitcast(strct, T_pint8), thePtr, size, 1);
@@ -673,7 +673,7 @@ static jl_cgval_t emit_pointerset(jl_cgval_t *argv, jl_codectx_t *ctx)
673673
thePtr = emit_unbox(T_pint8, e, e.typ);
674674
uint64_t size = jl_datatype_size(ety);
675675
im1 = builder.CreateMul(im1, ConstantInt::get(T_size,
676-
LLT_ALIGN(size, ((jl_datatype_t*)ety)->layout->alignment)));
676+
LLT_ALIGN(size, jl_datatype_align(ety))));
677677
builder.CreateMemCpy(builder.CreateGEP(thePtr, im1),
678678
data_pointer(x, ctx, T_pint8), size, align_nb);
679679
}

‎src/julia.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -773,6 +773,7 @@ STATIC_INLINE void jl_array_uint8_set(void *a, size_t i, uint8_t x)
773773
#define jl_field_type(st,i) jl_svecref(((jl_datatype_t*)st)->types, (i))
774774
#define jl_field_count(st) jl_svec_len(((jl_datatype_t*)st)->types)
775775
#define jl_datatype_size(t) (((jl_datatype_t*)t)->size)
776+
#define jl_datatype_align(t) (((jl_datatype_t*)t)->layout->alignment)
776777
#define jl_datatype_nbits(t) ((((jl_datatype_t*)t)->size)*8)
777778
#define jl_datatype_nfields(t) (((jl_datatype_t*)(t))->layout->nfields)
778779

‎src/julia_internal.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,8 @@ STATIC_INLINE int JL_CONST_FUNC jl_gc_szclass(size_t sz)
223223
#endif
224224
#define JL_SMALL_BYTE_ALIGNMENT 16
225225
#define JL_CACHE_BYTE_ALIGNMENT 64
226+
// JL_HEAP_ALIGNMENT is the maximum alignment that the GC can provide
227+
#define JL_HEAP_ALIGNMENT JL_SMALL_BYTE_ALIGNMENT
226228
#define GC_MAX_SZCLASS (2032-sizeof(void*))
227229

228230
STATIC_INLINE jl_value_t *jl_gc_alloc_(jl_ptls_t ptls, size_t sz, void *ty)

‎src/runtime_intrinsics.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ JL_DLLEXPORT jl_value_t *jl_pointerref(jl_value_t *p, jl_value_t *i, jl_value_t
4646
else {
4747
if (!jl_is_datatype(ety))
4848
jl_error("pointerref: invalid pointer");
49-
size_t nb = LLT_ALIGN(jl_datatype_size(ety), ((jl_datatype_t*)ety)->layout->alignment);
49+
size_t nb = LLT_ALIGN(jl_datatype_size(ety), jl_datatype_align(ety));
5050
char *pp = (char*)jl_unbox_long(p) + (jl_unbox_long(i)-1)*nb;
5151
return jl_new_bits(ety, pp);
5252
}
@@ -67,7 +67,7 @@ JL_DLLEXPORT jl_value_t *jl_pointerset(jl_value_t *p, jl_value_t *x, jl_value_t
6767
else {
6868
if (!jl_is_datatype(ety))
6969
jl_error("pointerset: invalid pointer");
70-
size_t nb = LLT_ALIGN(jl_datatype_size(ety), ((jl_datatype_t*)ety)->layout->alignment);
70+
size_t nb = LLT_ALIGN(jl_datatype_size(ety), jl_datatype_align(ety));
7171
char *pp = (char*)jl_unbox_long(p) + (jl_unbox_long(i)-1)*nb;
7272
if (jl_typeof(x) != ety)
7373
jl_error("pointerset: type mismatch in assign");

‎test/vecelement.jl

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,13 @@ for i=1:20
2323
end
2424
end
2525

26+
# Try various large tuple lengths and element types #20961
27+
for i in (34, 36, 48, 64, 72, 80, 96)
28+
for t in [Bool, Int8, Int16, Int32, Int64, Float32, Float64]
29+
call_iota(i,t)
30+
end
31+
end
32+
2633
# Another crash report for #15244 motivated this test.
2734
struct Bunch{N,T}
2835
elts::NTuple{N,Base.VecElement{T}}
@@ -65,3 +72,50 @@ a[1] = Gr(5.0, Bunch((VecElement(6.0), VecElement(7.0))), 8.0)
6572
@test a[2] == Gr(1.0, Bunch((VecElement(2.0), VecElement(3.0))), 4.0)
6673

6774
@test isa(VecElement((1,2)), VecElement{Tuple{Int,Int}})
75+
76+
# The following test mimic SIMD.jl
77+
const _llvmtypes = Dict{DataType, String}(
78+
Float64 => "double",
79+
Float32 => "float",
80+
Int32 => "i32",
81+
Int64 => "i64"
82+
)
83+
84+
@generated function vecadd(x::Vec{N, T}, y::Vec{N, T}) where {N, T}
85+
llvmT = _llvmtypes[T]
86+
func = T <: AbstractFloat ? "fadd" : "add"
87+
exp = """
88+
%3 = $(func) <$(N) x $(llvmT)> %0, %1
89+
ret <$(N) x $(llvmT)> %3
90+
"""
91+
return quote
92+
Base.@_inline_meta
93+
Base.llvmcall($exp, Vec{$N, $T}, Tuple{Vec{$N, $T}, Vec{$N, $T}}, x, y)
94+
end
95+
end
96+
97+
function f20961(x::Vector{Vec{N, T}}, y::Vector{Vec{N, T}}) where{N, T}
98+
@inbounds begin
99+
a = x[1]
100+
b = y[1]
101+
return vecadd(a, b)
102+
end
103+
end
104+
105+
# Test various SIMD Vectors with known good sizes
106+
for T in (Float64, Float32, Int64, Int32)
107+
for N in 1:36
108+
# For some vectortypes Julia emits llvm arrays instead of vectors
109+
if N % 7 == 0 || N % 11 == 0 || N % 13 == 0 || N % 15 == 0 ||
110+
N % 19 == 0 || N % 23 == 0 || N % 25 == 0 || N % 27 == 0 ||
111+
N % 29 == 0 || N % 31 == 0
112+
continue
113+
end
114+
a = ntuple(i->VecElement(T(i)), N)
115+
result = ntuple(i-> VecElement(T(i+i)), N)
116+
b = vecadd(a, a)
117+
@test b == result
118+
b = f20961([a], [a])
119+
@test b == result
120+
end
121+
end

0 commit comments

Comments
 (0)