Skip to content

Commit

Permalink
[SYCL][CUDA][libclc] Add atomic loads and stores with various memory …
Browse files Browse the repository at this point in the history 
…orders and scopes (#5191)

Adds atomic loads and stores with various memory orders and scopes. These are implemented by adding intrinsics and builtins for   PTX loads and stores.

Tests for this are here: intel/llvm-test-suite#648
  • Loading branch information
@t4c1
t4c1 authored Jan 17, 2022
1 parent bf5d9d5 commit e15ac50
Show file tree
Hide file tree
Showing 9 changed files with 1,838 additions and 2 deletions.
86 changes: 86 additions & 0 deletions clang/include/clang/Basic/BuiltinsNVPTX.def
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2037,6 +2037,92 @@ TARGET_BUILTIN(__nvvm_atom_acq_rel_cas_shared_ll, "LLiLLiD*LLiLLi", "n", SM_70)
TARGET_BUILTIN(__nvvm_atom_acq_rel_cta_cas_shared_ll, "LLiLLiD*LLiLLi", "n", SM_70)
TARGET_BUILTIN(__nvvm_atom_acq_rel_sys_cas_shared_ll, "LLiLLiD*LLiLLi", "n", SM_70)

#pragma push_macro("LD_VOLATILE_BUILTIN_TYPES")
#define LD_VOLATILE_BUILTIN_TYPES(ADDR_SPACE) \
BUILTIN(__nvvm_volatile_ld##ADDR_SPACE##_i, "iiD*", "n") \
BUILTIN(__nvvm_volatile_ld##ADDR_SPACE##_l, "LiLiD*", "n") \
BUILTIN(__nvvm_volatile_ld##ADDR_SPACE##_ll, "LLiLLiD*", "n") \
BUILTIN(__nvvm_volatile_ld##ADDR_SPACE##_f, "ffD*", "n") \
BUILTIN(__nvvm_volatile_ld##ADDR_SPACE##_d, "ddD*", "n")

#pragma push_macro("LD_BUILTIN_TYPES")
#define LD_BUILTIN_TYPES(ORDER, SCOPE, ADDR_SPACE) \
TARGET_BUILTIN(__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_i, "iiD*", "n", SM_70) \
TARGET_BUILTIN(__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_l, "LiLiD*", "n", SM_70) \
TARGET_BUILTIN(__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_ll, "LLiLLiD*", "n", SM_70) \
TARGET_BUILTIN(__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_f, "ffD*", "n", SM_70) \
TARGET_BUILTIN(__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_d, "ddD*", "n", SM_70)

#pragma push_macro("LD_BUILTIN_AS_TYPES")
#define LD_BUILTIN_AS_TYPES(ORDER, SCOPE) \
LD_BUILTIN_TYPES(ORDER, SCOPE, _gen) \
LD_BUILTIN_TYPES(ORDER, SCOPE, _global) \
LD_BUILTIN_TYPES(ORDER, SCOPE, _shared)

#pragma push_macro("LD_BUILTIN_SCOPES_AS_TYPES")
#define LD_BUILTIN_SCOPES_AS_TYPES(ORDER) \
LD_BUILTIN_AS_TYPES(ORDER, ) \
LD_BUILTIN_AS_TYPES(ORDER, _cta) \
LD_BUILTIN_AS_TYPES(ORDER, _sys)

LD_BUILTIN_SCOPES_AS_TYPES()
LD_BUILTIN_SCOPES_AS_TYPES(_acquire)
LD_VOLATILE_BUILTIN_TYPES(_gen)
LD_VOLATILE_BUILTIN_TYPES(_global)
LD_VOLATILE_BUILTIN_TYPES(_shared)

#undef LD_VOLATILE_BUILTIN_TYPES
#pragma pop_macro("LD_VOLATILE_BUILTIN_TYPES")
#undef LD_BUILTIN_TYPES
#pragma pop_macro("LD_BUILTIN_TYPES")
#undef LD_BUILTIN_AS_TYPES
#pragma pop_macro("LD_BUILTIN_AS_TYPES")
#undef LD_BUILTIN_SCOPES_AS_TYPES
#pragma pop_macro("LD_BUILTIN_SCOPES_AS_TYPES")

#pragma push_macro("ST_VOLATILE_BUILTIN_TYPES")
#define ST_VOLATILE_BUILTIN_TYPES(ADDR_SPACE) \
BUILTIN(__nvvm_volatile_st##ADDR_SPACE##_i, "viD*i", "n") \
BUILTIN(__nvvm_volatile_st##ADDR_SPACE##_l, "vLiD*Li", "n") \
BUILTIN(__nvvm_volatile_st##ADDR_SPACE##_ll, "vLLiD*LLi", "n") \
BUILTIN(__nvvm_volatile_st##ADDR_SPACE##_f, "vfD*f", "n") \
BUILTIN(__nvvm_volatile_st##ADDR_SPACE##_d, "vdD*d", "n")

#pragma push_macro("ST_BUILTIN_TYPES")
#define ST_BUILTIN_TYPES(ORDER, SCOPE, ADDR_SPACE) \
TARGET_BUILTIN(__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_i, "viD*i", "n", SM_70) \
TARGET_BUILTIN(__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_l, "vLiD*Li", "n", SM_70) \
TARGET_BUILTIN(__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_ll, "vLLiD*LLi", "n", SM_70) \
TARGET_BUILTIN(__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_f, "vfD*f", "n", SM_70) \
TARGET_BUILTIN(__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_d, "vdD*d", "n", SM_70)

#pragma push_macro("ST_BUILTIN_AS_TYPES")
#define ST_BUILTIN_AS_TYPES(ORDER, SCOPE) \
ST_BUILTIN_TYPES(ORDER, SCOPE, _gen) \
ST_BUILTIN_TYPES(ORDER, SCOPE, _global) \
ST_BUILTIN_TYPES(ORDER, SCOPE, _shared)

#pragma push_macro("ST_BUILTIN_SCOPES_AS_TYPES")
#define ST_BUILTIN_SCOPES_AS_TYPES(ORDER) \
ST_BUILTIN_AS_TYPES(ORDER, ) \
ST_BUILTIN_AS_TYPES(ORDER, _cta) \
ST_BUILTIN_AS_TYPES(ORDER, _sys)

ST_BUILTIN_SCOPES_AS_TYPES()
ST_BUILTIN_SCOPES_AS_TYPES(_release)
ST_VOLATILE_BUILTIN_TYPES(_gen)
ST_VOLATILE_BUILTIN_TYPES(_global)
ST_VOLATILE_BUILTIN_TYPES(_shared)

#undef ST_VOLATILE_BUILTIN_TYPES
#pragma pop_macro("ST_VOLATILE_BUILTIN_TYPES")
#undef ST_BUILTIN_TYPES
#pragma pop_macro("ST_BUILTIN_TYPES")
#undef ST_BUILTIN_AS_TYPES
#pragma pop_macro("ST_BUILTIN_AS_TYPES")
#undef ST_BUILTIN_SCOPES_AS_TYPES
#pragma pop_macro("ST_BUILTIN_SCOPES_AS_TYPES")

// Compiler Error Warn
BUILTIN(__nvvm_compiler_error, "vcC*4", "n")
BUILTIN(__nvvm_compiler_warn, "vcC*4", "n")
Expand Down
94 changes: 94 additions & 0 deletions clang/lib/CodeGen/CGBuiltin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17492,6 +17492,21 @@ CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E) {
Ptr->getType()}),
{Ptr, ConstantInt::get(Builder.getInt32Ty(), Align.getQuantity())});
};
auto MakeScopedLd = [&](unsigned IntrinsicID) {
Value *Ptr = EmitScalarExpr(E->getArg(0));
return Builder.CreateCall(
CGM.getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(),
Ptr->getType()}),
{Ptr});
};
auto MakeScopedSt = [&](unsigned IntrinsicID) {
Value *Ptr = EmitScalarExpr(E->getArg(0));
return Builder.CreateCall(
CGM.getIntrinsic(
IntrinsicID,
{Ptr->getType(), Ptr->getType()->getPointerElementType()}),
{Ptr, EmitScalarExpr(E->getArg(1))});
};
auto MakeScopedAtomic = [&](unsigned IntrinsicID) {
Value *Ptr = EmitScalarExpr(E->getArg(0));
return Builder.CreateCall(
Expand All @@ -17507,6 +17522,85 @@ CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E) {
{Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
};
switch (BuiltinID) {

#define LD_VOLATILE_CASES(ADDR_SPACE) \
case NVPTX::BI__nvvm_volatile_ld##ADDR_SPACE##_i: \
case NVPTX::BI__nvvm_volatile_ld##ADDR_SPACE##_l: \
case NVPTX::BI__nvvm_volatile_ld##ADDR_SPACE##_ll: \
return MakeScopedLd(Intrinsic::nvvm_ld##ADDR_SPACE##_i_volatile); \
case NVPTX::BI__nvvm_volatile_ld##ADDR_SPACE##_f: \
case NVPTX::BI__nvvm_volatile_ld##ADDR_SPACE##_d: \
return MakeScopedLd(Intrinsic::nvvm_ld##ADDR_SPACE##_f_volatile);

#define LD_CASES(ORDER, SCOPE, ADDR_SPACE) \
case NVPTX::BI__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_i: \
case NVPTX::BI__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_l: \
case NVPTX::BI__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_ll: \
return MakeScopedLd(Intrinsic::nvvm_ld##ADDR_SPACE##_i##ORDER##SCOPE); \
case NVPTX::BI__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_f: \
case NVPTX::BI__nvvm##ORDER##SCOPE##_ld##ADDR_SPACE##_d: \
return MakeScopedLd(Intrinsic::nvvm_ld##ADDR_SPACE##_f##ORDER##SCOPE);

#define LD_CASES_AS(ORDER, SCOPE) \
LD_CASES(ORDER, SCOPE, _gen) \
LD_CASES(ORDER, SCOPE, _global) \
LD_CASES(ORDER, SCOPE, _shared)

#define LD_CASES_AS_SCOPES(ORDER) \
LD_CASES_AS(ORDER, ) \
LD_CASES_AS(ORDER, _cta) \
LD_CASES_AS(ORDER, _sys)

LD_CASES_AS_SCOPES()
LD_CASES_AS_SCOPES(_acquire)
LD_VOLATILE_CASES(_gen)
LD_VOLATILE_CASES(_global)
LD_VOLATILE_CASES(_shared)

#undef LD_VOLATILE_CASES
#undef LD_CASES
#undef LD_CASES_AS
#undef LD_CASES_AS_SCOPES

#define ST_VOLATILE_CASES(ADDR_SPACE) \
case NVPTX::BI__nvvm_volatile_st##ADDR_SPACE##_i: \
case NVPTX::BI__nvvm_volatile_st##ADDR_SPACE##_l: \
case NVPTX::BI__nvvm_volatile_st##ADDR_SPACE##_ll: \
return MakeScopedSt(Intrinsic::nvvm_st##ADDR_SPACE##_i_volatile); \
case NVPTX::BI__nvvm_volatile_st##ADDR_SPACE##_f: \
case NVPTX::BI__nvvm_volatile_st##ADDR_SPACE##_d: \
return MakeScopedSt(Intrinsic::nvvm_st##ADDR_SPACE##_f_volatile);

#define ST_CASES(ORDER, SCOPE, ADDR_SPACE) \
case NVPTX::BI__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_i: \
case NVPTX::BI__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_l: \
case NVPTX::BI__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_ll: \
return MakeScopedSt(Intrinsic::nvvm_st##ADDR_SPACE##_i##ORDER##SCOPE); \
case NVPTX::BI__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_f: \
case NVPTX::BI__nvvm##ORDER##SCOPE##_st##ADDR_SPACE##_d: \
return MakeScopedSt(Intrinsic::nvvm_st##ADDR_SPACE##_f##ORDER##SCOPE);

#define ST_CASES_AS(ORDER, SCOPE) \
ST_CASES(ORDER, SCOPE, _gen) \
ST_CASES(ORDER, SCOPE, _global) \
ST_CASES(ORDER, SCOPE, _shared)

#define ST_CASES_AS_SCOPES(ORDER) \
ST_CASES_AS(ORDER, ) \
ST_CASES_AS(ORDER, _cta) \
ST_CASES_AS(ORDER, _sys)

ST_CASES_AS_SCOPES()
ST_CASES_AS_SCOPES(_release)
ST_VOLATILE_CASES(_gen)
ST_VOLATILE_CASES(_global)
ST_VOLATILE_CASES(_shared)

#undef ST_VOLATILE_CASES
#undef ST_CASES
#undef ST_CASES_AS
#undef ST_CASES_AS_SCOPES

case NVPTX::BI__nvvm_atom_add_gen_i:
case NVPTX::BI__nvvm_atom_add_gen_l:
case NVPTX::BI__nvvm_atom_add_gen_ll:
Expand Down
Loading

0 comments on commit e15ac50

Please sign in to comment.