Skip to content

JIT: ARM64 - Added SVE APIs ExtractLastVector, ExtractLastScalar, ExtractAfterLastVector, ExtractAfterLastScalar #103847

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 20 commits into from
Closed

JIT: ARM64 - Added SVE APIs ExtractLastVector, ExtractLastScalar, ExtractAfterLastVector, ExtractAfterLastScalar #103847

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
59c5187
ExtractLastVector, ExtractAfterLastVector, ExtractLastScalar, Extract…
TIHan Jun 22, 2024
32c90d0
Special handling for ExtractLastScalar and ExtractAfterLastScalar
TIHan Jun 22, 2024
725d73d
Few tweaks
TIHan Jun 22, 2024
2fe7714
Handling float
TIHan Jun 22, 2024
b23e622
Merge branch 'main' into sve-extract-apis
TIHan Jun 22, 2024
7210af0
Formatting
TIHan Jun 22, 2024
5e6b9d9
Feedback
TIHan Jun 24, 2024
2125977
Feedback
TIHan Jun 24, 2024
23282da
Removing flag
TIHan Jun 24, 2024
353aeb9
Merge branch 'main' into sve-extract-apis
TIHan Jun 24, 2024
3726a22
remove return
TIHan Jun 24, 2024
66ce083
Cleanup
TIHan Jun 24, 2024
a208c15
Merging
TIHan Jun 25, 2024
1374fcb
Feedback
TIHan Jun 26, 2024
fc4a2e2
Trying to fix conditional-select
TIHan Jun 27, 2024
ce5cd51
Merging with main
TIHan Jun 27, 2024
042ee87
Revert some changes
TIHan Jun 27, 2024
90c65bc
Reverting test changes
TIHan Jun 27, 2024
3b88ec7
minor cleanup
TIHan Jun 27, 2024
f122401
Formatting
TIHan Jun 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions src/coreclr/jit/hwintrinsic.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -935,6 +935,24 @@ struct HWIntrinsicInfo
return (flags & HW_Flag_ExplicitMaskedOperation) != 0;
}

// Checks if the intrinsic has an embedded mask operation and the intrinsic returns a scalar.
static bool IsEmbeddedMaskForScalarResult(NamedIntrinsic id)
{
if (IsEmbeddedMaskedOperation(id))
{
switch (id)
{
case NI_Sve_ExtractAfterLastScalar:
case NI_Sve_ExtractLastScalar:
return true;

default:
break;
}
}
return false;
}

static bool HasEnumOperand(NamedIntrinsic id)
{
const HWIntrinsicFlag flags = lookupFlags(id);
Expand Down
50 changes: 39 additions & 11 deletions src/coreclr/jit/hwintrinsiccodegenarm64.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -482,6 +482,8 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
regNumber embMaskOp3Reg = REG_NA;
regNumber falseReg = op3Reg;

insOpts optEmb = opt;

switch (intrinEmbMask.numOperands)
{
case 3:
Expand Down Expand Up @@ -509,16 +511,25 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
{
assert(!instrIsRMW);

// Special handling for ConvertTo* APIs
// Just need to change the opt here.
insScalableOpts soptEmb = INS_SCALABLE_OPTS_NONE;
switch (intrinEmbMask.id)
{
case NI_Sve_ExtractAfterLastVector:
case NI_Sve_ExtractLastVector:
{
soptEmb = INS_SCALABLE_OPTS_WITH_SIMD_SCALAR;
break;
}

// Special handling for ConvertTo* APIs
// Just need to change the opt here.
case NI_Sve_ConvertToInt32:
case NI_Sve_ConvertToUInt32:
{
opt = intrinEmbMask.baseType == TYP_DOUBLE ? INS_OPTS_D_TO_S : INS_OPTS_SCALABLE_S;
optEmb = intrinEmbMask.baseType == TYP_DOUBLE ? INS_OPTS_D_TO_S : INS_OPTS_SCALABLE_S;
break;
}

default:
break;
}
Expand Down Expand Up @@ -555,7 +566,8 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)

// We cannot use use `movprfx` here to move falseReg to targetReg because that will
// overwrite the value of embMaskOp1Reg which is present in targetReg.
GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, opt);
GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, optEmb,
soptEmb);

GetEmitter()->emitIns_R_R_R_R(INS_sve_sel, emitSize, targetReg, maskReg, targetReg,
falseReg, opt);
Expand All @@ -569,7 +581,8 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
}
}

GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, opt);
GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, optEmb,
soptEmb);
break;
}

Expand All @@ -587,7 +600,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)

// Finally, perform the actual "predicated" operation so that `targetReg` is the first operand
// and `embMaskOp2Reg` is the second operand.
GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp2Reg, opt);
GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp2Reg, optEmb);
}
else if (targetReg != falseReg)
{
Expand All @@ -602,7 +615,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
// If the embedded instruction supports optional mask operation, use the "unpredicated"
// version of the instruction, followed by "sel" to select the active lanes.
GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, embMaskOp1Reg,
embMaskOp2Reg, opt);
embMaskOp2Reg, optEmb);
}
else
{
Expand All @@ -617,7 +630,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
GetEmitter()->emitIns_R_R(INS_sve_movprfx, EA_SCALABLE, targetReg, embMaskOp1Reg);

GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp2Reg,
opt);
optEmb);
}

GetEmitter()->emitIns_R_R_R_R(INS_sve_sel, emitSize, targetReg, maskReg, targetReg,
Expand All @@ -634,13 +647,13 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)

// Finally, perform the actual "predicated" operation so that `targetReg` is the first operand
// and `embMaskOp2Reg` is the second operand.
GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp2Reg, opt);
GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp2Reg, optEmb);
}
else
{
// Just perform the actual "predicated" operation so that `targetReg` is the first operand
// and `embMaskOp2Reg` is the second operand.
GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp2Reg, opt);
GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp2Reg, optEmb);
}

break;
Expand Down Expand Up @@ -809,7 +822,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)

// Finally, perform the desired operation.
GetEmitter()->emitIns_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp2Reg,
embMaskOp3Reg, opt);
embMaskOp3Reg, optEmb);

break;
}
Expand Down Expand Up @@ -2142,6 +2155,21 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
break;
}

case NI_Sve_ExtractAfterLastScalar:
case NI_Sve_ExtractLastScalar:
assert(HWIntrinsicInfo::IsEmbeddedMaskForScalarResult(intrin.id));

if (varTypeIsFloating(node))
{
GetEmitter()->emitIns_R_R_R(ins, EA_SCALABLE, targetReg, /* mask */ op1Reg, op2Reg, opt,
INS_SCALABLE_OPTS_WITH_SIMD_SCALAR);
}
else
{
GetEmitter()->emitIns_R_R_R(ins, emitTypeSize(node), targetReg, /* mask */ op1Reg, op2Reg, opt);
}
break;

case NI_Sve_TestAnyTrue:
case NI_Sve_TestFirstTrue:
case NI_Sve_TestLastTrue:
Expand Down
4 changes: 4 additions & 0 deletions src/coreclr/jit/hwintrinsiclistarm64sve.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,10 @@ HARDWARE_INTRINSIC(Sve, Divide,
HARDWARE_INTRINSIC(Sve, DotProduct, -1, 3, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sdot, INS_sve_udot, INS_sve_sdot, INS_sve_udot, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve, DotProductBySelectedScalar, -1, 4, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sdot, INS_sve_udot, INS_sve_sdot, INS_sve_udot, INS_invalid, INS_invalid}, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve, DuplicateSelectedScalarToVector, -1, 2, true, {INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup, INS_sve_dup}, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(Sve, ExtractAfterLastScalar, -1, -1, false, {INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve, ExtractAfterLastVector, -1, -1, false, {INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta, INS_sve_lasta}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, ExtractLastScalar, -1, -1, false, {INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve, ExtractLastVector, -1, -1, false, {INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb, INS_sve_lastb}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve, ExtractVector, -1, 3, true, {INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext, INS_sve_ext}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve, FusedMultiplyAdd, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmla, INS_sve_fmla}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve, FusedMultiplyAddBySelectedScalar, -1, 4, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmla, INS_sve_fmla}, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_FmaIntrinsic|HW_Flag_LowVectorOperation)
Expand Down
38 changes: 29 additions & 9 deletions src/coreclr/jit/lowerarmarch.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1330,17 +1330,37 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node)
unsigned simdSize = node->GetSimdSize();
var_types simdType = Compiler::getSIMDTypeForSize(simdSize);
GenTree* trueMask = comp->gtNewSimdAllTrueMaskNode(simdBaseJitType, simdSize);
GenTree* trueVal = node;
GenTree* falseVal = comp->gtNewZeroConNode(simdType);

GenTreeHWIntrinsic* condSelNode =
comp->gtNewSimdHWIntrinsicNode(simdType, trueMask, trueVal, falseVal, NI_Sve_ConditionalSelect,
simdBaseJitType, simdSize);
// The instruction uses "predicate" to pick lanes, but at the same time returns a scalar result.
// As such, we cannot wrap it inside ConditionalSelect because that node operates on TYP_SIMD.
// Hence, we will just add an operand so that we have a predicate register for such instructions.
if (HWIntrinsicInfo::IsEmbeddedMaskForScalarResult(intrinsicId))
{
// Create the same node with an additional operand to pass the mask.
GenTreeHWIntrinsic* newNode =
comp->gtNewSimdHWIntrinsicNode(node->TypeGet(), trueMask, node->Op(1), intrinsicId,
simdBaseJitType, simdSize);

BlockRange().InsertAfter(node->Op(1), trueMask);
BlockRange().InsertAfter(trueMask, newNode);
BlockRange().Remove(node);
use.ReplaceWith(newNode);

BlockRange().InsertBefore(node, trueMask);
BlockRange().InsertBefore(node, falseVal);
BlockRange().InsertAfter(node, condSelNode);
use.ReplaceWith(condSelNode);
node = newNode;
}
else
{
GenTree* trueVal = node;
GenTree* falseVal = comp->gtNewZeroConNode(simdType);
GenTreeHWIntrinsic* condSelNode =
comp->gtNewSimdHWIntrinsicNode(simdType, trueMask, trueVal, falseVal, NI_Sve_ConditionalSelect,
simdBaseJitType, simdSize);

BlockRange().InsertBefore(node, trueMask);
BlockRange().InsertBefore(node, falseVal);
BlockRange().InsertAfter(node, condSelNode);
use.ReplaceWith(condSelNode);
}
}
}
}
Expand Down
Loading
Loading