Skip to content

Commit ca8e63e

Browse files
mikabl-armmikabl-arm
authored
JIT ARM64-SVE: Add Sve.LoadVector*FirstFaulting APIs (#104964)
1 parent da757a1 commit ca8e63e

File tree

11 files changed

+458
-33
lines changed

11 files changed

+458
-33
lines changed

‎src/coreclr/jit/gentree.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26729,29 +26729,35 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const
2672926729
case NI_Sve_LoadVector:
2673026730
case NI_Sve_LoadVectorNonTemporal:
2673126731
case NI_Sve_LoadVector128AndReplicateToVector:
26732+
case NI_Sve_LoadVectorByteZeroExtendFirstFaulting:
2673226733
case NI_Sve_LoadVectorByteZeroExtendToInt16:
2673326734
case NI_Sve_LoadVectorByteZeroExtendToInt32:
2673426735
case NI_Sve_LoadVectorByteZeroExtendToInt64:
2673526736
case NI_Sve_LoadVectorByteZeroExtendToUInt16:
2673626737
case NI_Sve_LoadVectorByteZeroExtendToUInt32:
2673726738
case NI_Sve_LoadVectorByteZeroExtendToUInt64:
2673826739
case NI_Sve_LoadVectorFirstFaulting:
26740+
case NI_Sve_LoadVectorInt16SignExtendFirstFaulting:
2673926741
case NI_Sve_LoadVectorInt16SignExtendToInt32:
2674026742
case NI_Sve_LoadVectorInt16SignExtendToInt64:
2674126743
case NI_Sve_LoadVectorInt16SignExtendToUInt32:
2674226744
case NI_Sve_LoadVectorInt16SignExtendToUInt64:
26745+
case NI_Sve_LoadVectorInt32SignExtendFirstFaulting:
2674326746
case NI_Sve_LoadVectorInt32SignExtendToInt64:
2674426747
case NI_Sve_LoadVectorInt32SignExtendToUInt64:
26748+
case NI_Sve_LoadVectorSByteSignExtendFirstFaulting:
2674526749
case NI_Sve_LoadVectorSByteSignExtendToInt16:
2674626750
case NI_Sve_LoadVectorSByteSignExtendToInt32:
2674726751
case NI_Sve_LoadVectorSByteSignExtendToInt64:
2674826752
case NI_Sve_LoadVectorSByteSignExtendToUInt16:
2674926753
case NI_Sve_LoadVectorSByteSignExtendToUInt32:
2675026754
case NI_Sve_LoadVectorSByteSignExtendToUInt64:
26755+
case NI_Sve_LoadVectorUInt16ZeroExtendFirstFaulting:
2675126756
case NI_Sve_LoadVectorUInt16ZeroExtendToInt32:
2675226757
case NI_Sve_LoadVectorUInt16ZeroExtendToInt64:
2675326758
case NI_Sve_LoadVectorUInt16ZeroExtendToUInt32:
2675426759
case NI_Sve_LoadVectorUInt16ZeroExtendToUInt64:
26760+
case NI_Sve_LoadVectorUInt32ZeroExtendFirstFaulting:
2675526761
case NI_Sve_LoadVectorUInt32ZeroExtendToInt64:
2675626762
case NI_Sve_LoadVectorUInt32ZeroExtendToUInt64:
2675726763
case NI_Sve_Load2xVectorAndUnzip:

‎src/coreclr/jit/hwintrinsiccodegenarm64.cpp

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2392,6 +2392,10 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
23922392
}
23932393

23942394
case NI_Sve_LoadVectorFirstFaulting:
2395+
case NI_Sve_LoadVectorInt16SignExtendFirstFaulting:
2396+
case NI_Sve_LoadVectorInt32SignExtendFirstFaulting:
2397+
case NI_Sve_LoadVectorUInt16ZeroExtendFirstFaulting:
2398+
case NI_Sve_LoadVectorUInt32ZeroExtendFirstFaulting:
23952399
{
23962400
if (intrin.numOperands == 3)
23972401
{
@@ -2405,6 +2409,20 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
24052409
break;
24062410
}
24072411

2412+
case NI_Sve_LoadVectorByteZeroExtendFirstFaulting:
2413+
case NI_Sve_LoadVectorSByteSignExtendFirstFaulting:
2414+
{
2415+
if (intrin.numOperands == 3)
2416+
{
2417+
// We have extra argument which means there is a "use" of FFR here. Restore it back in FFR register.
2418+
assert(op3Reg != REG_NA);
2419+
GetEmitter()->emitIns_R(INS_sve_wrffr, emitSize, op3Reg, opt);
2420+
}
2421+
2422+
GetEmitter()->emitIns_R_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, REG_ZR, opt);
2423+
break;
2424+
}
2425+
24082426
case NI_Sve_SetFfr:
24092427
{
24102428
assert(targetReg == REG_NA);

‎src/coreclr/jit/hwintrinsiclistarm64sve.h

Lines changed: 6 additions & 0 deletions
Large diffs are not rendered by default.

‎src/coreclr/jit/lowerarmarch.cpp

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1825,7 +1825,13 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node)
18251825
StoreFFRValue(node);
18261826
break;
18271827
}
1828+
case NI_Sve_LoadVectorByteZeroExtendFirstFaulting:
18281829
case NI_Sve_LoadVectorFirstFaulting:
1830+
case NI_Sve_LoadVectorInt16SignExtendFirstFaulting:
1831+
case NI_Sve_LoadVectorInt32SignExtendFirstFaulting:
1832+
case NI_Sve_LoadVectorSByteSignExtendFirstFaulting:
1833+
case NI_Sve_LoadVectorUInt16ZeroExtendFirstFaulting:
1834+
case NI_Sve_LoadVectorUInt32ZeroExtendFirstFaulting:
18291835
{
18301836
LIR::Use use;
18311837
bool foundUse = BlockRange().TryGetUse(node, &use);
@@ -4140,7 +4146,13 @@ void Lowering::StoreFFRValue(GenTreeHWIntrinsic* node)
41404146
switch (node->GetHWIntrinsicId())
41414147
{
41424148
case NI_Sve_GatherVectorFirstFaulting:
4149+
case NI_Sve_LoadVectorByteZeroExtendFirstFaulting:
41434150
case NI_Sve_LoadVectorFirstFaulting:
4151+
case NI_Sve_LoadVectorInt16SignExtendFirstFaulting:
4152+
case NI_Sve_LoadVectorInt32SignExtendFirstFaulting:
4153+
case NI_Sve_LoadVectorSByteSignExtendFirstFaulting:
4154+
case NI_Sve_LoadVectorUInt16ZeroExtendFirstFaulting:
4155+
case NI_Sve_LoadVectorUInt32ZeroExtendFirstFaulting:
41444156
case NI_Sve_SetFfr:
41454157

41464158
break;

‎src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4844,6 +4844,43 @@ internal Arm64() { }
48444844
public static unsafe Vector<ulong> LoadVectorByteNonFaultingZeroExtendToUInt64(byte* address) { throw new PlatformNotSupportedException(); }
48454845

48464846

4847+
/// <summary>
4848+
/// svint16_t svldff1ub_s16(svbool_t pg, const uint8_t *base)
4849+
/// LDFF1B Zresult.H, Pg/Z, [Xbase, XZR]
4850+
/// </summary>
4851+
public static unsafe Vector<short> LoadVectorByteZeroExtendFirstFaulting(Vector<short> mask, byte* address) { throw new PlatformNotSupportedException(); }
4852+
4853+
/// <summary>
4854+
/// svint32_t svldff1ub_s32(svbool_t pg, const uint8_t *base)
4855+
/// LDFF1B Zresult.S, Pg/Z, [Xbase, XZR]
4856+
/// </summary>
4857+
public static unsafe Vector<int> LoadVectorByteZeroExtendFirstFaulting(Vector<int> mask, byte* address) { throw new PlatformNotSupportedException(); }
4858+
4859+
/// <summary>
4860+
/// svint64_t svldff1ub_s64(svbool_t pg, const uint8_t *base)
4861+
/// LDFF1B Zresult.D, Pg/Z, [Xbase, XZR]
4862+
/// </summary>
4863+
public static unsafe Vector<long> LoadVectorByteZeroExtendFirstFaulting(Vector<long> mask, byte* address) { throw new PlatformNotSupportedException(); }
4864+
4865+
/// <summary>
4866+
/// svuint16_t svldff1ub_u16(svbool_t pg, const uint8_t *base)
4867+
/// LDFF1B Zresult.H, Pg/Z, [Xbase, XZR]
4868+
/// </summary>
4869+
public static unsafe Vector<ushort> LoadVectorByteZeroExtendFirstFaulting(Vector<ushort> mask, byte* address) { throw new PlatformNotSupportedException(); }
4870+
4871+
/// <summary>
4872+
/// svuint32_t svldff1ub_u32(svbool_t pg, const uint8_t *base)
4873+
/// LDFF1B Zresult.S, Pg/Z, [Xbase, XZR]
4874+
/// </summary>
4875+
public static unsafe Vector<uint> LoadVectorByteZeroExtendFirstFaulting(Vector<uint> mask, byte* address) { throw new PlatformNotSupportedException(); }
4876+
4877+
/// <summary>
4878+
/// svuint64_t svldff1ub_u64(svbool_t pg, const uint8_t *base)
4879+
/// LDFF1B Zresult.D, Pg/Z, [Xbase, XZR]
4880+
/// </summary>
4881+
public static unsafe Vector<ulong> LoadVectorByteZeroExtendFirstFaulting(Vector<ulong> mask, byte* address) { throw new PlatformNotSupportedException(); }
4882+
4883+
48474884
// Load 8-bit data and zero-extend
48484885

48494886
/// <summary>
@@ -4995,6 +5032,33 @@ internal Arm64() { }
49955032
public static unsafe Vector<ulong> LoadVectorInt16NonFaultingSignExtendToUInt64(short* address) { throw new PlatformNotSupportedException(); }
49965033

49975034

5035+
/// Load 16-bit data and sign-extend, first-faulting
5036+
5037+
/// <summary>
5038+
/// svint32_t svldff1sh_s32(svbool_t pg, const int16_t *base)
5039+
/// LDFF1SH Zresult.S, Pg/Z, [Xbase, XZR, LSL #1]
5040+
/// </summary>
5041+
public static unsafe Vector<int> LoadVectorInt16SignExtendFirstFaulting(Vector<int> mask, short* address) { throw new PlatformNotSupportedException(); }
5042+
5043+
/// <summary>
5044+
/// svint64_t svldff1sh_s64(svbool_t pg, const int16_t *base)
5045+
/// LDFF1SH Zresult.D, Pg/Z, [Xbase, XZR, LSL #1]
5046+
/// </summary>
5047+
public static unsafe Vector<long> LoadVectorInt16SignExtendFirstFaulting(Vector<long> mask, short* address) { throw new PlatformNotSupportedException(); }
5048+
5049+
/// <summary>
5050+
/// svuint32_t svldff1sh_u32(svbool_t pg, const int16_t *base)
5051+
/// LDFF1SH Zresult.S, Pg/Z, [Xbase, XZR, LSL #1]
5052+
/// </summary>
5053+
public static unsafe Vector<uint> LoadVectorInt16SignExtendFirstFaulting(Vector<uint> mask, short* address) { throw new PlatformNotSupportedException(); }
5054+
5055+
/// <summary>
5056+
/// svuint64_t svldff1sh_u64(svbool_t pg, const int16_t *base)
5057+
/// LDFF1SH Zresult.D, Pg/Z, [Xbase, XZR, LSL #1]
5058+
/// </summary>
5059+
public static unsafe Vector<ulong> LoadVectorInt16SignExtendFirstFaulting(Vector<ulong> mask, short* address) { throw new PlatformNotSupportedException(); }
5060+
5061+
49985062
// Load 16-bit data and sign-extend
49995063

50005064
/// <summary>
@@ -5049,6 +5113,21 @@ internal Arm64() { }
50495113
public static unsafe Vector<ulong> LoadVectorInt32NonFaultingSignExtendToUInt64(int* address) { throw new PlatformNotSupportedException(); }
50505114

50515115

5116+
/// Load 32-bit data and sign-extend, first-faulting
5117+
5118+
/// <summary>
5119+
/// svint64_t svldff1sw_s64(svbool_t pg, const int32_t *base)
5120+
/// LDFF1SW Zresult.D, Pg/Z, [Xbase, XZR, LSL #2]
5121+
/// </summary>
5122+
public static unsafe Vector<long> LoadVectorInt32SignExtendFirstFaulting(Vector<long> mask, int* address) { throw new PlatformNotSupportedException(); }
5123+
5124+
/// <summary>
5125+
/// svuint64_t svldff1sw_u64(svbool_t pg, const int32_t *base)
5126+
/// LDFF1SW Zresult.D, Pg/Z, [Xbase, XZR, LSL #2]
5127+
/// </summary>
5128+
public static unsafe Vector<ulong> LoadVectorInt32SignExtendFirstFaulting(Vector<ulong> mask, int* address) { throw new PlatformNotSupportedException(); }
5129+
5130+
50525131
// Load 32-bit data and sign-extend
50535132

50545133
/// <summary>
@@ -5247,6 +5326,45 @@ internal Arm64() { }
52475326
public static unsafe Vector<ulong> LoadVectorSByteNonFaultingSignExtendToUInt64(sbyte* address) { throw new PlatformNotSupportedException(); }
52485327

52495328

5329+
/// Load 8-bit data and sign-extend, first-faulting
5330+
5331+
/// <summary>
5332+
/// svint16_t svldff1sb_s16(svbool_t pg, const int8_t *base)
5333+
/// LDFF1SB Zresult.H, Pg/Z, [Xbase, XZR]
5334+
/// </summary>
5335+
public static unsafe Vector<short> LoadVectorSByteSignExtendFirstFaulting(Vector<short> mask, sbyte* address) { throw new PlatformNotSupportedException(); }
5336+
5337+
/// <summary>
5338+
/// svint32_t svldff1sb_s32(svbool_t pg, const int8_t *base)
5339+
/// LDFF1SB Zresult.S, Pg/Z, [Xbase, XZR]
5340+
/// </summary>
5341+
public static unsafe Vector<int> LoadVectorSByteSignExtendFirstFaulting(Vector<int> mask, sbyte* address) { throw new PlatformNotSupportedException(); }
5342+
5343+
/// <summary>
5344+
/// svint64_t svldff1sb_s64(svbool_t pg, const int8_t *base)
5345+
/// LDFF1SB Zresult.D, Pg/Z, [Xbase, XZR]
5346+
/// </summary>
5347+
public static unsafe Vector<long> LoadVectorSByteSignExtendFirstFaulting(Vector<long> mask, sbyte* address) { throw new PlatformNotSupportedException(); }
5348+
5349+
/// <summary>
5350+
/// svuint16_t svldff1sb_u16(svbool_t pg, const int8_t *base)
5351+
/// LDFF1SB Zresult.H, Pg/Z, [Xbase, XZR]
5352+
/// </summary>
5353+
public static unsafe Vector<ushort> LoadVectorSByteSignExtendFirstFaulting(Vector<ushort> mask, sbyte* address) { throw new PlatformNotSupportedException(); }
5354+
5355+
/// <summary>
5356+
/// svuint32_t svldff1sb_u32(svbool_t pg, const int8_t *base)
5357+
/// LDFF1SB Zresult.S, Pg/Z, [Xbase, XZR]
5358+
/// </summary>
5359+
public static unsafe Vector<uint> LoadVectorSByteSignExtendFirstFaulting(Vector<uint> mask, sbyte* address) { throw new PlatformNotSupportedException(); }
5360+
5361+
/// <summary>
5362+
/// svuint64_t svldff1sb_u64(svbool_t pg, const int8_t *base)
5363+
/// LDFF1SB Zresult.D, Pg/Z, [Xbase, XZR]
5364+
/// </summary>
5365+
public static unsafe Vector<ulong> LoadVectorSByteSignExtendFirstFaulting(Vector<ulong> mask, sbyte* address) { throw new PlatformNotSupportedException(); }
5366+
5367+
52505368
// Load 8-bit data and sign-extend
52515369

52525370
/// <summary>
@@ -5337,6 +5455,33 @@ internal Arm64() { }
53375455
public static unsafe Vector<ulong> LoadVectorUInt16NonFaultingZeroExtendToUInt64(ushort* address) { throw new PlatformNotSupportedException(); }
53385456

53395457

5458+
/// Load 16-bit data and zero-extend, first-faulting
5459+
5460+
/// <summary>
5461+
/// svint32_t svldff1uh_s32(svbool_t pg, const uint16_t *base)
5462+
/// LDFF1H Zresult.S, Pg/Z, [Xbase, XZR, LSL #1]
5463+
/// </summary>
5464+
public static unsafe Vector<int> LoadVectorUInt16ZeroExtendFirstFaulting(Vector<int> mask, ushort* address) { throw new PlatformNotSupportedException(); }
5465+
5466+
/// <summary>
5467+
/// svint64_t svldff1uh_s64(svbool_t pg, const uint16_t *base)
5468+
/// LDFF1H Zresult.D, Pg/Z, [Xbase, XZR, LSL #1]
5469+
/// </summary>
5470+
public static unsafe Vector<long> LoadVectorUInt16ZeroExtendFirstFaulting(Vector<long> mask, ushort* address) { throw new PlatformNotSupportedException(); }
5471+
5472+
/// <summary>
5473+
/// svuint32_t svldff1uh_u32(svbool_t pg, const uint16_t *base)
5474+
/// LDFF1H Zresult.S, Pg/Z, [Xbase, XZR, LSL #1]
5475+
/// </summary>
5476+
public static unsafe Vector<uint> LoadVectorUInt16ZeroExtendFirstFaulting(Vector<uint> mask, ushort* address) { throw new PlatformNotSupportedException(); }
5477+
5478+
/// <summary>
5479+
/// svuint64_t svldff1uh_u64(svbool_t pg, const uint16_t *base)
5480+
/// LDFF1H Zresult.D, Pg/Z, [Xbase, XZR, LSL #1]
5481+
/// </summary>
5482+
public static unsafe Vector<ulong> LoadVectorUInt16ZeroExtendFirstFaulting(Vector<ulong> mask, ushort* address) { throw new PlatformNotSupportedException(); }
5483+
5484+
53405485
// Load 16-bit data and zero-extend
53415486

53425487
/// <summary>
@@ -5391,6 +5536,21 @@ internal Arm64() { }
53915536
public static unsafe Vector<ulong> LoadVectorUInt32NonFaultingZeroExtendToUInt64(uint* address) { throw new PlatformNotSupportedException(); }
53925537

53935538

5539+
/// Load 32-bit data and zero-extend, first-faulting
5540+
5541+
/// <summary>
5542+
/// svint64_t svldff1uw_s64(svbool_t pg, const uint32_t *base)
5543+
/// LDFF1W Zresult.D, Pg/Z, [Xbase, XZR, LSL #2]
5544+
/// </summary>
5545+
public static unsafe Vector<long> LoadVectorUInt32ZeroExtendFirstFaulting(Vector<long> mask, uint* address) { throw new PlatformNotSupportedException(); }
5546+
5547+
/// <summary>
5548+
/// svuint64_t svldff1uw_u64(svbool_t pg, const uint32_t *base)
5549+
/// LDFF1W Zresult.D, Pg/Z, [Xbase, XZR, LSL #2]
5550+
/// </summary>
5551+
public static unsafe Vector<ulong> LoadVectorUInt32ZeroExtendFirstFaulting(Vector<ulong> mask, uint* address) { throw new PlatformNotSupportedException(); }
5552+
5553+
53945554
// Load 32-bit data and zero-extend
53955555

53965556
/// <summary>

0 commit comments

Comments
 (0)