From 844b3e248cce8059ae6de7883938ab4d8320e053 Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 25 Jun 2024 12:01:34 -0700 Subject: [PATCH 01/11] Initial work --- src/coreclr/jit/hwintrinsiclistarm64sve.h | 2 + .../Arm/Sve.PlatformNotSupported.cs | 73 +++++++++++++++++++ .../src/System/Runtime/Intrinsics/Arm/Sve.cs | 73 +++++++++++++++++++ .../ref/System.Runtime.Intrinsics.cs | 14 ++++ 4 files changed, 162 insertions(+) diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 600c1c7d9eaaaa..fc2171538624b9 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -48,6 +48,8 @@ HARDWARE_INTRINSIC(Sve, CreateFalseMaskSingle, HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt16, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt32, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt64, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateMaskForFirstActiveElement, -1, 2, true, {INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_MaskedOperation|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateMaskForNextActiveElement, -1, 2, true, {INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_MaskedOperation|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateTrueMaskByte, -1, 1, false, {INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasEnumOperand|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateTrueMaskDouble, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasEnumOperand|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateTrueMaskInt16, -1, 1, false, {INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasEnumOperand|HW_Flag_ReturnsPerElementMask) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs index 493afebaab801e..2d006d1c8007b6 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs @@ -1016,6 +1016,79 @@ internal Arm64() { } public static unsafe Vector CreateFalseMaskUInt64() { throw new PlatformNotSupportedException(); } + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svpnext_b8(svbool_t pg, svbool_t op) + /// PNEXT Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForNextActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svpnext_b16(svbool_t pg, svbool_t op) + /// PNEXT Ptied.H, Pg, Ptied.H + /// + public static unsafe Vector CreateMaskForNextActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svpnext_b32(svbool_t pg, svbool_t op) + /// PNEXT Ptied.S, Pg, Ptied.S + /// + public static unsafe Vector CreateMaskForNextActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svpnext_b64(svbool_t pg, svbool_t op) + /// PNEXT Ptied.D, Pg, Ptied.D + /// + public static unsafe Vector CreateMaskForNextActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + + /// CreateTrueMaskByte : Set predicate elements to true /// diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs index fba023cbb54c60..2894a8225412d5 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs @@ -1073,6 +1073,79 @@ internal Arm64() { } public static unsafe Vector CreateFalseMaskUInt64() => CreateFalseMaskUInt64(); + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForFirstActiveElement(totalMask, fromMask); + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForFirstActiveElement(totalMask, fromMask); + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForFirstActiveElement(totalMask, fromMask); + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForFirstActiveElement(totalMask, fromMask); + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForFirstActiveElement(totalMask, fromMask); + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForFirstActiveElement(totalMask, fromMask); + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForFirstActiveElement(totalMask, fromMask); + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForFirstActiveElement(totalMask, fromMask); + + /// + /// svbool_t svpnext_b8(svbool_t pg, svbool_t op) + /// PNEXT Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForNextActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForNextActiveElement(totalMask, fromMask); + + /// + /// svbool_t svpnext_b16(svbool_t pg, svbool_t op) + /// PNEXT Ptied.H, Pg, Ptied.H + /// + public static unsafe Vector CreateMaskForNextActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForNextActiveElement(totalMask, fromMask); + + /// + /// svbool_t svpnext_b32(svbool_t pg, svbool_t op) + /// PNEXT Ptied.S, Pg, Ptied.S + /// + public static unsafe Vector CreateMaskForNextActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForNextActiveElement(totalMask, fromMask); + + /// + /// svbool_t svpnext_b64(svbool_t pg, svbool_t op) + /// PNEXT Ptied.D, Pg, Ptied.D + /// + public static unsafe Vector CreateMaskForNextActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForNextActiveElement(totalMask, fromMask); + + /// CreateTrueMaskByte : Set predicate elements to true /// diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index 3e85d1af8d5839..d463b8991ebda2 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -4335,6 +4335,20 @@ internal Arm64() { } public static System.Numerics.Vector CreateFalseMaskUInt16() { throw null; } public static System.Numerics.Vector CreateFalseMaskUInt32() { throw null; } public static System.Numerics.Vector CreateFalseMaskUInt64() { throw null; } + + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForNextActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForNextActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForNextActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForNextActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } + public static System.Numerics.Vector CreateTrueMaskByte([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } public static System.Numerics.Vector CreateTrueMaskDouble([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } public static System.Numerics.Vector CreateTrueMaskInt16([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } From 5cb3058d9e6f3e96b23115f8b7307e5bfc17907e Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 25 Jun 2024 13:55:02 -0700 Subject: [PATCH 02/11] Added tests. Fixed parameter names. --- src/coreclr/jit/hwintrinsic.cpp | 2 + src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 26 ++ src/coreclr/jit/hwintrinsiclistarm64sve.h | 4 +- .../Arm/Sve.PlatformNotSupported.cs | 24 +- .../src/System/Runtime/Intrinsics/Arm/Sve.cs | 24 +- .../ref/System.Runtime.Intrinsics.cs | 24 +- .../GenerateHWIntrinsicTests_Arm.cs | 15 + .../HardwareIntrinsics/Arm/Shared/Helpers.cs | 340 ++++++++++++++++++ 8 files changed, 421 insertions(+), 38 deletions(-) diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 0b963d23e39a57..b9ef25f199c45a 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -1916,6 +1916,8 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, break; } + case NI_Sve_CreateMaskForFirstActiveElement: + case NI_Sve_CreateMaskForNextActiveElement: case NI_Sve_GetActiveElementCount: case NI_Sve_TestAnyTrue: case NI_Sve_TestFirstTrue: diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index ed59e82e5823f2..a4095b2e36d342 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -2172,6 +2172,32 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) break; } + case NI_Sve_CreateMaskForFirstActiveElement: + { + assert(isRMW); + if (targetReg != op2Reg) + { + assert(targetReg != op1Reg); + GetEmitter()->emitIns_Mov(INS_mov, emitTypeSize(node), targetReg, op2Reg, /* canSkip */ true); + } + + GetEmitter()->emitIns_R_R(ins, emitSize, targetReg, op1Reg, INS_OPTS_SCALABLE_B); + break; + } + + case NI_Sve_CreateMaskForNextActiveElement: + { + assert(isRMW); + if (targetReg != op2Reg) + { + assert(targetReg != op1Reg); + GetEmitter()->emitIns_Mov(INS_mov, emitTypeSize(node), targetReg, op2Reg, /* canSkip */ true); + } + + GetEmitter()->emitIns_R_R(ins, emitSize, targetReg, op1Reg, opt); + break; + } + default: unreached(); } diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 9ffd9031fb60a5..7d64548bcd5bdd 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -47,8 +47,8 @@ HARDWARE_INTRINSIC(Sve, CreateFalseMaskSingle, HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt16, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt32, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt64, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateMaskForFirstActiveElement, -1, 2, true, {INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_MaskedOperation|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateMaskForNextActiveElement, -1, 2, true, {INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_MaskedOperation|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateMaskForFirstActiveElement, -1, 2, true, {INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, CreateMaskForNextActiveElement, -1, 2, true, {INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, CreateTrueMaskByte, -1, 1, false, {INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasEnumOperand|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateTrueMaskDouble, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasEnumOperand|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateTrueMaskInt16, -1, 1, false, {INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasEnumOperand|HW_Flag_ReturnsPerElementMask) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs index efadb2145bef30..18cb0de5a22fcc 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs @@ -1020,73 +1020,73 @@ internal Arm64() { } /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) /// PFIRST Ptied.B, Pg, Ptied.B /// - public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } /// /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) /// PFIRST Ptied.B, Pg, Ptied.B /// - public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } /// /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) /// PFIRST Ptied.B, Pg, Ptied.B /// - public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } /// /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) /// PFIRST Ptied.B, Pg, Ptied.B /// - public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } /// /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) /// PFIRST Ptied.B, Pg, Ptied.B /// - public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } /// /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) /// PFIRST Ptied.B, Pg, Ptied.B /// - public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } /// /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) /// PFIRST Ptied.B, Pg, Ptied.B /// - public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } /// /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) /// PFIRST Ptied.B, Pg, Ptied.B /// - public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } /// /// svbool_t svpnext_b8(svbool_t pg, svbool_t op) /// PNEXT Ptied.B, Pg, Ptied.B /// - public static unsafe Vector CreateMaskForNextActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + public static unsafe Vector CreateMaskForNextActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } /// /// svbool_t svpnext_b16(svbool_t pg, svbool_t op) /// PNEXT Ptied.H, Pg, Ptied.H /// - public static unsafe Vector CreateMaskForNextActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + public static unsafe Vector CreateMaskForNextActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } /// /// svbool_t svpnext_b32(svbool_t pg, svbool_t op) /// PNEXT Ptied.S, Pg, Ptied.S /// - public static unsafe Vector CreateMaskForNextActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + public static unsafe Vector CreateMaskForNextActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } /// /// svbool_t svpnext_b64(svbool_t pg, svbool_t op) /// PNEXT Ptied.D, Pg, Ptied.D /// - public static unsafe Vector CreateMaskForNextActiveElement(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + public static unsafe Vector CreateMaskForNextActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } /// CreateTrueMaskByte : Set predicate elements to true diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs index a0b33522150fb1..87bc1b56ecbd2c 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs @@ -1077,73 +1077,73 @@ internal Arm64() { } /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) /// PFIRST Ptied.B, Pg, Ptied.B /// - public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForFirstActiveElement(totalMask, fromMask); + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) => CreateMaskForFirstActiveElement(mask, srcMask); /// /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) /// PFIRST Ptied.B, Pg, Ptied.B /// - public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForFirstActiveElement(totalMask, fromMask); + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) => CreateMaskForFirstActiveElement(mask, srcMask); /// /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) /// PFIRST Ptied.B, Pg, Ptied.B /// - public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForFirstActiveElement(totalMask, fromMask); + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) => CreateMaskForFirstActiveElement(mask, srcMask); /// /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) /// PFIRST Ptied.B, Pg, Ptied.B /// - public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForFirstActiveElement(totalMask, fromMask); + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) => CreateMaskForFirstActiveElement(mask, srcMask); /// /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) /// PFIRST Ptied.B, Pg, Ptied.B /// - public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForFirstActiveElement(totalMask, fromMask); + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) => CreateMaskForFirstActiveElement(mask, srcMask); /// /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) /// PFIRST Ptied.B, Pg, Ptied.B /// - public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForFirstActiveElement(totalMask, fromMask); + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) => CreateMaskForFirstActiveElement(mask, srcMask); /// /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) /// PFIRST Ptied.B, Pg, Ptied.B /// - public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForFirstActiveElement(totalMask, fromMask); + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) => CreateMaskForFirstActiveElement(mask, srcMask); /// /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) /// PFIRST Ptied.B, Pg, Ptied.B /// - public static unsafe Vector CreateMaskForFirstActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForFirstActiveElement(totalMask, fromMask); + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) => CreateMaskForFirstActiveElement(mask, srcMask); /// /// svbool_t svpnext_b8(svbool_t pg, svbool_t op) /// PNEXT Ptied.B, Pg, Ptied.B /// - public static unsafe Vector CreateMaskForNextActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForNextActiveElement(totalMask, fromMask); + public static unsafe Vector CreateMaskForNextActiveElement(Vector mask, Vector srcMask) => CreateMaskForNextActiveElement(mask, srcMask); /// /// svbool_t svpnext_b16(svbool_t pg, svbool_t op) /// PNEXT Ptied.H, Pg, Ptied.H /// - public static unsafe Vector CreateMaskForNextActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForNextActiveElement(totalMask, fromMask); + public static unsafe Vector CreateMaskForNextActiveElement(Vector mask, Vector srcMask) => CreateMaskForNextActiveElement(mask, srcMask); /// /// svbool_t svpnext_b32(svbool_t pg, svbool_t op) /// PNEXT Ptied.S, Pg, Ptied.S /// - public static unsafe Vector CreateMaskForNextActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForNextActiveElement(totalMask, fromMask); + public static unsafe Vector CreateMaskForNextActiveElement(Vector mask, Vector srcMask) => CreateMaskForNextActiveElement(mask, srcMask); /// /// svbool_t svpnext_b64(svbool_t pg, svbool_t op) /// PNEXT Ptied.D, Pg, Ptied.D /// - public static unsafe Vector CreateMaskForNextActiveElement(Vector totalMask, Vector fromMask) => CreateMaskForNextActiveElement(totalMask, fromMask); + public static unsafe Vector CreateMaskForNextActiveElement(Vector mask, Vector srcMask) => CreateMaskForNextActiveElement(mask, srcMask); /// CreateTrueMaskByte : Set predicate elements to true diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index 233855ba57277b..47c17d7a802e5b 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -4336,18 +4336,18 @@ internal Arm64() { } public static System.Numerics.Vector CreateFalseMaskUInt32() { throw null; } public static System.Numerics.Vector CreateFalseMaskUInt64() { throw null; } - public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } - public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } - public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } - public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } - public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } - public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } - public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } - public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } - public static unsafe System.Numerics.Vector CreateMaskForNextActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } - public static unsafe System.Numerics.Vector CreateMaskForNextActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } - public static unsafe System.Numerics.Vector CreateMaskForNextActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } - public static unsafe System.Numerics.Vector CreateMaskForNextActiveElement(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForNextActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForNextActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForNextActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForNextActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } public static System.Numerics.Vector CreateTrueMaskByte([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } public static System.Numerics.Vector CreateTrueMaskDouble([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index 88a5d1b75f648c..f0a15ebc24b410 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -3179,6 +3179,21 @@ ("SveCreateTrueMaskTest.template", new Dictionary { ["TestName"] = "Sve_CreateTrueMaskUInt32", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateTrueMaskUInt32", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1Type"] = "SveMaskPattern"}), ("SveCreateTrueMaskTest.template", new Dictionary { ["TestName"] = "Sve_CreateTrueMaskUInt64", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateTrueMaskUInt64", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1Type"] = "SveMaskPattern"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForFirstActiveElement_sbyte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForFirstActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSByte()", ["NextValueOp2"] = "Helpers.getMaskSByte()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForFirstActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForFirstActiveElement(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForFirstActiveElement_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForFirstActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt16()", ["NextValueOp2"] = "Helpers.getMaskInt16()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForFirstActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForFirstActiveElement(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForFirstActiveElement_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForFirstActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "Helpers.getMaskInt32()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForFirstActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForFirstActiveElement(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForFirstActiveElement_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForFirstActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "Helpers.getMaskInt64()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForFirstActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForFirstActiveElement(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForFirstActiveElement_byte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForFirstActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskByte()", ["NextValueOp2"] = "Helpers.getMaskByte()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForFirstActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForFirstActiveElement(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForFirstActiveElement_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForFirstActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt16()", ["NextValueOp2"] = "Helpers.getMaskUInt16()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForFirstActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForFirstActiveElement(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForFirstActiveElement_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForFirstActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "Helpers.getMaskUInt32()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForFirstActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForFirstActiveElement(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForFirstActiveElement_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForFirstActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "Helpers.getMaskUInt64()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForFirstActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForFirstActiveElement(left, right)"}), + + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForNextActiveElement_byte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForNextActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskByte()", ["NextValueOp2"] = "Helpers.getMaskByte()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForNextActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForNextActiveElement(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForNextActiveElement_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForNextActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt16()", ["NextValueOp2"] = "Helpers.getMaskUInt16()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForNextActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForNextActiveElement(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForNextActiveElement_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForNextActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "Helpers.getMaskUInt32()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForNextActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForNextActiveElement(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForNextActiveElement_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForNextActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "Helpers.getMaskUInt64()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForNextActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForNextActiveElement(left, right)"}), + + ("SveExtractVectorTest.template", new Dictionary { ["TestName"] = "SveExtractVector_Byte_1", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ExtractVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ElementIndex"] = "1", ["ValidateIterResult"] = "Helpers.ExtractVector(firstOp, secondOp, ElementIndex, i) != result[i]"}), ("SveExtractVectorTest.template", new Dictionary { ["TestName"] = "SveExtractVector_Double_1", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ExtractVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ElementIndex"] = "1", ["ValidateIterResult"] = "BitConverter.DoubleToInt64Bits(Helpers.ExtractVector(firstOp, secondOp, ElementIndex, i)) != BitConverter.DoubleToInt64Bits(result[i])"}), ("SveExtractVectorTest.template", new Dictionary { ["TestName"] = "SveExtractVector_Int16_1", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ExtractVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ElementIndex"] = "1", ["ValidateIterResult"] = "Helpers.ExtractVector(firstOp, secondOp, ElementIndex, i) != result[i]"}), diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs index 5cdd61a162c82f..070a639e180c5f 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs @@ -28,6 +28,346 @@ public static Vector InitVector(Func f) return new Vector(arr); } + public static byte[] CreateMaskForFirstActiveElement(byte[] mask, byte[] srcMask) + { + var count = Vector.Count; + + var first = -1; + + for (var i = 0; i < count; i++) + { + if (mask[i] > 0 && first == -1) + { + first = i; + break; + } + } + + var result = new byte[count]; + Array.Copy(srcMask, 0, result, 0, count); + + if (first >= 0) + { + result[first] = 1; + } + return result; + } + + public static short[] CreateMaskForFirstActiveElement(short[] mask, short[] srcMask) + { + var count = Vector.Count; + + var first = -1; + + for (var i = 0; i < count; i++) + { + if (mask[i] > 0 && first == -1) + { + first = i; + break; + } + } + + var result = new short[count]; + Array.Copy(srcMask, 0, result, 0, count); + + if (first >= 0) + { + result[first] = 1; + } + return result; + } + + public static int[] CreateMaskForFirstActiveElement(int[] mask, int[] srcMask) + { + var count = Vector.Count; + + var first = -1; + + for (var i = 0; i < count; i++) + { + if (mask[i] > 0 && first == -1) + { + first = i; + break; + } + } + + var result = new int[count]; + Array.Copy(srcMask, 0, result, 0, count); + + if (first >= 0) + { + result[first] = 1; + } + return result; + } + + public static long[] CreateMaskForFirstActiveElement(long[] mask, long[] srcMask) + { + var count = Vector.Count; + + var first = -1; + + for (var i = 0; i < count; i++) + { + if (mask[i] > 0 && first == -1) + { + first = i; + break; + } + } + + var result = new long[count]; + Array.Copy(srcMask, 0, result, 0, count); + + if (first >= 0) + { + result[first] = 1; + } + return result; + } + + public static sbyte[] CreateMaskForFirstActiveElement(sbyte[] mask, sbyte[] srcMask) + { + var count = Vector.Count; + + var first = -1; + + for (var i = 0; i < count; i++) + { + if (mask[i] > 0 && first == -1) + { + first = i; + break; + } + } + + var result = new sbyte[count]; + Array.Copy(srcMask, 0, result, 0, count); + + if (first >= 0) + { + result[first] = 1; + } + return result; + } + + public static ushort[] CreateMaskForFirstActiveElement(ushort[] mask, ushort[] srcMask) + { + var count = Vector.Count; + + var first = -1; + + for (var i = 0; i < count; i++) + { + if (mask[i] > 0 && first == -1) + { + first = i; + break; + } + } + + var result = new ushort[count]; + Array.Copy(srcMask, 0, result, 0, count); + + if (first >= 0) + { + result[first] = 1; + } + return result; + } + + public static uint[] CreateMaskForFirstActiveElement(uint[] mask, uint[] srcMask) + { + var count = Vector.Count; + + var first = -1; + + for (var i = 0; i < count; i++) + { + if (mask[i] > 0 && first == -1) + { + first = i; + break; + } + } + + var result = new uint[count]; + Array.Copy(srcMask, 0, result, 0, count); + + if (first >= 0) + { + result[first] = 1; + } + return result; + } + + public static ulong[] CreateMaskForFirstActiveElement(ulong[] mask, ulong[] srcMask) + { + var count = Vector.Count; + + var result = new ulong[count]; + Array.Copy(srcMask, 0, result, 0, count); + + var first = -1; + + for (var i = 0; i < count; i++) + { + if (mask[i] > 0 && first == -1) + { + first = i; + break; + } + } + + if (first >= 0) + { + result[first] = 1; + } + return result; + } + + public static int LastActiveElement(byte[] v) + { + var count = Vector.Count; + + for (var i = count - 1; i >= 0; i--) + { + if (v[i] > 0) + { + return i; + } + } + + return -1; + } + + public static int LastActiveElement(ushort[] v) + { + var count = Vector.Count; + + for (var i = count - 1; i >= 0; i--) + { + if (v[i] > 0) + { + return i; + } + } + + return -1; + } + + public static int LastActiveElement(uint[] v) + { + var count = Vector.Count; + + for (var i = count - 1; i >= 0; i--) + { + if (v[i] > 0) + { + return i; + } + } + + return -1; + } + + public static int LastActiveElement(ulong[] v) + { + var count = Vector.Count; + + for (var i = count - 1; i >= 0; i--) + { + if (v[i] > 0) + { + return i; + } + } + + return -1; + } + + public static byte[] CreateMaskForNextActiveElement(byte[] mask, byte[] srcMask) + { + var count = Vector.Count; + + var next = LastActiveElement(srcMask) + 1; + + while (next < count && (mask[next] == 0)) + { + next++; + } + + var result = new byte[count]; + if (next < count) + { + result[next] = 1; + } + + return result; + } + + public static ushort[] CreateMaskForNextActiveElement(ushort[] mask, ushort[] srcMask) + { + var count = Vector.Count; + + var next = LastActiveElement(srcMask) + 1; + + while (next < count && (mask[next] == 0)) + { + next++; + } + + var result = new ushort[count]; + if (next < count) + { + result[next] = 1; + } + + return result; + } + + public static uint[] CreateMaskForNextActiveElement(uint[] mask, uint[] srcMask) + { + var count = Vector.Count; + + var next = LastActiveElement(srcMask) + 1; + + while (next < count && (mask[next] == 0)) + { + next++; + } + + var result = new uint[count]; + if (next < count) + { + result[next] = 1; + } + + return result; + } + + public static ulong[] CreateMaskForNextActiveElement(ulong[] mask, ulong[] srcMask) + { + var count = Vector.Count; + + var next = LastActiveElement(srcMask) + 1; + + while (next < count && (mask[next] == 0)) + { + next++; + } + + var result = new ulong[count]; + if (next < count) + { + result[next] = 1; + } + + return result; + } + public static sbyte CountLeadingSignBits(sbyte op1) { return (sbyte)(CountLeadingZeroBits((sbyte)((ulong)op1 ^ ((ulong)op1 >> 1))) - 1); From 7e0a371ee858cf43811c1d6eeb1fe85efba0eaba Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 25 Jun 2024 16:19:21 -0700 Subject: [PATCH 03/11] Use delay free for op1 if the target preference is op2. Use sve_mov instead of mov. --- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 8 ++++++-- src/coreclr/jit/lsraarm64.cpp | 9 ++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index a4095b2e36d342..76dbd7325bbc7c 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -2175,10 +2175,12 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) case NI_Sve_CreateMaskForFirstActiveElement: { assert(isRMW); + assert(HWIntrinsicInfo::IsExplicitMaskedOperation(intrin.id)); + if (targetReg != op2Reg) { assert(targetReg != op1Reg); - GetEmitter()->emitIns_Mov(INS_mov, emitTypeSize(node), targetReg, op2Reg, /* canSkip */ true); + GetEmitter()->emitIns_Mov(INS_sve_mov, emitTypeSize(node), targetReg, op2Reg, /* canSkip */ true); } GetEmitter()->emitIns_R_R(ins, emitSize, targetReg, op1Reg, INS_OPTS_SCALABLE_B); @@ -2188,10 +2190,12 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) case NI_Sve_CreateMaskForNextActiveElement: { assert(isRMW); + assert(HWIntrinsicInfo::IsExplicitMaskedOperation(intrin.id)); + if (targetReg != op2Reg) { assert(targetReg != op1Reg); - GetEmitter()->emitIns_Mov(INS_mov, emitTypeSize(node), targetReg, op2Reg, /* canSkip */ true); + GetEmitter()->emitIns_Mov(INS_sve_mov, emitTypeSize(node), targetReg, op2Reg, /* canSkip */ true); } GetEmitter()->emitIns_R_R(ins, emitSize, targetReg, op1Reg, opt); diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 8547fe2e39bae0..d1591107623a74 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1623,7 +1623,14 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou predMask = RBM_LOWMASK.GetPredicateRegSet(); } - srcCount += BuildOperandUses(intrin.op1, predMask); + if (tgtPrefOp2) + { + srcCount += BuildDelayFreeUses(intrin.op1, intrin.op2, predMask); + } + else + { + srcCount += BuildOperandUses(intrin.op1, predMask); + } } } else if (intrinsicTree->OperIsMemoryLoadOrStore()) From f913b6674a02cedb143557265e3762350983b010 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 26 Jun 2024 12:17:57 -0700 Subject: [PATCH 04/11] Feedback --- src/coreclr/jit/hwintrinsiclistarm64sve.h | 4 +- .../HardwareIntrinsics/Arm/Shared/Helpers.cs | 266 ++++++------------ 2 files changed, 83 insertions(+), 187 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 7d64548bcd5bdd..546242900e7b4c 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -47,8 +47,8 @@ HARDWARE_INTRINSIC(Sve, CreateFalseMaskSingle, HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt16, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt32, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt64, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateMaskForFirstActiveElement, -1, 2, true, {INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, CreateMaskForNextActiveElement, -1, 2, true, {INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, CreateMaskForFirstActiveElement, -1, 2, true, {INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, CreateMaskForNextActiveElement, -1, 2, true, {INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, CreateTrueMaskByte, -1, 1, false, {INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasEnumOperand|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateTrueMaskDouble, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasEnumOperand|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateTrueMaskInt16, -1, 1, false, {INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasEnumOperand|HW_Flag_ReturnsPerElementMask) diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs index 070a639e180c5f..77039e1e765ff4 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs @@ -30,341 +30,237 @@ public static Vector InitVector(Func f) public static byte[] CreateMaskForFirstActiveElement(byte[] mask, byte[] srcMask) { - var count = Vector.Count; - - var first = -1; - + var count = srcMask.Length; + var result = new byte[count]; + Array.Copy(srcMask, 0, result, 0, count); for (var i = 0; i < count; i++) { - if (mask[i] > 0 && first == -1) + if (mask[i] != 0) { - first = i; - break; + result[i] = 1; + return result; } } - - var result = new byte[count]; - Array.Copy(srcMask, 0, result, 0, count); - - if (first >= 0) - { - result[first] = 1; - } return result; } public static short[] CreateMaskForFirstActiveElement(short[] mask, short[] srcMask) { - var count = Vector.Count; - - var first = -1; - + var count = srcMask.Length; + var result = new short[count]; + Array.Copy(srcMask, 0, result, 0, count); for (var i = 0; i < count; i++) { - if (mask[i] > 0 && first == -1) + if (mask[i] != 0) { - first = i; - break; + result[i] = 1; + return result; } } - - var result = new short[count]; - Array.Copy(srcMask, 0, result, 0, count); - - if (first >= 0) - { - result[first] = 1; - } return result; } public static int[] CreateMaskForFirstActiveElement(int[] mask, int[] srcMask) { - var count = Vector.Count; - - var first = -1; - + var count = srcMask.Length; + var result = new int[count]; + Array.Copy(srcMask, 0, result, 0, count); for (var i = 0; i < count; i++) { - if (mask[i] > 0 && first == -1) + if (mask[i] != 0) { - first = i; - break; + result[i] = 1; + return result; } } - - var result = new int[count]; - Array.Copy(srcMask, 0, result, 0, count); - - if (first >= 0) - { - result[first] = 1; - } return result; } public static long[] CreateMaskForFirstActiveElement(long[] mask, long[] srcMask) { - var count = Vector.Count; - - var first = -1; - + var count = srcMask.Length; + var result = new long[count]; + Array.Copy(srcMask, 0, result, 0, count); for (var i = 0; i < count; i++) { - if (mask[i] > 0 && first == -1) + if (mask[i] != 0) { - first = i; - break; + result[i] = 1; + return result; } } - - var result = new long[count]; - Array.Copy(srcMask, 0, result, 0, count); - - if (first >= 0) - { - result[first] = 1; - } return result; } public static sbyte[] CreateMaskForFirstActiveElement(sbyte[] mask, sbyte[] srcMask) { - var count = Vector.Count; - - var first = -1; - + var count = srcMask.Length; + var result = new sbyte[count]; + Array.Copy(srcMask, 0, result, 0, count); for (var i = 0; i < count; i++) { - if (mask[i] > 0 && first == -1) + if (mask[i] != 0) { - first = i; - break; + result[i] = 1; + return result; } } - - var result = new sbyte[count]; - Array.Copy(srcMask, 0, result, 0, count); - - if (first >= 0) - { - result[first] = 1; - } return result; } public static ushort[] CreateMaskForFirstActiveElement(ushort[] mask, ushort[] srcMask) { - var count = Vector.Count; - - var first = -1; - + var count = srcMask.Length; + var result = new ushort[count]; + Array.Copy(srcMask, 0, result, 0, count); for (var i = 0; i < count; i++) { - if (mask[i] > 0 && first == -1) + if (mask[i] != 0) { - first = i; - break; + result[i] = 1; + return result; } } - - var result = new ushort[count]; - Array.Copy(srcMask, 0, result, 0, count); - - if (first >= 0) - { - result[first] = 1; - } return result; } public static uint[] CreateMaskForFirstActiveElement(uint[] mask, uint[] srcMask) { - var count = Vector.Count; - - var first = -1; - + var count = srcMask.Length; + var result = new uint[count]; + Array.Copy(srcMask, 0, result, 0, count); for (var i = 0; i < count; i++) { - if (mask[i] > 0 && first == -1) + if (mask[i] != 0) { - first = i; - break; + result[i] = 1; + return result; } } - - var result = new uint[count]; - Array.Copy(srcMask, 0, result, 0, count); - - if (first >= 0) - { - result[first] = 1; - } return result; } public static ulong[] CreateMaskForFirstActiveElement(ulong[] mask, ulong[] srcMask) { - var count = Vector.Count; - + var count = srcMask.Length; var result = new ulong[count]; Array.Copy(srcMask, 0, result, 0, count); - - var first = -1; - for (var i = 0; i < count; i++) { - if (mask[i] > 0 && first == -1) + if (mask[i] != 0) { - first = i; - break; + result[i] = 1; + return result; } } - - if (first >= 0) - { - result[first] = 1; - } return result; } public static int LastActiveElement(byte[] v) { - var count = Vector.Count; - - for (var i = count - 1; i >= 0; i--) + for (var i = v.Length - 1; i >= 0; i--) { - if (v[i] > 0) + if (v[i] != 0) { return i; } } - return -1; } public static int LastActiveElement(ushort[] v) { - var count = Vector.Count; - - for (var i = count - 1; i >= 0; i--) + for (var i = v.Length - 1; i >= 0; i--) { - if (v[i] > 0) + if (v[i] != 0) { return i; } } - return -1; } public static int LastActiveElement(uint[] v) { - var count = Vector.Count; - - for (var i = count - 1; i >= 0; i--) + for (var i = v.Length - 1; i >= 0; i--) { - if (v[i] > 0) + if (v[i] != 0) { return i; } } - return -1; } public static int LastActiveElement(ulong[] v) { - var count = Vector.Count; - - for (var i = count - 1; i >= 0; i--) + for (var i = v.Length - 1; i >= 0; i--) { if (v[i] > 0) { return i; } } - return -1; } public static byte[] CreateMaskForNextActiveElement(byte[] mask, byte[] srcMask) { - var count = Vector.Count; - - var next = LastActiveElement(srcMask) + 1; - - while (next < count && (mask[next] == 0)) - { - next++; - } - + var count = srcMask.Length; var result = new byte[count]; - if (next < count) + for (var i = LastActiveElement(srcMask) + 1; i < count; i++) { - result[next] = 1; + if (mask[i] != 0) + { + result[i] = 1; + return result; + } } - return result; } public static ushort[] CreateMaskForNextActiveElement(ushort[] mask, ushort[] srcMask) { - var count = Vector.Count; - - var next = LastActiveElement(srcMask) + 1; - - while (next < count && (mask[next] == 0)) - { - next++; - } - + var count = srcMask.Length; var result = new ushort[count]; - if (next < count) + for (var i = LastActiveElement(srcMask) + 1; i < count; i++) { - result[next] = 1; + if (mask[i] != 0) + { + result[i] = 1; + return result; + } } - return result; } public static uint[] CreateMaskForNextActiveElement(uint[] mask, uint[] srcMask) { - var count = Vector.Count; - - var next = LastActiveElement(srcMask) + 1; - - while (next < count && (mask[next] == 0)) - { - next++; - } - + var count = srcMask.Length; var result = new uint[count]; - if (next < count) + for (var i = LastActiveElement(srcMask) + 1; i < count; i++) { - result[next] = 1; + if (mask[i] != 0) + { + result[i] = 1; + return result; + } } - return result; } public static ulong[] CreateMaskForNextActiveElement(ulong[] mask, ulong[] srcMask) { - var count = Vector.Count; - - var next = LastActiveElement(srcMask) + 1; - - while (next < count && (mask[next] == 0)) - { - next++; - } - + var count = srcMask.Length; var result = new ulong[count]; - if (next < count) + for (var i = LastActiveElement(srcMask) + 1; i < count; i++) { - result[next] = 1; + if (mask[i] != 0) + { + result[i] = 1; + return result; + } } - return result; } From 91a0202ce694a100c9036cf7e6e90951aeb33a31 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 26 Jun 2024 12:42:14 -0700 Subject: [PATCH 05/11] Feedback --- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 41 +++++++++++++-------- src/coreclr/jit/hwintrinsiclistarm64sve.h | 2 +- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 76dbd7325bbc7c..cc278c68853d46 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -824,10 +824,34 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) assert(!node->IsEmbMaskOp()); if (HWIntrinsicInfo::IsExplicitMaskedOperation(intrin.id)) { - GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, opt); + if (isRMW) + { + if (targetReg != op2Reg) + { + assert(targetReg != op1Reg); + + instruction insMov = INS_mov; + + if (GetEmitter()->isPredicateRegister(targetReg) && + GetEmitter()->isPredicateRegister(op2Reg)) + { + insMov = INS_sve_mov; + } + + GetEmitter()->emitIns_Mov(insMov, emitTypeSize(node), targetReg, op2Reg, + /* canSkip */ true); + } + + GetEmitter()->emitIns_R_R(ins, emitSize, targetReg, op1Reg, opt); + } + else + { + GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, opt); + } } else { + assert(!isRMW); // This generates an unpredicated version // Implicitly predicated should be taken care above `intrin.op2->IsEmbMaskOp()` GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, opt); @@ -2187,21 +2211,6 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) break; } - case NI_Sve_CreateMaskForNextActiveElement: - { - assert(isRMW); - assert(HWIntrinsicInfo::IsExplicitMaskedOperation(intrin.id)); - - if (targetReg != op2Reg) - { - assert(targetReg != op1Reg); - GetEmitter()->emitIns_Mov(INS_sve_mov, emitTypeSize(node), targetReg, op2Reg, /* canSkip */ true); - } - - GetEmitter()->emitIns_R_R(ins, emitSize, targetReg, op1Reg, opt); - break; - } - default: unreached(); } diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 546242900e7b4c..b5f7a9c7c0f125 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -48,7 +48,7 @@ HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt16, HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt32, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt64, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateMaskForFirstActiveElement, -1, 2, true, {INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) -HARDWARE_INTRINSIC(Sve, CreateMaskForNextActiveElement, -1, 2, true, {INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, CreateMaskForNextActiveElement, -1, 2, true, {INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, CreateTrueMaskByte, -1, 1, false, {INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasEnumOperand|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateTrueMaskDouble, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasEnumOperand|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateTrueMaskInt16, -1, 1, false, {INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasEnumOperand|HW_Flag_ReturnsPerElementMask) From 22b5851acdf3f7d78b727c1decc5b701dc2242f1 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Wed, 26 Jun 2024 12:49:55 -0700 Subject: [PATCH 06/11] Update Helpers.cs --- src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs index 77039e1e765ff4..d9ca97ee2b1928 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs @@ -196,7 +196,7 @@ public static int LastActiveElement(ulong[] v) { for (var i = v.Length - 1; i >= 0; i--) { - if (v[i] > 0) + if (v[i] != 0) { return i; } From 9aa43333f8ccf9ed3393b9891b74ade739ccd70f Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 26 Jun 2024 12:57:42 -0700 Subject: [PATCH 07/11] Handle RMW for non-explicit masked operation --- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index cc278c68853d46..9597c4f8448b5c 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -851,10 +851,24 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) } else { - assert(!isRMW); // This generates an unpredicated version // Implicitly predicated should be taken care above `intrin.op2->IsEmbMaskOp()` - GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, opt); + if (isRMW) + { + if (targetReg != op1Reg) + { + assert(targetReg != op2Reg); + + GetEmitter()->emitIns_Mov(INS_mov, emitTypeSize(node), targetReg, op1Reg, + /* canSkip */ true); + } + + GetEmitter()->emitIns_R_R(ins, emitSize, targetReg, op2Reg, opt); + } + else + { + GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, opt); + } } } else if (isRMW) From 5985bbda5dd6c8c8a6ba6285901c5261fa067280 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 26 Jun 2024 13:35:07 -0700 Subject: [PATCH 08/11] Remove handling as its already handled it looks like --- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 9597c4f8448b5c..ad287992073809 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -853,22 +853,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { // This generates an unpredicated version // Implicitly predicated should be taken care above `intrin.op2->IsEmbMaskOp()` - if (isRMW) - { - if (targetReg != op1Reg) - { - assert(targetReg != op2Reg); - - GetEmitter()->emitIns_Mov(INS_mov, emitTypeSize(node), targetReg, op1Reg, - /* canSkip */ true); - } - - GetEmitter()->emitIns_R_R(ins, emitSize, targetReg, op2Reg, opt); - } - else - { - GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, opt); - } + GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, opt); } } else if (isRMW) From 615296dcf78173c44678099b924308ffd4aef9d5 Mon Sep 17 00:00:00 2001 From: TIHan Date: Wed, 26 Jun 2024 15:36:52 -0700 Subject: [PATCH 09/11] Feedback --- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index ad287992073809..ab30868f66296e 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -830,15 +830,10 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { assert(targetReg != op1Reg); - instruction insMov = INS_mov; + assert(GetEmitter()->isPredicateRegister(targetReg) && + GetEmitter()->isPredicateRegister(op2Reg)); - if (GetEmitter()->isPredicateRegister(targetReg) && - GetEmitter()->isPredicateRegister(op2Reg)) - { - insMov = INS_sve_mov; - } - - GetEmitter()->emitIns_Mov(insMov, emitTypeSize(node), targetReg, op2Reg, + GetEmitter()->emitIns_Mov(INS_sve_mov, emitTypeSize(node), targetReg, op2Reg, /* canSkip */ true); } From 42abb9ecadaa78b923044a0c283a35e2b321920e Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 27 Jun 2024 16:52:21 -0700 Subject: [PATCH 10/11] Feedback --- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 6 ++---- src/coreclr/jit/instr.cpp | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index ab30868f66296e..5604456e98cf0e 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -830,10 +830,8 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { assert(targetReg != op1Reg); - assert(GetEmitter()->isPredicateRegister(targetReg) && - GetEmitter()->isPredicateRegister(op2Reg)); - - GetEmitter()->emitIns_Mov(INS_sve_mov, emitTypeSize(node), targetReg, op2Reg, + GetEmitter()->emitIns_Mov(ins_Move_Extend(intrin.op2->TypeGet(), false), + emitTypeSize(node), targetReg, op2Reg, /* canSkip */ true); } diff --git a/src/coreclr/jit/instr.cpp b/src/coreclr/jit/instr.cpp index f8c05010f22c22..43dff7999cf4d1 100644 --- a/src/coreclr/jit/instr.cpp +++ b/src/coreclr/jit/instr.cpp @@ -1721,7 +1721,6 @@ instruction CodeGen::ins_Move_Extend(var_types srcType, bool srcInReg) #if defined(TARGET_XARCH) return INS_kmovq_msk; #elif defined(TARGET_ARM64) - unreached(); // TODO-SVE: This needs testing return INS_sve_mov; #endif } From ef88f082441821e66ee69328de18d4936fbb0b71 Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 28 Jun 2024 10:31:25 -0700 Subject: [PATCH 11/11] Feedback --- src/coreclr/jit/instr.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/coreclr/jit/instr.cpp b/src/coreclr/jit/instr.cpp index 43dff7999cf4d1..382350ebfc5fc9 100644 --- a/src/coreclr/jit/instr.cpp +++ b/src/coreclr/jit/instr.cpp @@ -2084,7 +2084,6 @@ instruction CodeGen::ins_Copy(regNumber srcReg, var_types dstType) #if defined(TARGET_XARCH) return INS_kmovq_gpr; #elif defined(TARGET_ARM64) - unreached(); // TODO-SVE: This needs testing return INS_sve_mov; #endif }