diff --git a/crates/core_arch/src/s390x/macros.rs b/crates/core_arch/src/s390x/macros.rs index 2c40fedae2..4f0f84ec91 100644 --- a/crates/core_arch/src/s390x/macros.rs +++ b/crates/core_arch/src/s390x/macros.rs @@ -250,6 +250,19 @@ macro_rules! l_t_t { u8 }; + (vector_bool_long_long ) => { + u64 + }; + (vector_bool_int ) => { + u32 + }; + (vector_bool_short ) => { + u16 + }; + (vector_bool_char ) => { + u8 + }; + (vector_float) => { f32 }; @@ -338,6 +351,9 @@ macro_rules! t_u { (vector_bool_int) => { vector_unsigned_int }; + (vector_bool_long_long) => { + vector_unsigned_long_long + }; (vector_unsigned_char) => { vector_unsigned_char }; @@ -380,6 +396,9 @@ macro_rules! t_b { (vector_bool_int) => { vector_bool_int }; + (vector_bool_long_long) => { + vector_bool_long_long + }; (vector_signed_char) => { vector_bool_char }; diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index 3ec27b579c..5103273ed9 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -97,6 +97,10 @@ unsafe extern "unadjusted" { #[link_name = "llvm.s390.vsrlb"] fn vsrlb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char; #[link_name = "llvm.s390.vslb"] fn vslb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char; + #[link_name = "llvm.s390.vsldb"] fn vsldb(a: i8x16, b: i8x16, c: u32) -> i8x16; + #[link_name = "llvm.s390.vsld"] fn vsld(a: i8x16, b: i8x16, c: u32) -> i8x16; + #[link_name = "llvm.s390.vsrd"] fn vsrd(a: i8x16, b: i8x16, c: u32) -> i8x16; + #[link_name = "llvm.fshl.v16i8"] fn fshlb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char; #[link_name = "llvm.fshl.v8i16"] fn fshlh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short; #[link_name = "llvm.fshl.v4i32"] fn fshlf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int; @@ -197,6 +201,54 @@ unsafe extern "unadjusted" { #[link_name = "llvm.s390.vmleh"] fn vmleh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int; #[link_name = "llvm.s390.vmlef"] fn vmlef(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long; + #[link_name = "llvm.s390.vmob"] fn vmob(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short; + #[link_name = "llvm.s390.vmoh"] fn vmoh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int; + #[link_name = "llvm.s390.vmof"] fn vmof(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long; + + #[link_name = "llvm.s390.vmlob"] fn vmlob(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short; + #[link_name = "llvm.s390.vmloh"] fn vmloh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int; + #[link_name = "llvm.s390.vmlof"] fn vmlof(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long; + + #[link_name = "llvm.s390.vmhb"] fn vmhb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char; + #[link_name = "llvm.s390.vmhh"] fn vmhh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short; + #[link_name = "llvm.s390.vmhf"] fn vmhf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int; + + #[link_name = "llvm.s390.vmlhb"] fn vmlhb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char; + #[link_name = "llvm.s390.vmlhh"] fn vmlhh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short; + #[link_name = "llvm.s390.vmlhf"] fn vmlhf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int; + + #[link_name = "llvm.s390.vmaeb"] fn vmaeb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short; + #[link_name = "llvm.s390.vmaeh"] fn vmaeh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int; + #[link_name = "llvm.s390.vmaef"] fn vmaef(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long; + + #[link_name = "llvm.s390.vmaleb"] fn vmaleb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short; + #[link_name = "llvm.s390.vmaleh"] fn vmaleh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int; + #[link_name = "llvm.s390.vmalef"] fn vmalef(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long; + + #[link_name = "llvm.s390.vmaob"] fn vmaob(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short; + #[link_name = "llvm.s390.vmaoh"] fn vmaoh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int; + #[link_name = "llvm.s390.vmaof"] fn vmaof(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long; + + #[link_name = "llvm.s390.vmalob"] fn vmalob(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short; + #[link_name = "llvm.s390.vmaloh"] fn vmaloh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int; + #[link_name = "llvm.s390.vmalof"] fn vmalof(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long; + + #[link_name = "llvm.s390.vmahb"] fn vmahb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char; + #[link_name = "llvm.s390.vmahh"] fn vmahh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short; + #[link_name = "llvm.s390.vmahf"] fn vmahf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int; + + #[link_name = "llvm.s390.vmalhb"] fn vmalhb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char; + #[link_name = "llvm.s390.vmalhh"] fn vmalhh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short; + #[link_name = "llvm.s390.vmalhf"] fn vmalhf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int; + + #[link_name = "llvm.s390.vmalb"] fn vmalb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char; + #[link_name = "llvm.s390.vmalh"] fn vmalh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short; + #[link_name = "llvm.s390.vmalf"] fn vmalf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int; + + #[link_name = "llvm.s390.vmallb"] fn vmallb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char; + #[link_name = "llvm.s390.vmallh"] fn vmallh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short; + #[link_name = "llvm.s390.vmallf"] fn vmallf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int; + #[link_name = "llvm.s390.vgfmb"] fn vgfmb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short; #[link_name = "llvm.s390.vgfmh"] fn vgfmh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int; #[link_name = "llvm.s390.vgfmf"] fn vgfmf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long; @@ -206,6 +258,79 @@ unsafe extern "unadjusted" { #[link_name = "llvm.s390.vgfmah"] fn vgfmah(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int; #[link_name = "llvm.s390.vgfmaf"] fn vgfmaf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long; #[link_name = "llvm.s390.vgfmag"] fn vgfmag(a: vector_unsigned_long_long, b: vector_unsigned_long_long, c: u128) -> u128; + + #[link_name = "llvm.s390.vbperm"] fn vbperm(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_long_long; + + #[link_name = "llvm.s390.vftcisb"] fn vftcisb(a: vector_float, b: u32) -> PackedTuple; + #[link_name = "llvm.s390.vftcidb"] fn vftcidb(a: vector_double, b: u32) -> PackedTuple; + + #[link_name = "llvm.s390.vtm"] fn vtm(a: i8x16, b: i8x16) -> i32; + + #[link_name = "llvm.s390.vstrsb"] fn vstrsb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> PackedTuple; + #[link_name = "llvm.s390.vstrsh"] fn vstrsh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_char) -> PackedTuple; + #[link_name = "llvm.s390.vstrsf"] fn vstrsf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_char) -> PackedTuple; + + #[link_name = "llvm.s390.vstrszb"] fn vstrszb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> PackedTuple; + #[link_name = "llvm.s390.vstrszh"] fn vstrszh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_char) -> PackedTuple; + #[link_name = "llvm.s390.vstrszf"] fn vstrszf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_char) -> PackedTuple; + + #[link_name = "llvm.s390.vistrb"] fn vistrb(a: vector_unsigned_char) -> vector_unsigned_char; + #[link_name = "llvm.s390.vistrh"] fn vistrh(a: vector_unsigned_short) -> vector_unsigned_short; + #[link_name = "llvm.s390.vistrf"] fn vistrf(a: vector_unsigned_int) -> vector_unsigned_int; + + #[link_name = "llvm.s390.vistrbs"] fn vistrbs(a: vector_unsigned_char) -> PackedTuple; + #[link_name = "llvm.s390.vistrhs"] fn vistrhs(a: vector_unsigned_short) -> PackedTuple; + #[link_name = "llvm.s390.vistrfs"] fn vistrfs(a: vector_unsigned_int) -> PackedTuple; + + #[link_name = "llvm.s390.vmslg"] fn vmslg(a: vector_unsigned_long_long, b: vector_unsigned_long_long, c: u128, d: u32) -> u128; + + #[link_name = "llvm.s390.vstrcb"] fn vstrcb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> vector_bool_char; + #[link_name = "llvm.s390.vstrch"] fn vstrch(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> vector_bool_short; + #[link_name = "llvm.s390.vstrcf"] fn vstrcf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> vector_bool_int; + + #[link_name = "llvm.s390.vstrcbs"] fn vstrcbs(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> PackedTuple; + #[link_name = "llvm.s390.vstrchs"] fn vstrchs(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> PackedTuple; + #[link_name = "llvm.s390.vstrcfs"] fn vstrcfs(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> PackedTuple; + + #[link_name = "llvm.s390.vstrczb"] fn vstrczb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> vector_bool_char; + #[link_name = "llvm.s390.vstrczh"] fn vstrczh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> vector_bool_short; + #[link_name = "llvm.s390.vstrczf"] fn vstrczf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> vector_bool_int; + + #[link_name = "llvm.s390.vstrczbs"] fn vstrczbs(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> PackedTuple; + #[link_name = "llvm.s390.vstrczhs"] fn vstrczhs(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> PackedTuple; + #[link_name = "llvm.s390.vstrczfs"] fn vstrczfs(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> PackedTuple; + + #[link_name = "llvm.s390.vfeeb"] fn vfeeb(a: i8x16, b: i8x16) -> i8x16; + #[link_name = "llvm.s390.vfeeh"] fn vfeeh(a: i16x8, b: i16x8) -> i16x8; + #[link_name = "llvm.s390.vfeef"] fn vfeef(a: i32x4, b: i32x4) -> i32x4; + + #[link_name = "llvm.s390.vfeezb"] fn vfeezb(a: i8x16, b: i8x16) -> i8x16; + #[link_name = "llvm.s390.vfeezh"] fn vfeezh(a: i16x8, b: i16x8) -> i16x8; + #[link_name = "llvm.s390.vfeezf"] fn vfeezf(a: i32x4, b: i32x4) -> i32x4; + + #[link_name = "llvm.s390.vfeebs"] fn vfeebs(a: i8x16, b: i8x16) -> PackedTuple; + #[link_name = "llvm.s390.vfeehs"] fn vfeehs(a: i16x8, b: i16x8) -> PackedTuple; + #[link_name = "llvm.s390.vfeefs"] fn vfeefs(a: i32x4, b: i32x4) -> PackedTuple; + + #[link_name = "llvm.s390.vfeezbs"] fn vfeezbs(a: i8x16, b: i8x16) -> PackedTuple; + #[link_name = "llvm.s390.vfeezhs"] fn vfeezhs(a: i16x8, b: i16x8) -> PackedTuple; + #[link_name = "llvm.s390.vfeezfs"] fn vfeezfs(a: i32x4, b: i32x4) -> PackedTuple; + + #[link_name = "llvm.s390.vfeneb"] fn vfeneb(a: i8x16, b: i8x16) -> i8x16; + #[link_name = "llvm.s390.vfeneh"] fn vfeneh(a: i16x8, b: i16x8) -> i16x8; + #[link_name = "llvm.s390.vfenef"] fn vfenef(a: i32x4, b: i32x4) -> i32x4; + + #[link_name = "llvm.s390.vfenezb"] fn vfenezb(a: i8x16, b: i8x16) -> i8x16; + #[link_name = "llvm.s390.vfenezh"] fn vfenezh(a: i16x8, b: i16x8) -> i16x8; + #[link_name = "llvm.s390.vfenezf"] fn vfenezf(a: i32x4, b: i32x4) -> i32x4; + + #[link_name = "llvm.s390.vfenebs"] fn vfenebs(a: i8x16, b: i8x16) -> PackedTuple; + #[link_name = "llvm.s390.vfenehs"] fn vfenehs(a: i16x8, b: i16x8) -> PackedTuple; + #[link_name = "llvm.s390.vfenefs"] fn vfenefs(a: i32x4, b: i32x4) -> PackedTuple; + + #[link_name = "llvm.s390.vfenezbs"] fn vfenezbs(a: i8x16, b: i8x16) -> PackedTuple; + #[link_name = "llvm.s390.vfenezhs"] fn vfenezhs(a: i16x8, b: i16x8) -> PackedTuple; + #[link_name = "llvm.s390.vfenezfs"] fn vfenezfs(a: i32x4, b: i32x4) -> PackedTuple; } impl_from! { i8x16, u8x16, i16x8, u16x8, i32x4, u32x4, i64x2, u64x2, f32x4, f64x2 } @@ -336,6 +461,13 @@ const fn validate_block_boundary(block_boundary: u16) -> u32 { block_boundary as u32 >> 7 } +enum FindImm { + Eq = 4, + Ne = 12, + EqIdx = 0, + NeIdx = 8, +} + #[macro_use] mod sealed { use super::*; @@ -1797,10 +1929,9 @@ mod sealed { type Result = $r; #[inline] #[target_feature(enable = "vector")] - unsafe fn $m(self, b: Self, c: *mut i32) -> Self::Result { + unsafe fn $m(self, b: Self) -> (Self::Result, i32) { let PackedTuple { x, y } = $fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b)); - c.write(y); - transmute(x) + (transmute(x), y) } } )* @@ -1827,10 +1958,9 @@ mod sealed { type Result = t_b!($ty); #[inline] #[target_feature(enable = "vector")] - unsafe fn $m(self, b: Self, c: *mut i32) -> Self::Result { + unsafe fn $m(self, b: Self) -> (Self::Result, i32) { let PackedTuple { x, y } = $fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b)); - c.write(y); - transmute(x) + (transmute(x), y) } } )* @@ -1893,13 +2023,6 @@ mod sealed { }; } - enum FindImm { - Eq = 4, - Ne = 12, - EqIdx = 0, - NeIdx = 8, - } - #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorFindAnyEq { type Result; @@ -2001,7 +2124,7 @@ mod sealed { #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorFindAnyEqCC { type Result; - unsafe fn vec_find_any_eq_cc(self, other: Other, c: *mut i32) -> Self::Result; + unsafe fn vec_find_any_eq_cc(self, other: Other) -> (Self::Result, i32); } impl_vfae! { [cc VectorFindAnyEqCC vec_find_any_eq_cc] Eq vfaebs vfaehs vfaefs } @@ -2009,7 +2132,7 @@ mod sealed { #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorFindAnyNeCC { type Result; - unsafe fn vec_find_any_ne_cc(self, other: Other, c: *mut i32) -> Self::Result; + unsafe fn vec_find_any_ne_cc(self, other: Other) -> (Self::Result, i32); } impl_vfae! { [cc VectorFindAnyNeCC vec_find_any_ne_cc] Ne vfaebs vfaehs vfaefs } @@ -2017,7 +2140,7 @@ mod sealed { #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorFindAnyEqIdxCC { type Result; - unsafe fn vec_find_any_eq_idx_cc(self, other: Other, c: *mut i32) -> Self::Result; + unsafe fn vec_find_any_eq_idx_cc(self, other: Other) -> (Self::Result, i32); } impl_vfae! { [idx_cc VectorFindAnyEqIdxCC vec_find_any_eq_idx_cc] EqIdx vfaebs vfaehs vfaefs } @@ -2025,7 +2148,7 @@ mod sealed { #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorFindAnyNeIdxCC { type Result; - unsafe fn vec_find_any_ne_idx_cc(self, other: Other, c: *mut i32) -> Self::Result; + unsafe fn vec_find_any_ne_idx_cc(self, other: Other) -> (Self::Result, i32); } impl_vfae! { [idx_cc VectorFindAnyNeIdxCC vec_find_any_ne_idx_cc] NeIdx vfaebs vfaehs vfaefs } @@ -2033,7 +2156,7 @@ mod sealed { #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorFindAnyEqOrZeroIdxCC { type Result; - unsafe fn vec_find_any_eq_or_0_idx_cc(self, other: Other, c: *mut i32) -> Self::Result; + unsafe fn vec_find_any_eq_or_0_idx_cc(self, other: Other) -> (Self::Result, i32); } impl_vfae! { [idx_cc VectorFindAnyEqOrZeroIdxCC vec_find_any_eq_or_0_idx_cc] EqIdx vfaezbs vfaezhs vfaezfs } @@ -2041,7 +2164,7 @@ mod sealed { #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorFindAnyNeOrZeroIdxCC { type Result; - unsafe fn vec_find_any_ne_or_0_idx_cc(self, other: Other, c: *mut i32) -> Self::Result; + unsafe fn vec_find_any_ne_or_0_idx_cc(self, other: Other) -> (Self::Result, i32); } impl_vfae! { [idx_cc VectorFindAnyNeOrZeroIdxCC vec_find_any_ne_or_0_idx_cc] NeIdx vfaezbs vfaezhs vfaezfs } @@ -2298,11 +2421,9 @@ mod sealed { unsafe fn $intr( a: $ty, b: $ty, - c: *mut i32, - ) -> $outty { + ) -> ($outty, i32) { let PackedTuple { x, y } = super::$intr(a, b); - c.write(y); - x + (x, y) } #[unstable(feature = "stdarch_s390x", issue = "135681")] @@ -2311,8 +2432,8 @@ mod sealed { #[inline] #[target_feature(enable = "vector")] - unsafe fn vec_packs_cc(self, b: Self, c: *mut i32) -> Self::Result { - $intr(self, b, c) + unsafe fn vec_packs_cc(self, b: Self) -> (Self::Result, i32) { + $intr(self, b) } } )* @@ -2322,7 +2443,7 @@ mod sealed { #[unstable(feature = "stdarch_powerpc", issue = "111145")] pub trait VectorPacksCC { type Result; - unsafe fn vec_packs_cc(self, b: Self, c: *mut i32) -> Self::Result; + unsafe fn vec_packs_cc(self, b: Self) -> (Self::Result, i32); } impl_vector_packs_cc! { @@ -2343,8 +2464,8 @@ mod sealed { #[inline] #[target_feature(enable = "vector")] - unsafe fn vec_packsu_cc(self, b: Self, c: *mut i32) -> Self::Result { - $intr(self, b, c) + unsafe fn vec_packsu_cc(self, b: Self) -> (Self::Result, i32) { + $intr(self, b) } } )* @@ -2354,7 +2475,7 @@ mod sealed { #[unstable(feature = "stdarch_powerpc", issue = "111145")] pub trait VectorPacksuCC { type Result; - unsafe fn vec_packsu_cc(self, b: Self, c: *mut i32) -> Self::Result; + unsafe fn vec_packsu_cc(self, b: Self) -> (Self::Result, i32); } impl_vector_packsu_cc! { @@ -2501,6 +2622,17 @@ mod sealed { } } }; + ([$Trait:ident $m:ident] $fun:ident ($a:ty, $b:ty, $c:ty) -> $r:ty) => { + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl $Trait for $a { + type Result = $r; + #[inline] + #[target_feature(enable = "vector")] + unsafe fn $m(self, b: $b, c: $c) -> $r { + $fun(self, b, c) + } + } + }; } #[unstable(feature = "stdarch_s390x", issue = "135681")] @@ -2543,6 +2675,142 @@ mod sealed { impl_mul!([VectorMule vec_mule] vec_vmleh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int); impl_mul!([VectorMule vec_mule] vec_vmlef (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long ); + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorMulo { + unsafe fn vec_mulo(self, b: Self) -> Result; + } + + test_impl! { vec_vmob(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short [ vmob, vmob ] } + test_impl! { vec_vmoh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int[ vmoh, vmoh ] } + test_impl! { vec_vmof(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long [ vmof, vmof ] } + + test_impl! { vec_vmlob(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short [ vmlob, vmlob ] } + test_impl! { vec_vmloh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int[ vmloh, vmloh ] } + test_impl! { vec_vmlof(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [ vmlof, vmlof ] } + + impl_mul!([VectorMulo vec_mulo] vec_vmob (vector_signed_char, vector_signed_char) -> vector_signed_short ); + impl_mul!([VectorMulo vec_mulo] vec_vmoh (vector_signed_short, vector_signed_short) -> vector_signed_int); + impl_mul!([VectorMulo vec_mulo] vec_vmof (vector_signed_int, vector_signed_int) -> vector_signed_long_long ); + + impl_mul!([VectorMulo vec_mulo] vec_vmlob (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short ); + impl_mul!([VectorMulo vec_mulo] vec_vmloh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int); + impl_mul!([VectorMulo vec_mulo] vec_vmlof (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long ); + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorMulh { + unsafe fn vec_mulh(self, b: Self) -> Result; + } + + test_impl! { vec_vmhb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [ vmhb, vmhb ] } + test_impl! { vec_vmhh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [ vmhh, vmhh ] } + test_impl! { vec_vmhf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [ vmhf, vmhf ] } + + test_impl! { vec_vmlhb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [ vmlhb, vmlhb ] } + test_impl! { vec_vmlhh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [ vmlhh, vmlhh ] } + test_impl! { vec_vmlhf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [ vmlhf, vmlhf ] } + + impl_mul!([VectorMulh vec_mulh] vec_vmhb (vector_signed_char, vector_signed_char) -> vector_signed_char); + impl_mul!([VectorMulh vec_mulh] vec_vmhh (vector_signed_short, vector_signed_short) -> vector_signed_short); + impl_mul!([VectorMulh vec_mulh] vec_vmhf (vector_signed_int, vector_signed_int) -> vector_signed_int); + + impl_mul!([VectorMulh vec_mulh] vec_vmlhb (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char); + impl_mul!([VectorMulh vec_mulh] vec_vmlhh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short); + impl_mul!([VectorMulh vec_mulh] vec_vmlhf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int); + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorMeadd { + type Result; + unsafe fn vec_meadd(self, b: Self, c: Self::Result) -> Self::Result; + } + + test_impl! { vec_vmaeb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short [ vmaeb, vmaeb ] } + test_impl! { vec_vmaeh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int[ vmaeh, vmaeh ] } + test_impl! { vec_vmaef(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long [ vmaef, vmaef ] } + + test_impl! { vec_vmaleb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short [ vmaleb, vmaleb ] } + test_impl! { vec_vmaleh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int[ vmaleh, vmaleh ] } + test_impl! { vec_vmalef(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long [ vmalef, vmalef ] } + + impl_mul!([VectorMeadd vec_meadd] vec_vmaeb (vector_signed_char, vector_signed_char, vector_signed_short) -> vector_signed_short ); + impl_mul!([VectorMeadd vec_meadd] vec_vmaeh (vector_signed_short, vector_signed_short, vector_signed_int) -> vector_signed_int); + impl_mul!([VectorMeadd vec_meadd] vec_vmaef (vector_signed_int, vector_signed_int, vector_signed_long_long) -> vector_signed_long_long ); + + impl_mul!([VectorMeadd vec_meadd] vec_vmaleb (vector_unsigned_char, vector_unsigned_char, vector_unsigned_short) -> vector_unsigned_short ); + impl_mul!([VectorMeadd vec_meadd] vec_vmaleh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_int) -> vector_unsigned_int); + impl_mul!([VectorMeadd vec_meadd] vec_vmalef (vector_unsigned_int, vector_unsigned_int, vector_unsigned_long_long) -> vector_unsigned_long_long ); + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorMoadd { + type Result; + unsafe fn vec_moadd(self, b: Self, c: Self::Result) -> Self::Result; + } + + test_impl! { vec_vmaob(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short [ vmaob, vmaob ] } + test_impl! { vec_vmaoh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int[ vmaoh, vmaoh ] } + test_impl! { vec_vmaof(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long [ vmaof, vmaof ] } + + test_impl! { vec_vmalob(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short [ vmalob, vmalob ] } + test_impl! { vec_vmaloh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int[ vmaloh, vmaloh ] } + test_impl! { vec_vmalof(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long [ vmalof, vmalof ] } + + impl_mul!([VectorMoadd vec_moadd] vec_vmaob (vector_signed_char, vector_signed_char, vector_signed_short) -> vector_signed_short ); + impl_mul!([VectorMoadd vec_moadd] vec_vmaoh (vector_signed_short, vector_signed_short, vector_signed_int) -> vector_signed_int); + impl_mul!([VectorMoadd vec_moadd] vec_vmaof (vector_signed_int, vector_signed_int, vector_signed_long_long) -> vector_signed_long_long ); + + impl_mul!([VectorMoadd vec_moadd] vec_vmalob (vector_unsigned_char, vector_unsigned_char, vector_unsigned_short) -> vector_unsigned_short ); + impl_mul!([VectorMoadd vec_moadd] vec_vmaloh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_int) -> vector_unsigned_int); + impl_mul!([VectorMoadd vec_moadd] vec_vmalof (vector_unsigned_int, vector_unsigned_int, vector_unsigned_long_long) -> vector_unsigned_long_long ); + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorMhadd { + type Result; + unsafe fn vec_mhadd(self, b: Self, c: Self::Result) -> Self::Result; + } + + test_impl! { vec_vmahb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char [ vmahb, vmahb ] } + test_impl! { vec_vmahh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short[ vmahh, vmahh ] } + test_impl! { vec_vmahf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int [ vmahf, vmahf ] } + + test_impl! { vec_vmalhb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char [ vmalhb, vmalhb ] } + test_impl! { vec_vmalhh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short[ vmalhh, vmalhh ] } + test_impl! { vec_vmalhf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int [ vmalhf, vmalhf ] } + + impl_mul!([VectorMhadd vec_mhadd] vec_vmahb (vector_signed_char, vector_signed_char, vector_signed_char) -> vector_signed_char ); + impl_mul!([VectorMhadd vec_mhadd] vec_vmahh (vector_signed_short, vector_signed_short, vector_signed_short) -> vector_signed_short); + impl_mul!([VectorMhadd vec_mhadd] vec_vmahf (vector_signed_int, vector_signed_int, vector_signed_int) -> vector_signed_int ); + + impl_mul!([VectorMhadd vec_mhadd] vec_vmalhb (vector_unsigned_char, vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char ); + impl_mul!([VectorMhadd vec_mhadd] vec_vmalhh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short); + impl_mul!([VectorMhadd vec_mhadd] vec_vmalhf (vector_unsigned_int, vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int ); + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorMladd { + type Result; + unsafe fn vec_mladd(self, b: Self, c: Self::Result) -> Self::Result; + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn simd_mladd(a: T, b: T, c: T) -> T { + simd_add(simd_mul(a, b), c) + } + + test_impl! { vec_vmal_ib(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char [simd_mladd, vmalb ] } + test_impl! { vec_vmal_ih(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short[simd_mladd, vmalh ] } + test_impl! { vec_vmal_if(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int [simd_mladd, vmalf ] } + + test_impl! { vec_vmal_ub(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char [simd_mladd, vmalb ] } + test_impl! { vec_vmal_uh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short[simd_mladd, vmalh ] } + test_impl! { vec_vmal_uf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int [simd_mladd, vmalf ] } + + impl_mul!([VectorMladd vec_mladd] vec_vmal_ib (vector_signed_char, vector_signed_char, vector_signed_char) -> vector_signed_char ); + impl_mul!([VectorMladd vec_mladd] vec_vmal_ih (vector_signed_short, vector_signed_short, vector_signed_short) -> vector_signed_short); + impl_mul!([VectorMladd vec_mladd] vec_vmal_if (vector_signed_int, vector_signed_int, vector_signed_int) -> vector_signed_int ); + + impl_mul!([VectorMladd vec_mladd] vec_vmal_ub (vector_unsigned_char, vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char ); + impl_mul!([VectorMladd vec_mladd] vec_vmal_uh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short); + impl_mul!([VectorMladd vec_mladd] vec_vmal_uf (vector_unsigned_int, vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int ); + #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorGfmsum { unsafe fn vec_gfmsum(self, b: Self) -> Result; @@ -2593,1150 +2861,2744 @@ mod sealed { vec_vgfmaf(self, b, c) } } -} -/// Load Count to Block Boundary -#[inline] -#[target_feature(enable = "vector")] -#[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(lcbb, BLOCK_BOUNDARY = 512))] -unsafe fn __lcbb(ptr: *const u8) -> u32 { - lcbb(ptr, const { validate_block_boundary(BLOCK_BOUNDARY) }) -} + #[inline] + #[target_feature(enable = "vector")] + #[cfg_attr(test, assert_instr(vgef, D = 3))] + unsafe fn vgef( + a: vector_unsigned_int, + b: vector_unsigned_int, + c: *const u32, + ) -> vector_unsigned_int { + static_assert_uimm_bits!(D, 2); + let offset: u32 = simd_extract(b, D); + let ptr = c.byte_add(offset as usize); + let value = ptr.read(); + simd_insert(a, D, value) + } -/// Vector element-wise addition. -#[inline] -#[target_feature(enable = "vector")] -#[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_add(a: T, b: U) -> >::Result -where - T: sealed::VectorAdd, -{ - a.vec_add(b) -} + #[inline] + #[target_feature(enable = "vector")] + #[cfg_attr(test, assert_instr(vgeg, D = 1))] + unsafe fn vgeg( + a: vector_unsigned_long_long, + b: vector_unsigned_long_long, + c: *const u64, + ) -> vector_unsigned_long_long { + static_assert_uimm_bits!(D, 1); + let offset: u64 = simd_extract(b, D); + let ptr = c.byte_add(offset as usize); + let value = ptr.read(); + simd_insert(a, D, value) + } -/// Vector element-wise subtraction. -#[inline] -#[target_feature(enable = "vector")] -#[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_sub(a: T, b: U) -> >::Result -where - T: sealed::VectorSub, -{ - a.vec_sub(b) -} + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorGatherElement { + type Element; + type Offset; + unsafe fn vec_gather_element( + self, + b: Self::Offset, + c: *const Self::Element, + ) -> Self; + } + + macro_rules! impl_vec_gather_element { + ($($instr:ident $ty:ident)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorGatherElement for $ty { + type Element = l_t_t!($ty); + type Offset = t_u!($ty); -/// Vector element-wise multiplication. -/// -/// ## Purpose -/// Compute the products of corresponding elements of two vectors. -/// -/// ## Result value -/// Each element of r receives the product of the corresponding elements of a and b. -#[inline] -#[target_feature(enable = "vector")] -#[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_mul(a: T, b: T) -> T -where - T: sealed::VectorMul, -{ - a.vec_mul(b) -} + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_gather_element(self, b: Self::Offset, c: *const Self::Element) -> Self { + transmute($instr::(transmute(self), b, c.cast())) + } + } + )* + } + } -/// Vector Count Leading Zeros -#[inline] -#[target_feature(enable = "vector")] -#[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_cntlz(a: T) -> ::Result -where - T: sealed::CountBits, -{ - a.vec_cntlz() -} + impl_vec_gather_element! { + vgef vector_signed_int + vgef vector_bool_int + vgef vector_unsigned_int -/// Vector Count Trailing Zeros -#[inline] -#[target_feature(enable = "vector")] -#[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_cnttz(a: T) -> ::Result -where - T: sealed::CountBits, -{ - a.vec_cnttz() -} + vgeg vector_signed_long_long + vgeg vector_bool_long_long + vgeg vector_unsigned_long_long -/// Vector Population Count -/// -/// Computes the population count (number of set bits) in each element of the input. -#[inline] -#[target_feature(enable = "vector")] -#[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_popcnt(a: T) -> ::Result -where - T: sealed::CountBits, -{ - a.vec_popcnt() -} + vgef vector_float + vgeg vector_double + } -/// Vector element-wise maximum. -#[inline] -#[target_feature(enable = "vector")] -#[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_max(a: T, b: U) -> >::Result -where - T: sealed::VectorMax, -{ - a.vec_max(b) -} + #[inline] + #[target_feature(enable = "vector")] + #[cfg_attr(test, assert_instr(vscef, D = 3))] + unsafe fn vscef(a: vector_unsigned_int, b: vector_unsigned_int, c: *mut u32) { + static_assert_uimm_bits!(D, 2); + let value = simd_extract(a, D); + let offset: u32 = simd_extract(b, D); + let ptr = c.byte_add(offset as usize); + ptr.write(value); + } -/// Vector element-wise minimum. -#[inline] -#[target_feature(enable = "vector")] -#[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_min(a: T, b: U) -> >::Result -where - T: sealed::VectorMin, -{ - a.vec_min(b) -} + #[inline] + #[target_feature(enable = "vector")] + #[cfg_attr(test, assert_instr(vsceg, D = 1))] + unsafe fn vsceg( + a: vector_unsigned_long_long, + b: vector_unsigned_long_long, + c: *mut u64, + ) { + static_assert_uimm_bits!(D, 1); + let value = simd_extract(a, D); + let offset: u64 = simd_extract(b, D); + let ptr = c.byte_add(offset as usize); + ptr.write(value); + } -/// Vector abs. -#[inline] -#[target_feature(enable = "vector")] -#[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_abs(a: T) -> T -where - T: sealed::VectorAbs, -{ - a.vec_abs() -} + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorScatterElement { + type Element; + type Offset; + unsafe fn vec_scatter_element(self, b: Self::Offset, c: *mut Self::Element); + } -/// Vector negative abs. -#[inline] -#[target_feature(enable = "vector")] -#[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_nabs(a: T) -> T { - a.vec_nabs() -} + macro_rules! impl_vec_scatter_element { + ($($instr:ident $ty:ident)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorScatterElement for $ty { + type Element = l_t_t!($ty); + type Offset = t_u!($ty); -/// Vector Negative Multiply Add -#[inline] -#[target_feature(enable = "vector")] -#[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_nmadd(a: T, b: T, c: T) -> T { - a.vec_nmadd(b, c) -} + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_scatter_element(self, b: Self::Offset, c: *mut Self::Element) { + $instr::(transmute(self), b, c.cast()) + } + } + )* + } + } -/// Vector Negative Multiply Subtract -#[inline] -#[target_feature(enable = "vector")] -#[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_nmsub(a: T, b: T, c: T) -> T { - a.vec_nmsub(b, c) -} + impl_vec_scatter_element! { + vscef vector_signed_int + vscef vector_bool_int + vscef vector_unsigned_int -/// Vector square root. -#[inline] -#[target_feature(enable = "vector")] -#[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_sqrt(a: T) -> T -where - T: sealed::VectorSqrt, -{ - a.vec_sqrt() -} + vsceg vector_signed_long_long + vsceg vector_bool_long_long + vsceg vector_unsigned_long_long -/// Vector Splat -#[inline] -#[target_feature(enable = "vector")] -#[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_splat(a: T) -> T -where - T: sealed::VectorSplat, -{ - a.vec_splat::() -} + vscef vector_float + vsceg vector_double + } -/// Vector splats. + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorSel: Sized { + unsafe fn vec_sel(self, b: Self, c: Mask) -> Self; + } + + macro_rules! impl_vec_sel { + ($($ty:ident)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorSel for $ty { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_sel(self, b: Self, c: t_u!($ty)) -> Self { + let b = simd_and(b, transmute(c)); + let a = simd_and(self, simd_xor(transmute(c), transmute(vector_signed_char([!0; 16])))); + simd_or(a, b) + } + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorSel for $ty { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_sel(self, b: Self, c: t_b!($ty)) -> Self { + // defer to the implementation with an unsigned mask + self.vec_sel(b, transmute::<_, t_u!($ty)>(c)) + } + } + )* + } + } + + impl_vec_sel! { + vector_signed_char + vector_signed_short + vector_signed_int + vector_signed_long_long + + vector_unsigned_char + vector_unsigned_short + vector_unsigned_int + vector_unsigned_long_long + + vector_bool_char + vector_bool_short + vector_bool_int + vector_bool_long_long + + vector_float + vector_double + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorFpTestDataClass { + type Result; + unsafe fn vec_fp_test_data_class(self) -> (Self::Result, i32); + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorFpTestDataClass for vector_float { + type Result = vector_bool_int; + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_fp_test_data_class(self) -> (Self::Result, i32) { + let PackedTuple { x, y } = vftcisb(self, CLASS); + (x, y) + } + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorFpTestDataClass for vector_double { + type Result = vector_bool_long_long; + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_fp_test_data_class(self) -> (Self::Result, i32) { + let PackedTuple { x, y } = vftcidb(self, CLASS); + (x, y) + } + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorCompare { + unsafe fn vec_all_lt(self, other: Self) -> i32; + unsafe fn vec_all_le(self, other: Self) -> i32; + unsafe fn vec_all_gt(self, other: Self) -> i32; + unsafe fn vec_all_ge(self, other: Self) -> i32; + } + + // NOTE: this implementation is currently non-optimal, but it does work for floats even with + // only `vector` enabled. + // + // - https://github.com/llvm/llvm-project/issues/129434 + // - https://github.com/llvm/llvm-project/issues/130424 + macro_rules! impl_vec_compare { + ($($ty:ident)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorCompare for $ty { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_all_lt(self, other: Self) -> i32 { + simd_reduce_all(simd_lt::<_, t_b!($ty)>(self, other)) as i32 + } + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_all_le(self, other: Self) -> i32 { + simd_reduce_all(simd_le::<_, t_b!($ty)>(self, other)) as i32 + } + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_all_gt(self, other: Self) -> i32 { + simd_reduce_all(simd_gt::<_, t_b!($ty)>(self, other)) as i32 + } + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_all_ge(self, other: Self) -> i32 { + simd_reduce_all(simd_ge::<_, t_b!($ty)>(self, other)) as i32 + } + } + )* + } + } + + impl_vec_compare! { + vector_signed_char + vector_unsigned_char + + vector_signed_short + vector_unsigned_short + + vector_signed_int + vector_unsigned_int + vector_float + + vector_signed_long_long + vector_unsigned_long_long + vector_double + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorTestMask { + type Mask; + unsafe fn vec_test_mask(self, other: Self::Mask) -> i32; + } + + macro_rules! impl_vec_test_mask { + ($($instr:ident $ty:ident)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorTestMask for $ty { + type Mask = t_u!($ty); + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_test_mask(self, other: Self::Mask) -> i32 { + vtm(transmute(self), transmute(other)) + } + } + )* + } + } + + impl_vec_test_mask! { + vector_signed_char + vector_signed_short + vector_signed_int + vector_signed_long_long + + vector_unsigned_char + vector_unsigned_short + vector_unsigned_int + vector_unsigned_long_long + + vector_float + vector_double + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorSearchString { + unsafe fn vec_search_string_cc( + self, + b: Self, + c: vector_unsigned_char, + ) -> (vector_unsigned_char, i32); + + unsafe fn vec_search_string_until_zero_cc( + self, + b: Self, + c: vector_unsigned_char, + ) -> (vector_unsigned_char, i32); + } + + macro_rules! impl_vec_search_string{ + ($($intr_s:ident $intr_sz:ident $ty:ident)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorSearchString for $ty { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_search_string_cc(self, b: Self, c: vector_unsigned_char) -> (vector_unsigned_char, i32) { + let PackedTuple { x,y } = $intr_s(transmute(self), transmute(b), c); + (x, y) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_search_string_until_zero_cc(self, b: Self, c: vector_unsigned_char) -> (vector_unsigned_char, i32) { + let PackedTuple { x,y } = $intr_sz(transmute(self), transmute(b), c); + (x, y) + } + } + + )* + } + } + + impl_vec_search_string! { + vstrsb vstrszb vector_signed_char + vstrsb vstrszb vector_bool_char + vstrsb vstrszb vector_unsigned_char + + vstrsh vstrszh vector_signed_short + vstrsh vstrszh vector_bool_short + vstrsh vstrszh vector_unsigned_short + + vstrsf vstrszf vector_signed_int + vstrsf vstrszf vector_bool_int + vstrsf vstrszf vector_unsigned_int + } + + #[inline] + #[target_feature(enable = "vector")] + #[cfg_attr(test, assert_instr(vcdgb))] + pub unsafe fn vcdgb(a: vector_signed_long_long) -> vector_double { + simd_as(a) + } + + #[inline] + #[target_feature(enable = "vector")] + #[cfg_attr(test, assert_instr(vcdlgb))] + pub unsafe fn vcdlgb(a: vector_unsigned_long_long) -> vector_double { + simd_as(a) + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorDouble { + unsafe fn vec_double(self) -> vector_double; + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorDouble for vector_signed_long_long { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_double(self) -> vector_double { + vcdgb(self) + } + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorDouble for vector_unsigned_long_long { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_double(self) -> vector_double { + vcdlgb(self) + } + } + + #[inline] + #[target_feature(enable = "vector")] + #[cfg_attr( + all(test, target_feature = "vector-enhancements-2"), + assert_instr(vcefb) + )] + pub unsafe fn vcefb(a: vector_signed_int) -> vector_float { + simd_as(a) + } + + #[inline] + #[target_feature(enable = "vector")] + #[cfg_attr( + all(test, target_feature = "vector-enhancements-2"), + assert_instr(vcelfb) + )] + pub unsafe fn vcelfb(a: vector_unsigned_int) -> vector_float { + simd_as(a) + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorFloat { + unsafe fn vec_float(self) -> vector_float; + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorFloat for vector_signed_int { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_float(self) -> vector_float { + vcefb(self) + } + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorFloat for vector_unsigned_int { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_float(self) -> vector_float { + vcelfb(self) + } + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorExtendSigned64 { + unsafe fn vec_extend_s64(self) -> vector_signed_long_long; + } + + #[inline] + #[target_feature(enable = "vector")] + // FIXME(llvm): https://github.com/llvm/llvm-project/issues/129899 + // #[cfg_attr(test, assert_instr(vsegb))] + pub unsafe fn vsegb(a: vector_signed_char) -> vector_signed_long_long { + simd_as(simd_shuffle::<_, _, i8x2>( + a, + a, + const { u32x2::from_array([7, 15]) }, + )) + } + + #[inline] + #[target_feature(enable = "vector")] + // FIXME(llvm): https://github.com/llvm/llvm-project/issues/129899 + // #[cfg_attr(test, assert_instr(vsegh))] + pub unsafe fn vsegh(a: vector_signed_short) -> vector_signed_long_long { + simd_as(simd_shuffle::<_, _, i16x2>( + a, + a, + const { u32x2::from_array([3, 7]) }, + )) + } + + #[inline] + #[target_feature(enable = "vector")] + // FIXME(llvm): https://github.com/llvm/llvm-project/issues/129899 + // #[cfg_attr(test, assert_instr(vsegf))] + pub unsafe fn vsegf(a: vector_signed_int) -> vector_signed_long_long { + simd_as(simd_shuffle::<_, _, i32x2>( + a, + a, + const { u32x2::from_array([1, 3]) }, + )) + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorExtendSigned64 for vector_signed_char { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_extend_s64(self) -> vector_signed_long_long { + vsegb(self) + } + } + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorExtendSigned64 for vector_signed_short { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_extend_s64(self) -> vector_signed_long_long { + vsegh(self) + } + } + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorExtendSigned64 for vector_signed_int { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_extend_s64(self) -> vector_signed_long_long { + vsegf(self) + } + } + + // NOTE: VectorSigned and VectorUnsigned make strong safety assumptions around floats. + // This is what C provides, but even IBM does not clearly document these constraints. + // + // https://doc.rust-lang.org/std/intrinsics/simd/fn.simd_cast.html + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorSigned { + type Result; + unsafe fn vec_signed(self) -> Self::Result; + } + + test_impl! { vcgsb (a: vector_float) -> vector_signed_int [simd_cast, "vector-enhancements-2" vcgsb] } + test_impl! { vcgdb (a: vector_double) -> vector_signed_long_long [simd_cast, vcgdb] } + + impl_vec_trait! { [VectorSigned vec_signed] vcgsb (vector_float) -> vector_signed_int } + impl_vec_trait! { [VectorSigned vec_signed] vcgdb (vector_double) -> vector_signed_long_long } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorUnsigned { + type Result; + unsafe fn vec_unsigned(self) -> Self::Result; + } + + test_impl! { vclgsb (a: vector_float) -> vector_unsigned_int [simd_cast, "vector-enhancements-2" vclgsb] } + test_impl! { vclgdb (a: vector_double) -> vector_unsigned_long_long [simd_cast, vclgdb] } + + impl_vec_trait! { [VectorUnsigned vec_unsigned] vclgsb (vector_float) -> vector_unsigned_int } + impl_vec_trait! { [VectorUnsigned vec_unsigned] vclgdb (vector_double) -> vector_unsigned_long_long } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorCopyUntilZero { + unsafe fn vec_cp_until_zero(self) -> Self; + } + + test_impl! { vec_vistrb (a: vector_unsigned_char) -> vector_unsigned_char [vistrb, vistrb] } + test_impl! { vec_vistrh (a: vector_unsigned_short) -> vector_unsigned_short [vistrh, vistrh] } + test_impl! { vec_vistrf (a: vector_unsigned_int) -> vector_unsigned_int [vistrf, vistrf] } + + impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_signed_char) } + impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_bool_char) } + impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_unsigned_char) } + + impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_signed_short) } + impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_bool_short) } + impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_unsigned_short) } + + impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_signed_int) } + impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_bool_int) } + impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_unsigned_int) } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorCopyUntilZeroCC: Sized { + unsafe fn vec_cp_until_zero_cc(self) -> (Self, i32); + } + + test_impl! { vec_vistrbs (a: vector_unsigned_char) -> PackedTuple [vistrbs, vistrbs] } + test_impl! { vec_vistrhs (a: vector_unsigned_short) -> PackedTuple [vistrhs, vistrhs] } + test_impl! { vec_vistrfs (a: vector_unsigned_int) -> PackedTuple [vistrfs, vistrfs] } + + macro_rules! impl_vec_copy_until_zero_cc { + ($($intr:ident $ty:ident)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorCopyUntilZeroCC for $ty { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cp_until_zero_cc(self) -> (Self, i32) { + let PackedTuple { x,y } = $intr(transmute(self)); + (transmute(x), y) + } + } + + )* + } + } + + impl_vec_copy_until_zero_cc! { + vec_vistrbs vector_signed_char + vec_vistrbs vector_bool_char + vec_vistrbs vector_unsigned_char + + vec_vistrhs vector_signed_short + vec_vistrhs vector_bool_short + vec_vistrhs vector_unsigned_short + + vec_vistrfs vector_signed_int + vec_vistrfs vector_bool_int + vec_vistrfs vector_unsigned_int + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorSrdb { + unsafe fn vec_srdb(self, b: Self) -> Self; + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorSld { + unsafe fn vec_sld(self, b: Self) -> Self; + + unsafe fn vec_sldw(self, b: Self) -> Self; + + unsafe fn vec_sldb(self, b: Self) -> Self; + } + + // FIXME(llvm) https://github.com/llvm/llvm-project/issues/129955 + // ideally we could implement this in terms of llvm.fshl.i128 + // #[link_name = "llvm.fshl.i128"] fn fshl_i128(a: u128, b: u128, c: u128) -> u128; + // transmute(fshl_i128(transmute(a), transmute(b), const { C * 8 } )) + + macro_rules! impl_vec_sld { + ($($ty:ident)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorSld for $ty { + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_sld(self, b: Self) -> Self { + static_assert_uimm_bits!(C, 4); + transmute(vsldb(transmute(self), transmute(b), C)) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_sldw(self, b: Self) -> Self { + static_assert_uimm_bits!(C, 2); + transmute(vsldb(transmute(self), transmute(b), const { 4 * C })) + } + + #[inline] + #[target_feature(enable = "vector-enhancements-2")] + unsafe fn vec_sldb(self, b: Self) -> Self { + static_assert_uimm_bits!(C, 3); + transmute(vsld(transmute(self), transmute(b), C)) + } + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorSrdb for $ty { + #[inline] + #[target_feature(enable = "vector-enhancements-2")] + unsafe fn vec_srdb(self, b: Self) -> Self { + static_assert_uimm_bits!(C, 3); + transmute(vsrd(transmute(self), transmute(b), C)) + } + } + )* + } + } + + impl_vec_sld! { + vector_signed_char + vector_bool_char + vector_unsigned_char + + vector_signed_short + vector_bool_short + vector_unsigned_short + + vector_signed_int + vector_bool_int + vector_unsigned_int + + vector_signed_long_long + vector_bool_long_long + vector_unsigned_long_long + + vector_float + vector_double + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorCompareRange: Sized { + type Result; + + unsafe fn vstrc(self, b: Self, c: Self) -> Self::Result; + unsafe fn vstrcz(self, b: Self, c: Self) -> Self::Result; + unsafe fn vstrcs(self, b: Self, c: Self) -> (Self::Result, i32); + unsafe fn vstrczs(self, b: Self, c: Self) -> (Self::Result, i32); + } + + const fn validate_compare_range_imm(imm: u32) { + if !matches!(imm, 0 | 4 | 8 | 12) { + panic!("IMM needs to be one of 0, 4, 8, 12"); + } + } + + macro_rules! impl_compare_range { + ($($ty:ident $vstrc:ident $vstrcs:ident $vstrcz:ident $vstrczs:ident)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorCompareRange for $ty { + type Result = t_b!($ty); + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vstrc(self, b: Self, c: Self) -> Self::Result { + const { validate_compare_range_imm }; + $vstrc(self, b, c, IMM) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vstrcz(self, b: Self, c: Self) -> Self::Result { + const { validate_compare_range_imm }; + $vstrcz(self, b, c, IMM) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vstrcs(self, b: Self, c: Self) -> (Self::Result, i32) { + const { validate_compare_range_imm }; + let PackedTuple { x, y } = $vstrcs(self, b, c, IMM); + (x,y) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vstrczs(self, b: Self, c: Self) -> (Self::Result, i32) { + const { validate_compare_range_imm }; + let PackedTuple { x, y } = $vstrczs(self, b, c, IMM); + (x,y) + } + } + )* + } + } + + impl_compare_range! { + vector_unsigned_char vstrcb vstrcbs vstrczb vstrczbs + vector_unsigned_short vstrch vstrchs vstrczh vstrczhs + vector_unsigned_int vstrcf vstrcfs vstrczf vstrczfs + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorComparePredicate: Sized { + type Result; + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpgt(self, other: Self) -> Self::Result { + simd_gt(self, other) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpge(self, other: Self) -> Self::Result { + simd_ge(self, other) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmplt(self, other: Self) -> Self::Result { + simd_lt(self, other) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmple(self, other: Self) -> Self::Result { + simd_le(self, other) + } + } + + macro_rules! impl_compare_predicate { + ($($ty:ident)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorComparePredicate for $ty { + type Result = t_b!($ty); + } + )* + } + } + + impl_compare_predicate! { + vector_signed_char + vector_unsigned_char + + vector_signed_short + vector_unsigned_short + + vector_signed_int + vector_unsigned_int + vector_float + + vector_signed_long_long + vector_unsigned_long_long + vector_double + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorEquality: Sized { + type Result; + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpeq(self, other: Self) -> Self::Result { + simd_eq(self, other) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpne(self, other: Self) -> Self::Result { + simd_ne(self, other) + } + } + + macro_rules! impl_compare_equality { + ($($ty:ident)*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorEquality for $ty { + type Result = t_b!($ty); + } + )* + } + } + + impl_compare_equality! { + vector_bool_char + vector_signed_char + vector_unsigned_char + + vector_bool_short + vector_signed_short + vector_unsigned_short + + vector_bool_int + vector_signed_int + vector_unsigned_int + vector_float + + vector_bool_long_long + vector_signed_long_long + vector_unsigned_long_long + vector_double + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorEqualityIdx: Sized { + type Result; + + unsafe fn vec_cmpeq_idx(self, other: Self) -> Self::Result; + unsafe fn vec_cmpne_idx(self, other: Self) -> Self::Result; + + unsafe fn vec_cmpeq_idx_cc(self, other: Self) -> (Self::Result, i32); + unsafe fn vec_cmpne_idx_cc(self, other: Self) -> (Self::Result, i32); + + unsafe fn vec_cmpeq_or_0_idx(self, other: Self) -> Self::Result; + unsafe fn vec_cmpne_or_0_idx(self, other: Self) -> Self::Result; + + unsafe fn vec_cmpeq_or_0_idx_cc(self, other: Self) -> (Self::Result, i32); + unsafe fn vec_cmpne_or_0_idx_cc(self, other: Self) -> (Self::Result, i32); + } + + macro_rules! impl_compare_equality_idx { + ($($ty:ident $ret:ident + $cmpeq:ident $cmpne:ident + $cmpeq_or_0:ident $cmpne_or_0:ident + $cmpeq_cc:ident $cmpne_cc:ident + $cmpeq_or_0_cc:ident $cmpne_or_0_cc:ident + )*) => { + $( + #[unstable(feature = "stdarch_s390x", issue = "135681")] + impl VectorEqualityIdx for $ty { + type Result = $ret; + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpeq_idx(self, other: Self) -> Self::Result { + transmute($cmpeq(transmute(self), transmute(other))) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpne_idx(self, other: Self) -> Self::Result { + transmute($cmpne(transmute(self), transmute(other))) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpeq_or_0_idx(self, other: Self) -> Self::Result { + transmute($cmpeq_or_0(transmute(self), transmute(other))) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpne_or_0_idx(self, other: Self) -> Self::Result { + transmute($cmpne_or_0(transmute(self), transmute(other))) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpeq_idx_cc(self, other: Self) -> (Self::Result, i32) { + let PackedTuple { x, y } = $cmpeq_cc(transmute(self), transmute(other)); + (transmute(x), y) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpne_idx_cc(self, other: Self) -> (Self::Result, i32) { + let PackedTuple { x, y } = $cmpne_cc(transmute(self), transmute(other)); + (transmute(x),y) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpeq_or_0_idx_cc(self, other: Self) -> (Self::Result, i32) { + let PackedTuple { x, y } = $cmpeq_or_0_cc(transmute(self), transmute(other)); + (transmute(x), y) + } + + #[inline] + #[target_feature(enable = "vector")] + unsafe fn vec_cmpne_or_0_idx_cc(self, other: Self) -> (Self::Result, i32) { + let PackedTuple { x, y } = $cmpne_or_0_cc(transmute(self), transmute(other)); + (transmute(x),y) + } + } + )* + } + } + + impl_compare_equality_idx! { + vector_signed_char vector_signed_char vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs + vector_bool_char vector_unsigned_char vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs + vector_unsigned_char vector_unsigned_char vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs + vector_signed_short vector_signed_short vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs + vector_bool_short vector_unsigned_short vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs + vector_unsigned_short vector_unsigned_short vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs + vector_signed_int vector_signed_int vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs + vector_bool_int vector_unsigned_int vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs + vector_unsigned_int vector_unsigned_int vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs + } +} + +/// Load Count to Block Boundary +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +#[cfg_attr(test, assert_instr(lcbb, BLOCK_BOUNDARY = 512))] +unsafe fn __lcbb(ptr: *const u8) -> u32 { + lcbb(ptr, const { validate_block_boundary(BLOCK_BOUNDARY) }) +} + +/// Vector element-wise addition. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_add, U>(a: T, b: U) -> T::Result { + a.vec_add(b) +} + +/// Vector element-wise subtraction. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_sub, U>(a: T, b: U) -> T::Result { + a.vec_sub(b) +} + +/// Vector element-wise multiplication. +/// +/// ## Purpose +/// Compute the products of corresponding elements of two vectors. +/// +/// ## Result value +/// Each element of r receives the product of the corresponding elements of a and b. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_mul(a: T, b: T) -> T { + a.vec_mul(b) +} + +/// Vector Count Leading Zeros +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cntlz(a: T) -> T::Result { + a.vec_cntlz() +} + +/// Vector Count Trailing Zeros +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_cnttz(a: T) -> T::Result { + a.vec_cnttz() +} + +/// Vector Population Count +/// +/// Computes the population count (number of set bits) in each element of the input. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_popcnt(a: T) -> T::Result { + a.vec_popcnt() +} + +/// Vector element-wise maximum. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_max, U>(a: T, b: U) -> T::Result { + a.vec_max(b) +} + +/// Vector element-wise minimum. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_min, U>(a: T, b: U) -> T::Result { + a.vec_min(b) +} + +/// Vector abs. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_abs(a: T) -> T { + a.vec_abs() +} + +/// Vector negative abs. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_nabs(a: T) -> T { + a.vec_nabs() +} + +/// Vector Negative Multiply Add +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_nmadd(a: T, b: T, c: T) -> T { + a.vec_nmadd(b, c) +} + +/// Vector Negative Multiply Subtract +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_nmsub(a: T, b: T, c: T) -> T { + a.vec_nmsub(b, c) +} + +/// Vector square root. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_sqrt(a: T) -> T { + a.vec_sqrt() +} + +/// Vector Splat +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_splat(a: T) -> T { + a.vec_splat::() +} + +/// Vector splats. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_splats, U>(a: T) -> U { + a.vec_splats() +} + +/// Vector and +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_and, U>(a: T, b: U) -> T::Result { + a.vec_and(b) +} + +/// Vector or +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_or, U>(a: T, b: U) -> T::Result { + a.vec_or(b) +} + +/// Vector xor +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_xor, U>(a: T, b: U) -> T::Result { + a.vec_xor(b) +} + +/// Vector nor +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_nor, U>(a: T, b: U) -> T::Result { + a.vec_nor(b) +} + +/// Vector nand +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_nand, U>(a: T, b: U) -> T::Result { + a.vec_nand(b) +} + +/// Vector xnor +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_eqv, U>(a: T, b: U) -> T::Result { + a.vec_eqv(b) +} + +/// Vector andc. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_andc, U>(a: T, b: U) -> T::Result { + a.vec_andc(b) +} + +/// Vector OR with Complement +/// +/// ## Purpose +/// Performs a bitwise OR of the first vector with the bitwise-complemented second vector. +/// +/// ## Result value +/// r is the bitwise OR of a and the bitwise complement of b. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_orc, U>(a: T, b: U) -> T::Result { + a.vec_orc(b) +} + +/// Vector floor. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_floor(a: T) -> T { + a.vec_floor() +} + +/// Vector ceil. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_ceil(a: T) -> T { + a.vec_ceil() +} + +/// Returns a vector containing the truncated values of the corresponding elements of the given vector. +/// Each element of the result contains the value of the corresponding element of a, truncated to an integral value. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_trunc(a: T) -> T { + a.vec_trunc() +} + +/// Returns a vector containing the rounded values to the nearest representable floating-point integer, +/// using IEEE round-to-nearest rounding, of the corresponding elements of the given vector +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_round(a: T) -> T { + a.vec_round() +} + +/// Returns a vector by using the current rounding mode to round every +/// floating-point element in the given vector to integer. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_roundc(a: T) -> T { + a.vec_roundc() +} + +/// Returns a vector containing the largest representable floating-point integral values less +/// than or equal to the values of the corresponding elements of the given vector. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_roundm(a: T) -> T { + // the IBM docs note + // + // > vec_roundm provides the same functionality as vec_floor, except that vec_roundz would not trigger the IEEE-inexact exception. + // + // but in practice `vec_floor` also does not trigger that exception, so both are equivalent + a.vec_floor() +} + +/// Returns a vector containing the smallest representable floating-point integral values greater +/// than or equal to the values of the corresponding elements of the given vector. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_roundp(a: T) -> T { + // the IBM docs note + // + // > vec_roundp provides the same functionality as vec_ceil, except that vec_roundz would not trigger the IEEE-inexact exception. + // + // but in practice `vec_ceil` also does not trigger that exception, so both are equivalent + a.vec_ceil() +} + +/// Returns a vector containing the truncated values of the corresponding elements of the given vector. +/// Each element of the result contains the value of the corresponding element of a, truncated to an integral value. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_roundz(a: T) -> T { + // the IBM docs note + // + // > vec_roundz provides the same functionality as vec_trunc, except that vec_roundz would not trigger the IEEE-inexact exception. + // + // but in practice `vec_trunc` also does not trigger that exception, so both are equivalent + a.vec_trunc() +} + +/// Returns a vector by using the current rounding mode to round every floating-point element in the given vector to integer. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_rint(a: T) -> T { + a.vec_rint() +} + +/// Vector Average +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_avg, U>(a: T, b: U) -> T::Result { + a.vec_avg(b) +} + +/// Vector Shift Left +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_sl, U>(a: T, b: U) -> T::Result { + a.vec_sl(b) +} + +/// Vector Shift Right +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_sr, U>(a: T, b: U) -> T::Result { + a.vec_sr(b) +} + +/// Vector Shift Right Algebraic +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_sra, U>(a: T, b: U) -> T::Result { + a.vec_sra(b) +} + +/// Vector Shift Left by Byte +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_slb, U>(a: T, b: U) -> T::Result { + a.vec_slb(b) +} + +/// Vector Shift Right by Byte +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_srb, U>(a: T, b: U) -> T::Result { + a.vec_srb(b) +} + +/// Vector Shift Right Algebraic by Byte +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_srab, U>(a: T, b: U) -> T::Result { + a.vec_srab(b) +} + +/// Vector Element Rotate Left +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_rl, U>(a: T, b: U) -> T::Result { + a.vec_rl(b) +} + +/// Performs a left shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding +/// element of a left by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by zeros. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_sll(a: T, b: vector_unsigned_char) -> T +where + T: sealed::VectorSll, +{ + a.vec_sll(b) +} + +/// Performs a right shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding +/// element of a right by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by zeros. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_srl(a: T, b: vector_unsigned_char) -> T +where + T: sealed::VectorSrl, +{ + a.vec_srl(b) +} + +/// Performs an algebraic right shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding +/// element of a right by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by copies of +/// the most significant bit of the element of a. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_sral(a: T, b: vector_unsigned_char) -> T +where + T: sealed::VectorSral, +{ + a.vec_sral(b) +} + +/// Rotates each element of a vector left by a given number of bits. Each element of the result is obtained by rotating the corresponding element +/// of a left by the number of bits specified by b, modulo the number of bits in the element. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_rli(a: T, bits: core::ffi::c_ulong) -> T { + a.vec_rli(bits) +} + +/// Returns a vector with the elements of the input vector in reversed order. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_reve(a: T) -> T { + a.vec_reve() +} + +/// Returns a vector where each vector element contains the corresponding byte-reversed vector element of the input vector. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_revb(a: T) -> T { + a.vec_revb() +} + +/// Merges the most significant ("high") halves of two vectors. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_mergeh(a: T, b: T) -> T { + a.vec_mergeh(b) +} + +/// Vector Pack +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_pack, U>(a: T, b: U) -> T::Result { + a.vec_pack(b) +} + +/// Vector Pack Saturated +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_packs, U>(a: T, b: U) -> T::Result { + a.vec_packs(b) +} + +/// Vector Pack Saturated Condition Code +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_packs_cc(a: T, b: T, c: *mut i32) -> T::Result { + let (x, y) = a.vec_packs_cc(b); + unsafe { c.write(y) }; + x +} + +/// Vector Pack Saturated Unsigned +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_packsu, U>(a: T, b: U) -> T::Result { + a.vec_packsu(b) +} + +/// Vector Pack Saturated Unsigned Condition Code +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_packsu_cc(a: T, b: T, c: *mut i32) -> T::Result { + let (x, y) = a.vec_packsu_cc(b); + unsafe { c.write(y) }; + x +} + +/// Vector Unpack High +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_unpackh(a: T) -> ::Result { + a.vec_unpackh() +} + +/// Vector Unpack Low +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_unpackl(a: T) -> ::Result { + a.vec_unpackl() +} + +/// Merges the least significant ("low") halves of two vectors. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_mergel(a: T, b: T) -> T { + a.vec_mergel(b) +} + +/// Generates byte masks for elements in the return vector. For each bit in a, if the bit is one, all bit positions +/// in the corresponding byte element of d are set to ones. Otherwise, if the bit is zero, the corresponding byte element is set to zero. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +#[cfg_attr(test, assert_instr(vgbm, MASK = 0x00FF))] +pub unsafe fn vec_genmask() -> vector_unsigned_char { + vector_unsigned_char(const { genmask::() }) +} + +/// Vector Generate Mask (Byte) +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +#[cfg_attr(test, assert_instr(vrepib, L = 3, H = 5))] +pub unsafe fn vec_genmasks_8() -> vector_unsigned_char { + vector_unsigned_char(const { [genmasks(u8::BITS, L, H) as u8; 16] }) +} + +/// Vector Generate Mask (Halfword) +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +#[cfg_attr(test, assert_instr(vrepih, L = 3, H = 5))] +pub unsafe fn vec_genmasks_16() -> vector_unsigned_short { + vector_unsigned_short(const { [genmasks(u16::BITS, L, H) as u16; 8] }) +} + +/// Vector Generate Mask (Word) +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +#[cfg_attr(test, assert_instr(vgmf, L = 3, H = 5))] +pub unsafe fn vec_genmasks_32() -> vector_unsigned_int { + vector_unsigned_int(const { [genmasks(u32::BITS, L, H) as u32; 4] }) +} + +/// Vector Generate Mask (Doubleword) +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +#[cfg_attr(test, assert_instr(vgmg, L = 3, H = 5))] +pub unsafe fn vec_genmasks_64() -> vector_unsigned_long_long { + vector_unsigned_long_long(const { [genmasks(u64::BITS, L, H); 2] }) +} + +/// Returns a vector that contains some elements of two vectors, in the order specified by a third vector. +/// Each byte of the result is selected by using the least significant 5 bits of the corresponding byte of c as an index into the concatenated bytes of a and b. +/// Note: The vector generate mask built-in function [`vec_genmask`] could help generate the mask c. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_perm(a: T, b: T, c: vector_unsigned_char) -> T { + a.vec_perm(b, c) +} + +/// Vector Sum Across Quadword +/// +/// Returns a vector containing the results of performing a sum across all the elements in each of the quadword of vector a, +/// and the rightmost word or doubleword element of the b. The result is an unsigned 128-bit integer. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_sum_u128(a: T, b: T) -> vector_unsigned_char { + a.vec_sum_u128(b) +} + +/// Vector Sum Across Doubleword +/// +/// Returns a vector containing the results of performing a sum across all the elements in each of the doubleword of vector a, +/// and the rightmost sub-element of the corresponding doubleword of b. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_sum2(a: T, b: T) -> vector_unsigned_long_long { + a.vec_sum2(b) +} + +/// Vector Sum Across Word +/// +/// Returns a vector containing the results of performing a sum across all the elements in each of the word of vector a, +/// and the rightmost sub-element of the corresponding word of b. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_sum4(a: T, b: T) -> vector_unsigned_int { + a.vec_sum4(b) +} + +/// Vector Addition unsigned 128-bits +/// +/// Adds unsigned quadword values. +/// +/// This function operates on the vectors as 128-bit unsigned integers. It returns low 128 bits of a + b. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +#[cfg_attr(test, assert_instr(vaq))] +pub unsafe fn vec_add_u128( + a: vector_unsigned_char, + b: vector_unsigned_char, +) -> vector_unsigned_char { + let a: u128 = transmute(a); + let b: u128 = transmute(b); + transmute(a.wrapping_add(b)) +} + +/// Vector Subtract unsigned 128-bits +/// +/// Subtracts unsigned quadword values. +/// +/// This function operates on the vectors as 128-bit unsigned integers. It returns low 128 bits of a - b. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +#[cfg_attr(test, assert_instr(vsq))] +pub unsafe fn vec_sub_u128( + a: vector_unsigned_char, + b: vector_unsigned_char, +) -> vector_unsigned_char { + let a: u128 = transmute(a); + let b: u128 = transmute(b); + + transmute(a.wrapping_sub(b)) +} + +/// Vector Subtract Carryout +/// +/// Returns a vector containing the borrow produced by subtracting each of corresponding elements of b from a. +/// +/// On each resulting element, the value is 0 if a borrow occurred, or 1 if no borrow occurred. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_subc, U>(a: T, b: U) -> T::Result { + a.vec_subc(b) +} + +/// Gets the carry bit of the 128-bit subtraction of two quadword values. +/// This function operates on the vectors as 128-bit unsigned integers. It returns a vector containing the borrow produced by subtracting b from a, as unsigned 128-bits integers. +/// If no borrow occurred, the bit 127 of d is 1; otherwise it is set to 0. All other bits of d are 0. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +#[cfg_attr(test, assert_instr(vscbiq))] +pub unsafe fn vec_subc_u128( + a: vector_unsigned_char, + b: vector_unsigned_char, +) -> vector_unsigned_char { + // FIXME(llvm) sadly this does not work https://github.com/llvm/llvm-project/issues/129608 + // let a: u128 = transmute(a); + // let b: u128 = transmute(b); + // transmute(!a.overflowing_sub(b).1 as u128) + transmute(vscbiq(transmute(a), transmute(b))) +} + +/// Vector Add Compute Carryout unsigned 128-bits +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +#[cfg_attr(test, assert_instr(vaccq))] +pub unsafe fn vec_addc_u128( + a: vector_unsigned_char, + b: vector_unsigned_char, +) -> vector_unsigned_char { + let a: u128 = transmute(a); + let b: u128 = transmute(b); + transmute(a.overflowing_add(b).1 as u128) +} + +/// Vector Add With Carry unsigned 128-bits +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +#[cfg_attr(test, assert_instr(vacq))] +pub unsafe fn vec_adde_u128( + a: vector_unsigned_char, + b: vector_unsigned_char, + c: vector_unsigned_char, +) -> vector_unsigned_char { + let a: u128 = transmute(a); + let b: u128 = transmute(b); + let c: u128 = transmute(c); + // FIXME(llvm) sadly this does not work + // let (d, _carry) = a.carrying_add(b, c & 1 != 0); + // transmute(d) + transmute(vacq(a, b, c)) +} + +/// Vector Add With Carry Compute Carry unsigned 128-bits +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +#[cfg_attr(test, assert_instr(vacccq))] +pub unsafe fn vec_addec_u128( + a: vector_unsigned_char, + b: vector_unsigned_char, + c: vector_unsigned_char, +) -> vector_unsigned_char { + let a: u128 = transmute(a); + let b: u128 = transmute(b); + let c: u128 = transmute(c); + let (_d, carry) = a.carrying_add(b, c & 1 != 0); + transmute(carry as u128) +} + +/// Subtracts unsigned quadword values with carry bit from a previous operation. +/// +/// This function operates on the vectors as 128-bit unsigned integers. It returns a vector containing the result of subtracting of b from a, +/// and the carryout bit from a previous operation. +/// +/// Note: Only the borrow indication bit (127-bit) of c is used, and the other bits are ignored. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +#[cfg_attr(test, assert_instr(vsbiq))] +pub unsafe fn vec_sube_u128( + a: vector_unsigned_char, + b: vector_unsigned_char, + c: vector_unsigned_char, +) -> vector_unsigned_char { + transmute(vsbiq(transmute(a), transmute(b), transmute(c))) +} + +/// Vector Subtract with Carryout, Carryout +/// +/// Gets the carry bit of the 128-bit subtraction of two quadword values with carry bit from the previous operation. +/// +/// It returns a vector containing the carryout produced from the result of subtracting of b from a, +/// and the carryout bit from a previous operation. If no borrow occurred, the 127-bit of d is 1, otherwise 0. +/// All other bits of d are 0. +/// +/// Note: Only the borrow indication bit (127-bit) of c is used, and the other bits are ignored. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +#[cfg_attr(test, assert_instr(vsbcbiq))] +pub unsafe fn vec_subec_u128( + a: vector_unsigned_char, + b: vector_unsigned_char, + c: vector_unsigned_char, +) -> vector_unsigned_char { + transmute(vsbcbiq(transmute(a), transmute(b), transmute(c))) +} + +/// Vector Splat Signed Byte +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +#[cfg_attr(test, assert_instr(vrepib, IMM = 42))] +pub unsafe fn vec_splat_s8() -> vector_signed_char { + vector_signed_char([IMM; 16]) +} + +/// Vector Splat Signed Halfword #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_splats(a: T) -> U -where - T: sealed::VectorSplats, -{ - a.vec_splats() +#[cfg_attr(test, assert_instr(vrepih, IMM = 42))] +pub unsafe fn vec_splat_s16() -> vector_signed_short { + vector_signed_short([IMM as i16; 8]) } -/// Vector and +/// Vector Splat Signed Word #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_and(a: T, b: U) -> >::Result -where - T: sealed::VectorAnd, -{ - a.vec_and(b) +#[cfg_attr(test, assert_instr(vrepif, IMM = 42))] +pub unsafe fn vec_splat_s32() -> vector_signed_int { + vector_signed_int([IMM as i32; 4]) } -/// Vector or +/// Vector Splat Signed Doubleword #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_or(a: T, b: U) -> >::Result -where - T: sealed::VectorOr, -{ - a.vec_or(b) +#[cfg_attr(test, assert_instr(vrepig, IMM = 42))] +pub unsafe fn vec_splat_s64() -> vector_signed_long_long { + vector_signed_long_long([IMM as i64; 2]) } -/// Vector xor +/// Vector Splat Unsigned Byte #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_xor(a: T, b: U) -> >::Result -where - T: sealed::VectorXor, -{ - a.vec_xor(b) +#[cfg_attr(test, assert_instr(vrepib, IMM = 42))] +pub unsafe fn vec_splat_u8() -> vector_unsigned_char { + vector_unsigned_char([IMM; 16]) } -/// Vector nor +/// Vector Splat Unsigned Halfword #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_nor(a: T, b: U) -> >::Result -where - T: sealed::VectorNor, -{ - a.vec_nor(b) +#[cfg_attr(test, assert_instr(vrepih, IMM = 42))] +pub unsafe fn vec_splat_u16() -> vector_unsigned_short { + vector_unsigned_short([IMM as u16; 8]) } -/// Vector nand +/// Vector Splat Unsigned Word #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_nand(a: T, b: U) -> >::Result -where - T: sealed::VectorNand, -{ - a.vec_nand(b) +#[cfg_attr(test, assert_instr(vrepif, IMM = 42))] +pub unsafe fn vec_splat_u32() -> vector_unsigned_int { + vector_unsigned_int([IMM as u32; 4]) } -/// Vector xnor +/// Vector Splat Unsigned Doubleword #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_eqv(a: T, b: U) -> >::Result -where - T: sealed::VectorEqv, -{ - a.vec_eqv(b) +#[cfg_attr(test, assert_instr(vrepig, IMM = 42))] +pub unsafe fn vec_splat_u64() -> vector_unsigned_long_long { + vector_unsigned_long_long([IMM as u64; 2]) } -/// Vector andc. +macro_rules! vec_find_any { + ($($Trait:ident $fun:ident)*) => { + $( + #[inline] + #[target_feature(enable = "vector")] + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub unsafe fn $fun, U>(a: T, b: U) -> T::Result { + a.$fun(b) + } + )* + } +} + +vec_find_any! { + VectorFindAnyEq vec_find_any_eq + VectorFindAnyNe vec_find_any_ne + VectorFindAnyEqIdx vec_find_any_eq_idx + VectorFindAnyNeIdx vec_find_any_ne_idx + VectorFindAnyEqOrZeroIdx vec_find_any_eq_or_0_idx + VectorFindAnyNeOrZeroIdx vec_find_any_ne_or_0_idx +} + +macro_rules! vec_find_any_cc { + ($($Trait:ident $fun:ident)*) => { + $( + #[inline] + #[target_feature(enable = "vector")] + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub unsafe fn $fun, U>(a: T, b: U, c: *mut i32) -> T::Result { + let (x, y) = a.$fun(b); + unsafe { c.write(y) }; + x + } + )* + } +} + +vec_find_any_cc! { + VectorFindAnyEqCC vec_find_any_eq_cc + VectorFindAnyNeCC vec_find_any_ne_cc + VectorFindAnyEqIdxCC vec_find_any_eq_idx_cc + VectorFindAnyNeIdxCC vec_find_any_ne_idx_cc + VectorFindAnyEqOrZeroIdxCC vec_find_any_eq_or_0_idx_cc + VectorFindAnyNeOrZeroIdxCC vec_find_any_ne_or_0_idx_cc +} + +/// Vector Load #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_andc(a: T, b: U) -> >::Result -where - T: sealed::VectorAndc, -{ - a.vec_andc(b) +pub unsafe fn vec_xl(offset: isize, ptr: *const T::ElementType) -> T { + T::vec_xl(offset, ptr) } -/// Vector OR with Complement -/// -/// ## Purpose -/// Performs a bitwise OR of the first vector with the bitwise-complemented second vector. -/// -/// ## Result value -/// r is the bitwise OR of a and the bitwise complement of b. +/// Vector Load Pair #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_orc(a: T, b: U) -> >::Result -where - T: sealed::VectorOrc, -{ - a.vec_orc(b) +pub unsafe fn vec_load_pair(a: T::ElementType, b: T::ElementType) -> T { + T::vec_load_pair(a, b) } -/// Vector floor. +/// Vector Load to Block Boundary #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_floor(a: T) -> T -where - T: sealed::VectorFloor, -{ - a.vec_floor() +pub unsafe fn vec_load_bndry( + ptr: *const T::ElementType, +) -> MaybeUninit { + T::vec_load_bndry::(ptr) } -/// Vector ceil. +/// Vector Store #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_ceil(a: T) -> T -where - T: sealed::VectorCeil, -{ - a.vec_ceil() +pub unsafe fn vec_xst(vector: T, offset: isize, ptr: *mut T::ElementType) { + vector.vec_xst(offset, ptr) } -/// Returns a vector containing the truncated values of the corresponding elements of the given vector. -/// Each element of the result contains the value of the corresponding element of a, truncated to an integral value. +/// Vector Load with Length #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_trunc(a: T) -> T -where - T: sealed::VectorTrunc, -{ - a.vec_trunc() +pub unsafe fn vec_load_len( + ptr: *const T::ElementType, + byte_count: u32, +) -> T { + T::vec_load_len(ptr, byte_count) +} + +/// Vector Store with Length +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_store_len( + vector: T, + ptr: *mut T::ElementType, + byte_count: u32, +) { + vector.vec_store_len(ptr, byte_count) +} + +/// Vector Load Rightmost with Length +#[inline] +#[target_feature(enable = "vector-packed-decimal")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +#[cfg_attr(test, assert_instr(vlrlr))] +pub unsafe fn vec_load_len_r(ptr: *const u8, byte_count: u32) -> vector_unsigned_char { + vlrl(byte_count, ptr) +} + +/// Vector Store Rightmost with Length +#[inline] +#[target_feature(enable = "vector-packed-decimal")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +#[cfg_attr(test, assert_instr(vstrlr))] +pub unsafe fn vec_store_len_r(vector: vector_unsigned_char, ptr: *mut u8, byte_count: u32) { + vstrl(vector, byte_count, ptr) +} + +/// Vector Multiply Add +#[inline] +#[target_feature(enable = "vector-packed-decimal")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_madd(a: T, b: T, c: T) -> T { + a.vec_madd(b, c) +} + +/// Vector Multiply Add +#[inline] +#[target_feature(enable = "vector-packed-decimal")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_msub(a: T, b: T, c: T) -> T { + a.vec_msub(b, c) +} + +/// Vector Multiply and Add Even +#[inline] +#[target_feature(enable = "vector-packed-decimal")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_meadd(a: T, b: T, c: T::Result) -> T::Result { + a.vec_meadd(b, c) +} + +/// Vector Multiply and Add Odd +#[inline] +#[target_feature(enable = "vector-packed-decimal")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_moadd(a: T, b: T, c: T::Result) -> T::Result { + a.vec_moadd(b, c) +} + +/// Vector Multiply and Add High +#[inline] +#[target_feature(enable = "vector-packed-decimal")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_mhadd(a: T, b: T, c: T::Result) -> T::Result { + a.vec_mhadd(b, c) +} + +/// Vector Multiply and Add Low +#[inline] +#[target_feature(enable = "vector-packed-decimal")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_mladd(a: T, b: T, c: T::Result) -> T::Result { + a.vec_mladd(b, c) +} + +/// Vector Checksum +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +#[cfg_attr(test, assert_instr(vcksm))] +pub unsafe fn vec_checksum(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int { + vcksm(a, b) +} + +/// Vector Multiply Even +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_mule, U>(a: T, b: T) -> U { + a.vec_mule(b) +} + +/// Vector Multiply Odd +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_mulo, U>(a: T, b: T) -> U { + a.vec_mulo(b) +} + +/// Vector Multiply High +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_mulh, U>(a: T, b: T) -> U { + a.vec_mulh(b) +} + +/// Vector Galois Field Multiply Sum +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_gfmsum, U>(a: T, b: T) -> U { + a.vec_gfmsum(b) +} + +/// Vector Galois Field Multiply Sum +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_gfmsum_accum( + a: T, + b: T, + c: T::Result, +) -> T::Result { + a.vec_gfmsum_accum(b, c) } -/// Returns a vector containing the rounded values to the nearest representable floating-point integer, -/// using IEEE round-to-nearest rounding, of the corresponding elements of the given vector +/// Vector Galois Field Multiply Sum 128-bits #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_round(a: T) -> T -where - T: sealed::VectorRound, -{ - a.vec_round() +#[cfg_attr(test, assert_instr(vgfmg))] +pub unsafe fn vec_gfmsum_128( + a: vector_unsigned_long_long, + b: vector_unsigned_long_long, +) -> vector_unsigned_char { + transmute(vgfmg(a, b)) } -/// Returns a vector by using the current rounding mode to round every -/// floating-point element in the given vector to integer. +/// Vector Galois Field Multiply Sum and Accumulate 128-bits #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_roundc(a: T) -> T -where - T: sealed::VectorRoundc, -{ - a.vec_roundc() +#[cfg_attr(test, assert_instr(vgfmag))] +pub unsafe fn vec_gfmsum_accum_128( + a: vector_unsigned_long_long, + b: vector_unsigned_long_long, + c: vector_unsigned_char, +) -> vector_unsigned_char { + transmute(vgfmag(a, b, transmute(c))) } -/// Returns a vector containing the largest representable floating-point integral values less -/// than or equal to the values of the corresponding elements of the given vector. +/// Vector Bit Permute #[inline] -#[target_feature(enable = "vector")] +#[target_feature(enable = "vector-enhancements-1")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_roundm(a: T) -> T -where - T: sealed::VectorFloor, -{ - // the IBM docs note - // - // > vec_roundm provides the same functionality as vec_floor, except that vec_roundz would not trigger the IEEE-inexact exception. - // - // but in practice `vec_floor` also does not trigger that exception, so both are equivalent - a.vec_floor() +#[cfg_attr(test, assert_instr(vbperm))] +pub unsafe fn vec_bperm_u128( + a: vector_unsigned_char, + b: vector_unsigned_char, +) -> vector_unsigned_long_long { + vbperm(a, b) } -/// Returns a vector containing the smallest representable floating-point integral values greater -/// than or equal to the values of the corresponding elements of the given vector. +/// Vector Gather Element #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_roundp(a: T) -> T -where - T: sealed::VectorCeil, -{ - // the IBM docs note - // - // > vec_roundp provides the same functionality as vec_ceil, except that vec_roundz would not trigger the IEEE-inexact exception. - // - // but in practice `vec_ceil` also does not trigger that exception, so both are equivalent - a.vec_ceil() +pub unsafe fn vec_gather_element( + a: T, + b: T::Offset, + c: *const T::Element, +) -> T { + a.vec_gather_element::(b, c) } -/// Returns a vector containing the truncated values of the corresponding elements of the given vector. -/// Each element of the result contains the value of the corresponding element of a, truncated to an integral value. +/// Vector Select #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_roundz(a: T) -> T -where - T: sealed::VectorTrunc, -{ - // the IBM docs note - // - // > vec_roundz provides the same functionality as vec_trunc, except that vec_roundz would not trigger the IEEE-inexact exception. - // - // but in practice `vec_trunc` also does not trigger that exception, so both are equivalent - a.vec_trunc() +pub unsafe fn vec_sel, U>(a: T, b: T, c: U) -> T { + a.vec_sel(b, c) } -/// Returns a vector by using the current rounding mode to round every floating-point element in the given vector to integer. -#[inline] -#[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_rint(a: T) -> T -where - T: sealed::VectorRint, -{ - a.vec_rint() -} +pub const __VEC_CLASS_FP_ZERO_P: u32 = 1 << 11; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_ZERO_N: u32 = 1 << 10; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_ZERO: u32 = __VEC_CLASS_FP_ZERO_P | __VEC_CLASS_FP_ZERO_N; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_NORMAL_P: u32 = 1 << 9; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_NORMAL_N: u32 = 1 << 8; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_NORMAL: u32 = __VEC_CLASS_FP_NORMAL_P | __VEC_CLASS_FP_NORMAL_N; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_SUBNORMAL_P: u32 = 1 << 7; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_SUBNORMAL_N: u32 = 1 << 6; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_SUBNORMAL: u32 = __VEC_CLASS_FP_SUBNORMAL_P | __VEC_CLASS_FP_SUBNORMAL_N; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_INFINITY_P: u32 = 1 << 5; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_INFINITY_N: u32 = 1 << 4; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_INFINITY: u32 = __VEC_CLASS_FP_INFINITY_P | __VEC_CLASS_FP_INFINITY_N; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_QNAN_P: u32 = 1 << 3; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_QNAN_N: u32 = 1 << 2; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_QNAN: u32 = __VEC_CLASS_FP_QNAN_P | __VEC_CLASS_FP_QNAN_N; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_SNAN_P: u32 = 1 << 1; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_SNAN_N: u32 = 1 << 0; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_SNAN: u32 = __VEC_CLASS_FP_SNAN_P | __VEC_CLASS_FP_SNAN_N; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_NAN: u32 = __VEC_CLASS_FP_QNAN | __VEC_CLASS_FP_SNAN; +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub const __VEC_CLASS_FP_NOT_NORMAL: u32 = + __VEC_CLASS_FP_NAN | __VEC_CLASS_FP_SUBNORMAL | __VEC_CLASS_FP_ZERO | __VEC_CLASS_FP_INFINITY; -/// Vector Average +/// Vector Floating-Point Test Data Class +/// +/// You can use the `__VEC_CLASS_FP_*` constants as the argument for this operand #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_avg, U>(a: T, b: U) -> T::Result { - a.vec_avg(b) +pub unsafe fn vec_fp_test_data_class( + a: T, + c: *mut i32, +) -> T::Result { + let (x, y) = a.vec_fp_test_data_class::(); + c.write(y); + x } -/// Vector Shift Left #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_sl(a: T, b: U) -> >::Result -where - T: sealed::VectorSl, -{ - a.vec_sl(b) +pub unsafe fn vec_all_nan(a: T) -> i32 { + i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 == 0) } -/// Vector Shift Right #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_sr(a: T, b: U) -> >::Result -where - T: sealed::VectorSr, -{ - a.vec_sr(b) +pub unsafe fn vec_all_numeric(a: T) -> i32 { + i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 == 3) } -/// Vector Shift Right Algebraic #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_sra(a: T, b: U) -> >::Result -where - T: sealed::VectorSra, -{ - a.vec_sra(b) +pub unsafe fn vec_any_nan(a: T) -> i32 { + i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 != 3) } -/// Vector Shift Left by Byte #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_slb(a: T, b: U) -> >::Result -where - T: sealed::VectorSlb, -{ - a.vec_slb(b) +pub unsafe fn vec_any_numeric(a: T) -> i32 { + i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 != 0) } -/// Vector Shift Right by Byte +/// Vector Test under Mask #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_srb(a: T, b: U) -> >::Result -where - T: sealed::VectorSrb, -{ - a.vec_srb(b) +pub unsafe fn vec_test_mask(a: T, b: T::Mask) -> i32 { + // I can't find much information about this, but this might just be a check for whether the + // bitwise and of a and b is non-zero? + a.vec_test_mask(b) } -/// Vector Shift Right Algebraic by Byte +/// Vector Search String #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_srab(a: T, b: U) -> >::Result -where - T: sealed::VectorSrab, -{ - a.vec_srab(b) +pub unsafe fn vec_search_string_cc( + a: T, + b: T, + c: vector_unsigned_char, + d: *mut i32, +) -> vector_unsigned_char { + let (x, y) = a.vec_search_string_cc(b, c); + unsafe { d.write(y) }; + x } -/// Vector Element Rotate Left +/// Vector Search String Until Zero #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_rl(a: T, b: U) -> >::Result -where - T: sealed::VectorRl, -{ - a.vec_rl(b) +pub unsafe fn vec_search_string_until_zero_cc( + a: T, + b: T, + c: vector_unsigned_char, + d: *mut i32, +) -> vector_unsigned_char { + let (x, y) = a.vec_search_string_until_zero_cc(b, c); + unsafe { d.write(y) }; + x } -/// Performs a left shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding -/// element of a left by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by zeros. +/// Vector Convert from float (even elements) to double #[inline] -#[target_feature(enable = "vector")] +#[target_feature(enable = "vector-enhancements-1")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_sll(a: T, b: vector_unsigned_char) -> T -where - T: sealed::VectorSll, -{ - a.vec_sll(b) +// FIXME: this emits `vflls` where `vldeb` is expected +// #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vldeb))] +pub unsafe fn vec_doublee(a: vector_float) -> vector_double { + let even = simd_shuffle::<_, _, f32x2>(a, a, const { u32x2::from_array([0, 2]) }); + simd_as(even) } -/// Performs a right shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding -/// element of a right by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by zeros. +/// Vector Convert from double to float (even elements) #[inline] -#[target_feature(enable = "vector")] +#[target_feature(enable = "vector-enhancements-1")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_srl(a: T, b: vector_unsigned_char) -> T -where - T: sealed::VectorSrl, -{ - a.vec_srl(b) +// FIXME: the C version uses a shuffle mask with poison; we can't do that +// #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vledb))] +pub unsafe fn vec_floate(a: vector_double) -> vector_float { + let truncated: f32x2 = simd_as(a); + simd_shuffle( + truncated, + truncated, + const { u32x4::from_array([0, 0, 1, 1]) }, + ) } -/// Performs an algebraic right shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding -/// element of a right by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by copies of -/// the most significant bit of the element of a. +/// Vector Convert from int to float #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_sral(a: T, b: vector_unsigned_char) -> T -where - T: sealed::VectorSral, -{ - a.vec_sral(b) +pub unsafe fn vec_float(a: impl sealed::VectorFloat) -> vector_float { + a.vec_float() } -/// Rotates each element of a vector left by a given number of bits. Each element of the result is obtained by rotating the corresponding element -/// of a left by the number of bits specified by b, modulo the number of bits in the element. +/// Vector Convert from long long to double #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_rli(a: T, bits: core::ffi::c_ulong) -> T -where - T: sealed::VectorRli, -{ - a.vec_rli(bits) +pub unsafe fn vec_double(a: impl sealed::VectorDouble) -> vector_double { + a.vec_double() } -/// Returns a vector with the elements of the input vector in reversed order. +/// Vector Sign Extend to Doubleword #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_reve(a: T) -> T -where - T: sealed::VectorReve, -{ - a.vec_reve() +pub unsafe fn vec_extend_s64(a: impl sealed::VectorExtendSigned64) -> vector_signed_long_long { + a.vec_extend_s64() } -/// Returns a vector where each vector element contains the corresponding byte-reversed vector element of the input vector. +/// Vector Convert floating point to signed #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_revb(a: T) -> T -where - T: sealed::VectorRevb, -{ - a.vec_revb() +pub unsafe fn vec_signed(a: T) -> T::Result { + a.vec_signed() } -/// Merges the most significant ("high") halves of two vectors. +/// Vector Convert floating point to unsigned #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_mergeh(a: T, b: T) -> T -where - T: sealed::VectorMergeh, -{ - a.vec_mergeh(b) +pub unsafe fn vec_unsigned(a: T) -> T::Result { + a.vec_unsigned() } -/// Vector Pack +/// Vector Copy Until Zero #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_pack, U>(a: T, b: U) -> T::Result { - a.vec_pack(b) +pub unsafe fn vec_cp_until_zero(a: T) -> T { + a.vec_cp_until_zero() } -/// Vector Pack Saturated +/// Vector Copy Until Zero #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_packs, U>(a: T, b: U) -> T::Result { - a.vec_packs(b) +pub unsafe fn vec_cp_until_zero_cc(a: T, cc: *mut i32) -> T { + let (x, y) = a.vec_cp_until_zero_cc(); + unsafe { cc.write(y) }; + x } -/// Vector Pack Saturated Condition Code +/// Vector Multiply Sum Logical #[inline] -#[target_feature(enable = "vector")] +#[target_feature(enable = "vector-enhancements-1")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_packs_cc(a: T, b: T, c: *mut i32) -> T::Result { - a.vec_packs_cc(b, c) +#[cfg_attr( + all(test, target_feature = "vector-enhancements-1"), + assert_instr(vmslg, D = 4) +)] +pub unsafe fn vec_msum_u128( + a: vector_unsigned_long_long, + b: vector_unsigned_long_long, + c: vector_unsigned_char, +) -> vector_unsigned_char { + const { + if !matches!(D, 0 | 4 | 8 | 12) { + panic!("D needs to be one of 0, 4, 8, 12"); + } + }; + transmute(vmslg(a, b, transmute(c), D)) } -/// Vector Pack Saturated Unsigned +/// Vector Shift Left Double by Byte #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_packsu, U>(a: T, b: U) -> T::Result { - a.vec_packsu(b) +pub unsafe fn vec_sld(a: T, b: T) -> T { + static_assert_uimm_bits!(C, 4); + a.vec_sld::(b) } -/// Vector Pack Saturated Unsigned Condition Code +/// Vector Shift Left Double by Word #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_packsu_cc(a: T, b: T, c: *mut i32) -> T::Result { - a.vec_packsu_cc(b, c) +pub unsafe fn vec_sldw(a: T, b: T) -> T { + static_assert_uimm_bits!(C, 2); + a.vec_sldw::(b) } -/// Vector Unpack High +/// Vector Shift Left Double by Bit #[inline] -#[target_feature(enable = "vector")] +#[target_feature(enable = "vector-enhancements-2")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_unpackh(a: T) -> ::Result { - a.vec_unpackh() +pub unsafe fn vec_sldb(a: T, b: T) -> T { + static_assert_uimm_bits!(C, 3); + a.vec_sldb::(b) } -/// Vector Unpack Low +/// Vector Shift Right Double by Bit #[inline] -#[target_feature(enable = "vector")] +#[target_feature(enable = "vector-enhancements-2")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_unpackl(a: T) -> ::Result { - a.vec_unpackl() +pub unsafe fn vec_srdb(a: T, b: T) -> T { + static_assert_uimm_bits!(C, 3); + a.vec_srdb::(b) } -/// Merges the least significant ("low") halves of two vectors. +/// Vector Compare Ranges #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_mergel(a: T, b: T) -> T -where - T: sealed::VectorMergel, -{ - a.vec_mergel(b) +pub unsafe fn vec_cmprg(a: T, b: T, c: T) -> T::Result { + a.vstrc::<{ FindImm::Eq as u32 }>(b, c) } -/// Generates byte masks for elements in the return vector. For each bit in a, if the bit is one, all bit positions -/// in the corresponding byte element of d are set to ones. Otherwise, if the bit is zero, the corresponding byte element is set to zero. +/// Vector Compare Not in Ranges #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vgbm, MASK = 0x00FF))] -pub unsafe fn vec_genmask() -> vector_unsigned_char { - vector_unsigned_char(const { genmask::() }) +pub unsafe fn vec_cmpnrg(a: T, b: T, c: T) -> T::Result { + a.vstrc::<{ FindImm::Ne as u32 }>(b, c) } -/// Vector Generate Mask (Byte) +/// Vector Compare Ranges Index #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vrepib, L = 3, H = 5))] -pub unsafe fn vec_genmasks_8() -> vector_unsigned_char { - vector_unsigned_char(const { [genmasks(u8::BITS, L, H) as u8; 16] }) +pub unsafe fn vec_cmprg_idx(a: T, b: T, c: T) -> T::Result { + a.vstrc::<{ FindImm::EqIdx as u32 }>(b, c) } -/// Vector Generate Mask (Halfword) +/// Vector Compare Not in Ranges Index #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vrepih, L = 3, H = 5))] -pub unsafe fn vec_genmasks_16() -> vector_unsigned_short { - vector_unsigned_short(const { [genmasks(u16::BITS, L, H) as u16; 8] }) +pub unsafe fn vec_cmpnrg_idx(a: T, b: T, c: T) -> T::Result { + a.vstrc::<{ FindImm::NeIdx as u32 }>(b, c) } -/// Vector Generate Mask (Word) +/// Vector Compare Ranges with Condition Code #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vgmf, L = 3, H = 5))] -pub unsafe fn vec_genmasks_32() -> vector_unsigned_int { - vector_unsigned_int(const { [genmasks(u32::BITS, L, H) as u32; 4] }) +pub unsafe fn vec_cmprg_cc( + a: T, + b: T, + c: T, + d: *mut i32, +) -> T::Result { + let (x, y) = a.vstrcs::<{ FindImm::Eq as u32 }>(b, c); + d.write(y); + x } -/// Vector Generate Mask (Doubleword) +/// Vector Compare Not in Ranges with Condition Code #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vgmg, L = 3, H = 5))] -pub unsafe fn vec_genmasks_64() -> vector_unsigned_long_long { - vector_unsigned_long_long(const { [genmasks(u64::BITS, L, H); 2] }) +pub unsafe fn vec_cmpnrg_cc( + a: T, + b: T, + c: T, + d: *mut i32, +) -> T::Result { + let (x, y) = a.vstrcs::<{ FindImm::Ne as u32 }>(b, c); + d.write(y); + x } -/// Returns a vector that contains some elements of two vectors, in the order specified by a third vector. -/// Each byte of the result is selected by using the least significant 5 bits of the corresponding byte of c as an index into the concatenated bytes of a and b. -/// Note: The vector generate mask built-in function [`vec_genmask`] could help generate the mask c. +/// Vector Compare Ranges Index with Condition Code #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_perm(a: T, b: T, c: vector_unsigned_char) -> T { - a.vec_perm(b, c) +pub unsafe fn vec_cmprg_idx_cc( + a: T, + b: T, + c: T, + d: *mut i32, +) -> T::Result { + let (x, y) = a.vstrcs::<{ FindImm::EqIdx as u32 }>(b, c); + d.write(y); + x } -/// Vector Sum Across Quadword -/// -/// Returns a vector containing the results of performing a sum across all the elements in each of the quadword of vector a, -/// and the rightmost word or doubleword element of the b. The result is an unsigned 128-bit integer. +/// Vector Compare Not in Ranges Index with Condition Code #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_sum_u128(a: T, b: T) -> vector_unsigned_char { - a.vec_sum_u128(b) +pub unsafe fn vec_cmpnrg_idx_cc( + a: T, + b: T, + c: T, + d: *mut i32, +) -> T::Result { + let (x, y) = a.vstrcs::<{ FindImm::NeIdx as u32 }>(b, c); + d.write(y); + x } -/// Vector Sum Across Doubleword -/// -/// Returns a vector containing the results of performing a sum across all the elements in each of the doubleword of vector a, -/// and the rightmost sub-element of the corresponding doubleword of b. -#[inline] +/// Vector Compare Ranges or Zero Index#[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_sum2(a: T, b: T) -> vector_unsigned_long_long { - a.vec_sum2(b) +pub unsafe fn vec_cmprg_or_0_idx(a: T, b: T, c: T) -> T::Result { + a.vstrcz::<{ FindImm::EqIdx as u32 }>(b, c) } -/// Vector Sum Across Word -/// -/// Returns a vector containing the results of performing a sum across all the elements in each of the word of vector a, -/// and the rightmost sub-element of the corresponding word of b. +/// Vector Compare Not in Ranges or Zero Index #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_sum4(a: T, b: T) -> vector_unsigned_int { - a.vec_sum4(b) +pub unsafe fn vec_cmpnrg_or_0_idx(a: T, b: T, c: T) -> T::Result { + a.vstrcz::<{ FindImm::NeIdx as u32 }>(b, c) } -/// Vector Addition unsigned 128-bits -/// -/// Adds unsigned quadword values. -/// -/// This function operates on the vectors as 128-bit unsigned integers. It returns low 128 bits of a + b. +/// Vector Compare Ranges or Zero Index with Condition Code #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vaq))] -pub unsafe fn vec_add_u128( - a: vector_unsigned_char, - b: vector_unsigned_char, -) -> vector_unsigned_char { - let a: u128 = transmute(a); - let b: u128 = transmute(b); - transmute(a.wrapping_add(b)) +pub unsafe fn vec_cmprg_or_0_idx_cc( + a: T, + b: T, + c: T, + d: *mut i32, +) -> T::Result { + let (x, y) = a.vstrczs::<{ FindImm::EqIdx as u32 }>(b, c); + d.write(y); + x } -/// Vector Subtract unsigned 128-bits -/// -/// Subtracts unsigned quadword values. -/// -/// This function operates on the vectors as 128-bit unsigned integers. It returns low 128 bits of a - b. +/// Vector Compare Not in Ranges or Zero Index with Condition Code #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vsq))] -pub unsafe fn vec_sub_u128( - a: vector_unsigned_char, - b: vector_unsigned_char, -) -> vector_unsigned_char { - let a: u128 = transmute(a); - let b: u128 = transmute(b); - - transmute(a.wrapping_sub(b)) +pub unsafe fn vec_cmpnrg_or_0_idx_cc( + a: T, + b: T, + c: T, + d: *mut i32, +) -> T::Result { + let (x, y) = a.vstrczs::<{ FindImm::NeIdx as u32 }>(b, c); + d.write(y); + x } -/// Vector Subtract Carryout -/// -/// Returns a vector containing the borrow produced by subtracting each of corresponding elements of b from a. -/// -/// On each resulting element, the value is 0 if a borrow occurred, or 1 if no borrow occurred. +/// Vector Compare Equal #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_subc(a: T, b: U) -> >::Result -where - T: sealed::VectorSubc, -{ - a.vec_subc(b) +pub unsafe fn vec_cmpeq(a: T, b: T) -> T::Result { + a.vec_cmpeq(b) } -/// Gets the carry bit of the 128-bit subtraction of two quadword values. -/// This function operates on the vectors as 128-bit unsigned integers. It returns a vector containing the borrow produced by subtracting b from a, as unsigned 128-bits integers. -/// If no borrow occurred, the bit 127 of d is 1; otherwise it is set to 0. All other bits of d are 0. +/// Vector Compare Not Equal #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vscbiq))] -pub unsafe fn vec_subc_u128( - a: vector_unsigned_char, - b: vector_unsigned_char, -) -> vector_unsigned_char { - // FIXME(llvm) sadly this does not work https://github.com/llvm/llvm-project/issues/129608 - // let a: u128 = transmute(a); - // let b: u128 = transmute(b); - // transmute(!a.overflowing_sub(b).1 as u128) - transmute(vscbiq(transmute(a), transmute(b))) +pub unsafe fn vec_cmpne(a: T, b: T) -> T::Result { + a.vec_cmpne(b) } -/// Vector Add Compute Carryout unsigned 128-bits +/// Vector Compare Greater Than #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vaccq))] -pub unsafe fn vec_addc_u128( - a: vector_unsigned_char, - b: vector_unsigned_char, -) -> vector_unsigned_char { - let a: u128 = transmute(a); - let b: u128 = transmute(b); - transmute(a.overflowing_add(b).1 as u128) +pub unsafe fn vec_cmpgt(a: T, b: T) -> T::Result { + a.vec_cmpgt(b) } -/// Vector Add With Carry unsigned 128-bits +/// Vector Compare Greater Than or Equal #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vacq))] -pub unsafe fn vec_adde_u128( - a: vector_unsigned_char, - b: vector_unsigned_char, - c: vector_unsigned_char, -) -> vector_unsigned_char { - let a: u128 = transmute(a); - let b: u128 = transmute(b); - let c: u128 = transmute(c); - // FIXME(llvm) sadly this does not work - // let (d, _carry) = a.carrying_add(b, c & 1 != 0); - // transmute(d) - transmute(vacq(a, b, c)) +pub unsafe fn vec_cmpge(a: T, b: T) -> T::Result { + a.vec_cmpge(b) } -/// Vector Add With Carry Compute Carry unsigned 128-bits +/// Vector Compare Less #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vacccq))] -pub unsafe fn vec_addec_u128( - a: vector_unsigned_char, - b: vector_unsigned_char, - c: vector_unsigned_char, -) -> vector_unsigned_char { - let a: u128 = transmute(a); - let b: u128 = transmute(b); - let c: u128 = transmute(c); - let (_d, carry) = a.carrying_add(b, c & 1 != 0); - transmute(carry as u128) +pub unsafe fn vec_cmplt(a: T, b: T) -> T::Result { + a.vec_cmplt(b) } -/// Subtracts unsigned quadword values with carry bit from a previous operation. -/// -/// This function operates on the vectors as 128-bit unsigned integers. It returns a vector containing the result of subtracting of b from a, -/// and the carryout bit from a previous operation. -/// -/// Note: Only the borrow indication bit (127-bit) of c is used, and the other bits are ignored. +/// Vector Compare Less Than or Equal #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vsbiq))] -pub unsafe fn vec_sube_u128( - a: vector_unsigned_char, - b: vector_unsigned_char, - c: vector_unsigned_char, -) -> vector_unsigned_char { - transmute(vsbiq(transmute(a), transmute(b), transmute(c))) +pub unsafe fn vec_cmple(a: T, b: T) -> T::Result { + a.vec_cmple(b) } -/// Vector Subtract with Carryout, Carryout -/// -/// Gets the carry bit of the 128-bit subtraction of two quadword values with carry bit from the previous operation. -/// -/// It returns a vector containing the carryout produced from the result of subtracting of b from a, -/// and the carryout bit from a previous operation. If no borrow occurred, the 127-bit of d is 1, otherwise 0. -/// All other bits of d are 0. -/// -/// Note: Only the borrow indication bit (127-bit) of c is used, and the other bits are ignored. +/// Vector Compare Equal Index #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vsbcbiq))] -pub unsafe fn vec_subec_u128( - a: vector_unsigned_char, - b: vector_unsigned_char, - c: vector_unsigned_char, -) -> vector_unsigned_char { - transmute(vsbcbiq(transmute(a), transmute(b), transmute(c))) +pub unsafe fn vec_cmpeq_idx(a: T, b: T) -> T::Result { + a.vec_cmpeq_idx(b) } - -/// Vector Splat Signed Byte +/// Vector Compare Not Equal Index #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vrepib, IMM = 42))] -pub unsafe fn vec_splat_s8() -> vector_signed_char { - vector_signed_char([IMM; 16]) +pub unsafe fn vec_cmpne_idx(a: T, b: T) -> T::Result { + a.vec_cmpne_idx(b) } - -/// Vector Splat Signed Halfword +/// Vector Compare Equal Index with Condition Code #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vrepih, IMM = 42))] -pub unsafe fn vec_splat_s16() -> vector_signed_short { - vector_signed_short([IMM as i16; 8]) +pub unsafe fn vec_cmpeq_idx_cc( + a: T, + b: T, + cc: *mut i32, +) -> T::Result { + let (x, y) = a.vec_cmpeq_idx_cc(b); + unsafe { cc.write(y) }; + x } - -/// Vector Splat Signed Word +/// Vector Compare Not Equal Index with Condition Code #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vrepif, IMM = 42))] -pub unsafe fn vec_splat_s32() -> vector_signed_int { - vector_signed_int([IMM as i32; 4]) +pub unsafe fn vec_cmpne_idx_cc( + a: T, + b: T, + cc: *mut i32, +) -> T::Result { + let (x, y) = a.vec_cmpne_idx_cc(b); + unsafe { cc.write(y) }; + x } - -/// Vector Splat Signed Doubleword +/// Vector Compare Equal or Zero Index #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vrepig, IMM = 42))] -pub unsafe fn vec_splat_s64() -> vector_signed_long_long { - vector_signed_long_long([IMM as i64; 2]) +pub unsafe fn vec_cmpeq_or_0_idx(a: T, b: T) -> T::Result { + a.vec_cmpeq_or_0_idx(b) } - -/// Vector Splat Unsigned Byte +/// Vector Compare Not Equal or Zero Index #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vrepib, IMM = 42))] -pub unsafe fn vec_splat_u8() -> vector_unsigned_char { - vector_unsigned_char([IMM; 16]) +pub unsafe fn vec_cmpne_or_0_idx(a: T, b: T) -> T::Result { + a.vec_cmpne_or_0_idx(b) } - -/// Vector Splat Unsigned Halfword +/// Vector Compare Equal or Zero Index with Condition Code #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vrepih, IMM = 42))] -pub unsafe fn vec_splat_u16() -> vector_unsigned_short { - vector_unsigned_short([IMM as u16; 8]) +pub unsafe fn vec_cmpeq_or_0_idx_cc( + a: T, + b: T, + cc: *mut i32, +) -> T::Result { + let (x, y) = a.vec_cmpeq_or_0_idx_cc(b); + unsafe { cc.write(y) }; + x } - -/// Vector Splat Unsigned Word +/// Vector Compare Not Equal or Zero Index with Condition Code #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vrepif, IMM = 42))] -pub unsafe fn vec_splat_u32() -> vector_unsigned_int { - vector_unsigned_int([IMM as u32; 4]) +pub unsafe fn vec_cmpne_or_0_idx_cc( + a: T, + b: T, + cc: *mut i32, +) -> T::Result { + let (x, y) = a.vec_cmpne_or_0_idx_cc(b); + unsafe { cc.write(y) }; + x } -/// Vector Splat Unsigned Doubleword +/// All Elements Equal #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vrepig, IMM = 42))] -pub unsafe fn vec_splat_u64() -> vector_unsigned_long_long { - vector_unsigned_long_long([IMM as u64; 2]) -} - -macro_rules! vec_find_any { - ($($Trait:ident $fun:ident)*) => { - $( - #[inline] - #[target_feature(enable = "vector")] - #[unstable(feature = "stdarch_s390x", issue = "135681")] - pub unsafe fn $fun(a: T, b: U) -> >::Result - where - T: sealed::$Trait, - { - a.$fun(b) - } - )* - } +pub unsafe fn vec_all_eq(a: T, b: T) -> i32 { + simd_reduce_all(vec_cmpeq(a, b)) as i32 as i32 } -vec_find_any! { - VectorFindAnyEq vec_find_any_eq - VectorFindAnyNe vec_find_any_ne - VectorFindAnyEqIdx vec_find_any_eq_idx - VectorFindAnyNeIdx vec_find_any_ne_idx - VectorFindAnyEqOrZeroIdx vec_find_any_eq_or_0_idx - VectorFindAnyNeOrZeroIdx vec_find_any_ne_or_0_idx +/// All Elements Not Equal +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_all_ne(a: T, b: T) -> i32 { + simd_reduce_all(vec_cmpne(a, b)) as i32 } -macro_rules! vec_find_any_cc { - ($($Trait:ident $fun:ident)*) => { - $( - #[inline] - #[target_feature(enable = "vector")] - #[unstable(feature = "stdarch_s390x", issue = "135681")] - pub unsafe fn $fun(a: T, b: U, c: *mut i32) -> >::Result - where - T: sealed::$Trait, - { - a.$fun(b, c) - } - )* - } +/// Any Element Equal +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_any_eq(a: T, b: T) -> i32 { + simd_reduce_any(vec_cmpeq(a, b)) as i32 } -vec_find_any_cc! { - VectorFindAnyEqCC vec_find_any_eq_cc - VectorFindAnyNeCC vec_find_any_ne_cc - VectorFindAnyEqIdxCC vec_find_any_eq_idx_cc - VectorFindAnyNeIdxCC vec_find_any_ne_idx_cc - VectorFindAnyEqOrZeroIdxCC vec_find_any_eq_or_0_idx_cc - VectorFindAnyNeOrZeroIdxCC vec_find_any_ne_or_0_idx_cc +/// Any Element Not Equal +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_any_ne(a: T, b: T) -> i32 { + simd_reduce_any(vec_cmpne(a, b)) as i32 } -/// Vector Load +/// All Elements Less Than #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_xl(offset: isize, ptr: *const T::ElementType) -> T { - T::vec_xl(offset, ptr) +pub unsafe fn vec_all_lt(a: T, b: T) -> i32 { + a.vec_all_lt(b) } -/// Vector Load Pair +/// All Elements Less Than or Equal #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_load_pair(a: T::ElementType, b: T::ElementType) -> T { - T::vec_load_pair(a, b) +pub unsafe fn vec_all_le(a: T, b: T) -> i32 { + a.vec_all_le(b) } -/// Vector Load to Block Boundary +/// All Elements Greater Than #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_load_bndry( - ptr: *const T::ElementType, -) -> MaybeUninit { - T::vec_load_bndry::(ptr) +pub unsafe fn vec_all_gt(a: T, b: T) -> i32 { + a.vec_all_gt(b) } -/// Vector Store +/// All Elements Greater Than or Equal #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_xst(vector: T, offset: isize, ptr: *mut T::ElementType) { - vector.vec_xst(offset, ptr) +pub unsafe fn vec_all_ge(a: T, b: T) -> i32 { + a.vec_all_ge(b) } -/// Vector Load with Length +/// All Elements Not Less Than #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_load_len( - ptr: *const T::ElementType, - byte_count: u32, -) -> T { - T::vec_load_len(ptr, byte_count) +pub unsafe fn vec_all_nlt(a: T, b: T) -> i32 { + vec_all_ge(a, b) } -/// Vector Store with Length +/// All Elements Not Less Than or Equal #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_store_len( - vector: T, - ptr: *mut T::ElementType, - byte_count: u32, -) { - vector.vec_store_len(ptr, byte_count) +pub unsafe fn vec_all_nle(a: T, b: T) -> i32 { + vec_all_gt(a, b) } -/// Vector Load Rightmost with Length +/// All Elements Not Greater Than #[inline] -#[target_feature(enable = "vector-packed-decimal")] +#[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vlrlr))] -pub unsafe fn vec_load_len_r(ptr: *const u8, byte_count: u32) -> vector_unsigned_char { - vlrl(byte_count, ptr) +pub unsafe fn vec_all_ngt(a: T, b: T) -> i32 { + vec_all_le(a, b) } -/// Vector Store Rightmost with Length +/// All Elements Not Greater Than or Equal #[inline] -#[target_feature(enable = "vector-packed-decimal")] +#[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vstrlr))] -pub unsafe fn vec_store_len_r(vector: vector_unsigned_char, ptr: *mut u8, byte_count: u32) { - vstrl(vector, byte_count, ptr) +pub unsafe fn vec_all_nge(a: T, b: T) -> i32 { + vec_all_lt(a, b) } -/// Vector Multiply Add +/// Any Elements Less Than #[inline] -#[target_feature(enable = "vector-packed-decimal")] +#[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_madd(a: T, b: T, c: T) -> T { - a.vec_madd(b, c) +pub unsafe fn vec_any_lt(a: T, b: T) -> i32 { + !vec_all_ge(a, b) } -/// Vector Multiply Add +/// Any Elements Less Than or Equal #[inline] -#[target_feature(enable = "vector-packed-decimal")] +#[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_msub(a: T, b: T, c: T) -> T { - a.vec_msub(b, c) +pub unsafe fn vec_any_le(a: T, b: T) -> i32 { + !vec_all_gt(a, b) } -/// Vector Checksum +/// Any Elements Greater Than #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vcksm))] -pub unsafe fn vec_checksum(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int { - vcksm(a, b) +pub unsafe fn vec_any_gt(a: T, b: T) -> i32 { + !vec_all_le(a, b) } -/// Vector Multiply Even +/// Any Elements Greater Than or Equal #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_mule, U>(a: T, b: T) -> U { - a.vec_mule(b) +pub unsafe fn vec_any_ge(a: T, b: T) -> i32 { + !vec_all_lt(a, b) } -/// Vector Galois Field Multiply Sum +/// Any Elements Not Less Than #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_gfmsum, U>(a: T, b: T) -> U { - a.vec_gfmsum(b) +pub unsafe fn vec_any_nlt(a: T, b: T) -> i32 { + vec_any_ge(a, b) } -/// Vector Galois Field Multiply Sum +/// Any Elements Not Less Than or Equal #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -pub unsafe fn vec_gfmsum_accum( - a: T, - b: T, - c: T::Result, -) -> T::Result { - a.vec_gfmsum_accum(b, c) +pub unsafe fn vec_any_nle(a: T, b: T) -> i32 { + vec_any_gt(a, b) } -/// Vector Galois Field Multiply Sum 128-bits +/// Any Elements Not Greater Than #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vgfmg))] -pub unsafe fn vec_gfmsum_128( - a: vector_unsigned_long_long, - b: vector_unsigned_long_long, -) -> vector_unsigned_char { - transmute(vgfmg(a, b)) +pub unsafe fn vec_any_ngt(a: T, b: T) -> i32 { + vec_any_le(a, b) } -/// Vector Galois Field Multiply Sum and Accumulate 128-bits +/// Any Elements Not Greater Than or Equal #[inline] #[target_feature(enable = "vector")] #[unstable(feature = "stdarch_s390x", issue = "135681")] -#[cfg_attr(test, assert_instr(vgfmag))] -pub unsafe fn vec_gfmsum_accum_128( - a: vector_unsigned_long_long, - b: vector_unsigned_long_long, - c: vector_unsigned_char, -) -> vector_unsigned_char { - transmute(vgfmag(a, b, transmute(c))) +pub unsafe fn vec_any_nge(a: T, b: T) -> i32 { + vec_any_lt(a, b) } #[cfg(test)] @@ -4650,6 +6512,30 @@ mod tests { [0x4000_0000, -8, 0xFFFE, 2] } + test_vec_2! { test_vec_mulo_u, vec_mulo, u16x8, u16x8 -> u32x4, + [0, 0xFFFF, 0, 2, 0, 2, 0, 1], + [0, 0xFFFF, 0, 4, 0, 0xFFFF, 0, 2], + [0xFFFE_0001, 8, 0x0001_FFFE, 2] + } + + test_vec_2! { test_vec_mulo_i, vec_mulo, i16x8, i16x8 -> i32x4, + [0, i16::MIN, 0, -2, 0, 2, 0, 1], + [0, i16::MIN, 0, 4, 0, i16::MAX, 0, 2], + [0x4000_0000, -8, 0xFFFE, 2] + } + + test_vec_2! { test_vec_mulh_u, vec_mulh, u32x4, u32x4 -> u32x4, + [u32::MAX, 2, 2, 1], + [u32::MAX, 4, u32::MAX, 2], + [u32::MAX - 1, 0, 1, 0] + } + + test_vec_2! { test_vec_mulh_i, vec_mulh, i32x4, i32x4 -> i32x4, + [i32::MIN, -2, 2, 1], + [i32::MIN, 4, i32::MAX, 2], + [0x4000_0000, -1, 0, 0] + } + test_vec_2! { test_vec_gfmsum_1, vec_gfmsum, u16x8, u16x8 -> u32x4, [0x1234, 0x5678, 0x9ABC, 0xDEF0, 0x1357, 0x2468, 0xACE0, 0xBDF0], [0xFFFF, 0x0001, 0x8000, 0x7FFF, 0xAAAA, 0x5555, 0x1234, 0x5678], @@ -4676,4 +6562,586 @@ mod tests { let d: u128 = unsafe { transmute(vec_gfmsum_128(a, b)) }; assert_eq!(d, 0xE000E000E000E000E000E000E000E); } + + #[simd_test(enable = "vector-enhancements-1")] + fn test_vec_bperm_u128() { + let a = vector_unsigned_char([65, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]); + let b = vector_unsigned_char([ + 0, 0, 0, 0, 1, 1, 1, 1, 128, 128, 128, 128, 255, 255, 255, 255, + ]); + let d = unsafe { vec_bperm_u128(a, b) }; + assert_eq!(d.as_array(), &[0xF00, 0]); + } + + #[simd_test(enable = "vector")] + fn test_vec_sel() { + let a = vector_signed_int([1, 2, 3, 4]); + let b = vector_signed_int([5, 6, 7, 8]); + + let e = vector_unsigned_int([9, 10, 11, 12]); + let f = vector_unsigned_int([9, 9, 11, 11]); + + let c: vector_bool_int = unsafe { simd_eq(e, f) }; + assert_eq!(c.as_array(), &[!0, 0, !0, 0]); + let d: vector_signed_int = unsafe { vec_sel(a, b, c) }; + assert_eq!(d.as_array(), &[5, 2, 7, 4]); + } + + #[simd_test(enable = "vector")] + fn test_vec_gather_element() { + let a1: [u32; 10] = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]; + let a2: [u32; 10] = [20, 21, 22, 23, 24, 25, 26, 27, 28, 29]; + + let v1 = vector_unsigned_int([1, 2, 3, 4]); + let v2 = vector_unsigned_int([1, 2, 3, 4]); + + let sizeof_int = core::mem::size_of::() as u32; + let v3 = vector_unsigned_int([ + 5 * sizeof_int, + 8 * sizeof_int, + 9 * sizeof_int, + 6 * sizeof_int, + ]); + + unsafe { + let d1 = vec_gather_element::<_, 0>(v1, v3, a1.as_ptr()); + assert_eq!(d1.as_array(), &[15, 2, 3, 4]); + let d2 = vec_gather_element::<_, 0>(v2, v3, a2.as_ptr()); + assert_eq!(d2.as_array(), &[25, 2, 3, 4]); + } + } + + #[simd_test(enable = "vector")] + fn test_vec_fp_test_data_class() { + let mut cc = 42; + + let v1 = vector_double([0.0, f64::NAN]); + let v2 = vector_double([f64::INFINITY, 1.0]); + let v3 = vector_double([1.0, 2.0]); + + unsafe { + let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_ZERO>(v1, &mut cc); + assert_eq!(cc, 1); + assert_eq!(d.as_array(), &[!0, 0]); + + let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NAN>(v1, &mut cc); + assert_eq!(cc, 1); + assert_eq!(d.as_array(), &[0, !0]); + + let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_INFINITY>(v2, &mut cc); + assert_eq!(cc, 1); + assert_eq!(d.as_array(), &[!0, 0]); + + let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_INFINITY_N>(v2, &mut cc); + assert_eq!(cc, 3); + assert_eq!(d.as_array(), &[0, 0]); + + let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NORMAL>(v2, &mut cc); + assert_eq!(cc, 1); + assert_eq!(d.as_array(), &[0, !0]); + + let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NORMAL>(v3, &mut cc); + assert_eq!(cc, 0); + assert_eq!(d.as_array(), &[!0, !0]); + } + } + + #[simd_test(enable = "vector")] + fn test_vec_fp_any_all_nan_numeric() { + unsafe { + assert_eq!( + vec_all_nan(vector_double([f64::NAN, f64::NAN])), + i32::from(true) + ); + assert_eq!( + vec_all_nan(vector_double([f64::NAN, 1.0])), + i32::from(false) + ); + assert_eq!(vec_all_nan(vector_double([0.0, 1.0])), i32::from(false)); + + assert_eq!( + vec_any_nan(vector_double([f64::NAN, f64::NAN])), + i32::from(true) + ); + assert_eq!(vec_any_nan(vector_double([f64::NAN, 1.0])), i32::from(true)); + assert_eq!(vec_any_nan(vector_double([0.0, 1.0])), i32::from(false)); + + assert_eq!( + vec_all_numeric(vector_double([f64::NAN, f64::NAN])), + i32::from(false) + ); + assert_eq!( + vec_all_numeric(vector_double([f64::NAN, 1.0])), + i32::from(false) + ); + assert_eq!(vec_all_numeric(vector_double([0.0, 1.0])), i32::from(true)); + + assert_eq!( + vec_any_numeric(vector_double([f64::NAN, f64::NAN])), + i32::from(false) + ); + assert_eq!( + vec_any_numeric(vector_double([f64::NAN, 1.0])), + i32::from(true) + ); + assert_eq!(vec_any_numeric(vector_double([0.0, 1.0])), i32::from(true)); + + // "numeric" means "not NaN". infinities are numeric + assert_eq!( + vec_all_numeric(vector_double([f64::INFINITY, f64::NEG_INFINITY])), + i32::from(true) + ); + assert_eq!( + vec_any_numeric(vector_double([f64::INFINITY, f64::NEG_INFINITY])), + i32::from(true) + ); + } + } + + #[simd_test(enable = "vector")] + fn test_vec_test_mask() { + unsafe { + let v = vector_unsigned_long_long([0xFF00FF00FF00FF00; 2]); + let m = vector_unsigned_long_long([0x0000FF000000FF00; 2]); + assert_eq!(vec_test_mask(v, m), 3); + + let v = vector_unsigned_long_long([u64::MAX; 2]); + let m = vector_unsigned_long_long([0; 2]); + assert_eq!(vec_test_mask(v, m), 0); + + let v = vector_unsigned_long_long([0; 2]); + let m = vector_unsigned_long_long([u64::MAX; 2]); + assert_eq!(vec_test_mask(v, m), 0); + + let v = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA; 2]); + let m = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA; 2]); + assert_eq!(vec_test_mask(v, m), 3); + } + } + + #[simd_test(enable = "vector-enhancements-2")] + fn test_vec_search_string_cc() { + unsafe { + let b = vector_unsigned_char(*b"ABCD------------"); + let c = vector_unsigned_char([4; 16]); + let mut d = 0i32; + + let haystack = vector_unsigned_char(*b"__ABCD__________"); + let result = vec_search_string_cc(haystack, b, c, &mut d); + assert_eq!(result.as_array()[7], 2); + assert_eq!(d, 2); + + let haystack = vector_unsigned_char(*b"___ABCD_________"); + let result = vec_search_string_cc(haystack, b, c, &mut d); + assert_eq!(result.as_array()[7], 3); + assert_eq!(d, 2); + + let haystack = vector_unsigned_char(*b"________________"); + let result = vec_search_string_cc(haystack, b, c, &mut d); + assert_eq!(result.as_array()[7], 16); + assert_eq!(d, 0); + + let haystack = vector_unsigned_char(*b"______\0_________"); + let result = vec_search_string_cc(haystack, b, c, &mut d); + assert_eq!(result.as_array()[7], 16); + assert_eq!(d, 0); + + let haystack = vector_unsigned_char(*b"______\0__ABCD___"); + let result = vec_search_string_cc(haystack, b, c, &mut d); + assert_eq!(result.as_array()[7], 9); + assert_eq!(d, 2); + } + } + + #[simd_test(enable = "vector-enhancements-2")] + fn test_vec_search_string_until_zero_cc() { + unsafe { + let b = vector_unsigned_char(*b"ABCD\0\0\0\0\0\0\0\0\0\0\0\0"); + let c = vector_unsigned_char([16; 16]); + let mut d = 0i32; + + let haystack = vector_unsigned_char(*b"__ABCD__________"); + let result = vec_search_string_until_zero_cc(haystack, b, c, &mut d); + assert_eq!(result.as_array()[7], 2); + assert_eq!(d, 2); + + let haystack = vector_unsigned_char(*b"___ABCD_________"); + let result = vec_search_string_until_zero_cc(haystack, b, c, &mut d); + assert_eq!(result.as_array()[7], 3); + assert_eq!(d, 2); + + let haystack = vector_unsigned_char(*b"________________"); + let result = vec_search_string_until_zero_cc(haystack, b, c, &mut d); + assert_eq!(result.as_array()[7], 16); + assert_eq!(d, 0); + + let haystack = vector_unsigned_char(*b"______\0_________"); + let result = vec_search_string_until_zero_cc(haystack, b, c, &mut d); + assert_eq!(result.as_array()[7], 16); + assert_eq!(d, 1); + + let haystack = vector_unsigned_char(*b"______\0__ABCD___"); + let result = vec_search_string_until_zero_cc(haystack, b, c, &mut d); + assert_eq!(result.as_array()[7], 16); + assert_eq!(d, 1); + } + } + + #[simd_test(enable = "vector")] + fn test_vec_doublee() { + unsafe { + let v = vector_float([1.0, 2.0, 3.0, 4.0]); + assert_eq!(vec_doublee(v).as_array(), &[1.0, 3.0]); + + let v = vector_float([f32::NAN, 2.0, f32::INFINITY, 4.0]); + let d = vec_doublee(v); + assert!(d.as_array()[0].is_nan()); + assert_eq!(d.as_array()[1], f64::INFINITY); + } + } + + #[simd_test(enable = "vector")] + fn test_vec_floate() { + // NOTE: indices 1 and 3 can have an arbitrary value. With the C version + // these are poison values, our version initializes the memory but its + // value still should not be relied upon by application code. + unsafe { + let v = vector_double([1.0, 2.0]); + let d = vec_floate(v); + assert_eq!(d.as_array()[0], 1.0); + assert_eq!(d.as_array()[2], 2.0); + + let v = vector_double([f64::NAN, f64::INFINITY]); + let d = vec_floate(v); + assert!(d.as_array()[0].is_nan()); + assert_eq!(d.as_array()[2], f32::INFINITY); + + let v = vector_double([f64::MIN, f64::MAX]); + let d = vec_floate(v); + assert_eq!(d.as_array()[0], f64::MIN as f32); + assert_eq!(d.as_array()[2], f64::MAX as f32); + } + } + + #[simd_test(enable = "vector")] + fn test_vec_extend_s64() { + unsafe { + let v = vector_signed_char([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + assert_eq!(vec_extend_s64(v).as_array(), &[7, 15]); + + let v = vector_signed_short([0, 1, 2, 3, 4, 5, 6, 7]); + assert_eq!(vec_extend_s64(v).as_array(), &[3, 7]); + + let v = vector_signed_int([0, 1, 2, 3]); + assert_eq!(vec_extend_s64(v).as_array(), &[1, 3]); + } + } + + #[simd_test(enable = "vector")] + fn test_vec_signed() { + unsafe { + let v = vector_float([1.0, 2.5, -2.5, -0.0]); + assert_eq!(vec_signed(v).as_array(), &[1, 2, -2, 0]); + + let v = vector_double([2.5, -2.5]); + assert_eq!(vec_signed(v).as_array(), &[2, -2]); + } + } + + #[simd_test(enable = "vector")] + fn test_vec_unsigned() { + // NOTE: converting a negative floating point value is UB! + unsafe { + let v = vector_float([1.0, 2.5, 3.5, 0.0]); + assert_eq!(vec_unsigned(v).as_array(), &[1, 2, 3, 0]); + + let v = vector_double([2.5, 3.5]); + assert_eq!(vec_unsigned(v).as_array(), &[2, 3]); + } + } + + #[simd_test(enable = "vector")] + fn test_vec_cp_until_zero() { + unsafe { + let v = vector_signed_int([1, 2, 3, 4]); + let d = vec_cp_until_zero(v); + assert_eq!(d.as_array(), &[1, 2, 3, 4]); + + let v = vector_signed_int([1, 2, 0, 4]); + let d = vec_cp_until_zero(v); + assert_eq!(d.as_array(), &[1, 2, 0, 0]); + } + } + + #[simd_test(enable = "vector")] + fn test_vec_cp_until_zero_cc() { + let mut cc = 0; + unsafe { + let v = vector_signed_int([1, 2, 3, 4]); + let d = vec_cp_until_zero_cc(v, &mut cc); + assert_eq!(d.as_array(), &[1, 2, 3, 4]); + assert_eq!(cc, 3); + + let v = vector_signed_int([1, 2, 0, 4]); + let d = vec_cp_until_zero_cc(v, &mut cc); + assert_eq!(d.as_array(), &[1, 2, 0, 0]); + assert_eq!(cc, 0); + } + } + + #[simd_test(enable = "vector-enhancements-1")] + fn test_vec_msum_u128() { + let a = vector_unsigned_long_long([1, 2]); + let b = vector_unsigned_long_long([3, 4]); + + unsafe { + let c: vector_unsigned_char = transmute(100u128); + + let d: u128 = transmute(vec_msum_u128::<0>(a, b, c)); + assert_eq!(d, (1 * 3) + (2 * 4) + 100); + + let d: u128 = transmute(vec_msum_u128::<4>(a, b, c)); + assert_eq!(d, (1 * 3) + (2 * 4) * 2 + 100); + + let d: u128 = transmute(vec_msum_u128::<8>(a, b, c)); + assert_eq!(d, (1 * 3) * 2 + (2 * 4) + 100); + + let d: u128 = transmute(vec_msum_u128::<12>(a, b, c)); + assert_eq!(d, (1 * 3) * 2 + (2 * 4) * 2 + 100); + } + } + + #[simd_test(enable = "vector")] + fn test_vec_sld() { + let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]); + let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]); + + unsafe { + let d = vec_sld::<_, 4>(a, b); + assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAABBBBBBBB]); + } + } + + #[simd_test(enable = "vector")] + fn test_vec_sldw() { + let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]); + let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]); + + unsafe { + let d = vec_sldw::<_, 1>(a, b); + assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAABBBBBBBB]); + } + } + + #[simd_test(enable = "vector-enhancements-2")] + fn test_vec_sldb() { + let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]); + let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]); + + unsafe { + let d = vec_sldb::<_, 4>(a, b); + assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAB]); + } + } + + #[simd_test(enable = "vector-enhancements-2")] + fn test_vec_srdb() { + let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]); + let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]); + + unsafe { + let d = vec_srdb::<_, 4>(a, b); + assert_eq!(d.as_array(), &[0xABBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]); + } + } + + const GT: u32 = 0x20000000; + const LT: u32 = 0x40000000; + const EQ: u32 = 0x80000000; + + #[simd_test(enable = "vector")] + fn test_vec_cmprg() { + let a = vector_unsigned_int([11, 22, 33, 44]); + let b = vector_unsigned_int([10, 20, 30, 40]); + + let c = vector_unsigned_int([GT, LT, GT, LT]); + let d = unsafe { vec_cmprg(a, b, c) }; + assert_eq!(d.as_array(), &[!0, 0, !0, 0]); + + let c = vector_unsigned_int([GT, LT, 0, 0]); + let d = unsafe { vec_cmprg(a, b, c) }; + assert_eq!(d.as_array(), &[!0, 0, 0, 0]); + + let a = vector_unsigned_int([11, 22, 33, 30]); + let b = vector_unsigned_int([10, 20, 30, 30]); + + let c = vector_unsigned_int([GT, LT, EQ, EQ]); + let d = unsafe { vec_cmprg(a, b, c) }; + assert_eq!(d.as_array(), &[!0, 0, 0, !0]); + } + + #[simd_test(enable = "vector")] + fn test_vec_cmpnrg() { + let a = vector_unsigned_int([11, 22, 33, 44]); + let b = vector_unsigned_int([10, 20, 30, 40]); + + let c = vector_unsigned_int([GT, LT, GT, LT]); + let d = unsafe { vec_cmpnrg(a, b, c) }; + assert_eq!(d.as_array(), &[0, !0, 0, !0]); + + let c = vector_unsigned_int([GT, LT, 0, 0]); + let d = unsafe { vec_cmpnrg(a, b, c) }; + assert_eq!(d.as_array(), &[0, !0, !0, !0]); + + let a = vector_unsigned_int([11, 22, 33, 30]); + let b = vector_unsigned_int([10, 20, 30, 30]); + + let c = vector_unsigned_int([GT, LT, EQ, EQ]); + let d = unsafe { vec_cmpnrg(a, b, c) }; + assert_eq!(d.as_array(), &[0, !0, !0, 0]); + } + + #[simd_test(enable = "vector")] + fn test_vec_cmprg_idx() { + let a = vector_unsigned_int([1, 11, 22, 33]); + let b = vector_unsigned_int([10, 20, 30, 40]); + + let c = vector_unsigned_int([GT, LT, GT, LT]); + let d = unsafe { vec_cmprg_idx(a, b, c) }; + assert_eq!(d.as_array(), &[0, 4, 0, 0]); + } + + #[simd_test(enable = "vector")] + fn test_vec_cmpnrg_idx() { + let a = vector_unsigned_int([1, 11, 22, 33]); + let b = vector_unsigned_int([10, 20, 30, 40]); + + let c = vector_unsigned_int([GT, LT, GT, LT]); + let d = unsafe { vec_cmpnrg_idx(a, b, c) }; + assert_eq!(d.as_array(), &[0, 0, 0, 0]); + } + + #[simd_test(enable = "vector")] + fn test_vec_cmprg_or_0_idx() { + let a = vector_unsigned_int([1, 0, 22, 33]); + let b = vector_unsigned_int([10, 20, 30, 40]); + + let c = vector_unsigned_int([GT, LT, GT, LT]); + let d = unsafe { vec_cmprg_or_0_idx(a, b, c) }; + assert_eq!(d.as_array(), &[0, 4, 0, 0]); + } + + #[simd_test(enable = "vector")] + fn test_vec_cmpnrg_or_0_idx() { + let a = vector_unsigned_int([11, 33, 0, 22]); + let b = vector_unsigned_int([10, 20, 30, 40]); + + let c = vector_unsigned_int([GT, LT, GT, LT]); + let d = unsafe { vec_cmpnrg_or_0_idx(a, b, c) }; + assert_eq!(d.as_array(), &[0, 8, 0, 0]); + } + + test_vec_2! { test_vec_cmpgt, vec_cmpgt, f32x4, f32x4 -> i32x4, + [1.0, f32::NAN, f32::NAN, 3.14], + [2.0, f32::NAN, 5.0, 2.0], + [0, 0, 0, !0] + } + + test_vec_2! { test_vec_cmpge, vec_cmpge, f32x4, f32x4 -> i32x4, + [1.0, f32::NAN, f32::NAN, 3.14], + [1.0, f32::NAN, 5.0, 2.0], + [!0, 0, 0, !0] + } + + test_vec_2! { test_vec_cmplt, vec_cmplt, f32x4, f32x4 -> i32x4, + [1.0, f32::NAN, f32::NAN, 2.0], + [2.0, f32::NAN, 5.0, 2.0], + [!0, 0, 0, 0] + } + + test_vec_2! { test_vec_cmple, vec_cmple, f32x4, f32x4 -> i32x4, + [1.0, f32::NAN, f32::NAN, 2.0], + [1.0, f32::NAN, 5.0, 3.14], + [!0, 0, 0, !0] + } + + test_vec_2! { test_vec_cmpeq, vec_cmpeq, f32x4, f32x4 -> i32x4, + [1.0, f32::NAN, f32::NAN, 2.0], + [1.0, f32::NAN, 5.0, 3.14], + [!0, 0, 0, 0] + } + + test_vec_2! { test_vec_cmpne, vec_cmpne, f32x4, f32x4 -> i32x4, + [1.0, f32::NAN, f32::NAN, 2.0], + [1.0, f32::NAN, 5.0, 3.14], + [0, !0, !0, !0] + } + + #[simd_test(enable = "vector")] + fn test_vec_meadd() { + let a = vector_unsigned_short([1, 0, 2, 0, 3, 0, 4, 0]); + let b = vector_unsigned_short([5, 0, 6, 0, 7, 0, 8, 0]); + let c = vector_unsigned_int([2, 2, 2, 2]); + + let d = unsafe { vec_meadd(a, b, c) }; + assert_eq!(d.as_array(), &[7, 14, 23, 34]); + + let a = vector_signed_short([1, 0, 2, 0, 3, 0, 4, 0]); + let b = vector_signed_short([5, 0, 6, 0, 7, 0, 8, 0]); + let c = vector_signed_int([2, -2, 2, -2]); + + let d = unsafe { vec_meadd(a, b, c) }; + assert_eq!(d.as_array(), &[7, 10, 23, 30]); + } + + #[simd_test(enable = "vector")] + fn test_vec_moadd() { + let a = vector_unsigned_short([0, 1, 0, 2, 0, 3, 0, 4]); + let b = vector_unsigned_short([0, 5, 0, 6, 0, 7, 0, 8]); + let c = vector_unsigned_int([2, 2, 2, 2]); + + let d = unsafe { vec_moadd(a, b, c) }; + assert_eq!(d.as_array(), &[7, 14, 23, 34]); + + let a = vector_signed_short([0, 1, 0, 2, 0, 3, 0, 4]); + let b = vector_signed_short([0, 5, 0, 6, 0, 7, 0, 8]); + let c = vector_signed_int([2, -2, 2, -2]); + + let d = unsafe { vec_moadd(a, b, c) }; + assert_eq!(d.as_array(), &[7, 10, 23, 30]); + } + + #[simd_test(enable = "vector")] + fn test_vec_mhadd() { + let a = vector_unsigned_int([1, 2, 3, 4]); + let b = vector_unsigned_int([5, 6, 7, 8]); + let c = vector_unsigned_int([u32::MAX; 4]); + + let d = unsafe { vec_mhadd(a, b, c) }; + assert_eq!(d.as_array(), &[1, 1, 1, 1]); + + let a = vector_signed_int([-1, -2, -3, -4]); + let b = vector_signed_int([5, 6, 7, 8]); + let c = vector_signed_int([i32::MIN; 4]); + + let d = unsafe { vec_mhadd(a, b, c) }; + assert_eq!(d.as_array(), &[-1, -1, -1, -1]); + } + + #[simd_test(enable = "vector")] + fn test_vec_mladd() { + let a = vector_unsigned_int([1, 2, 3, 4]); + let b = vector_unsigned_int([5, 6, 7, 8]); + let c = vector_unsigned_int([2, 2, 2, 2]); + + let d = unsafe { vec_mladd(a, b, c) }; + assert_eq!(d.as_array(), &[7, 14, 23, 34]); + + let a = vector_signed_int([-1, -2, -3, -4]); + let b = vector_signed_int([5, 6, 7, 8]); + let c = vector_signed_int([2, 2, 2, 2]); + + let d = unsafe { vec_mladd(a, b, c) }; + assert_eq!(d.as_array(), &[-3, -10, -19, -30]); + } }