carry out (c)transpose in trivial cases for Ax_mul_Bx methods for Hermitian and Symmetric (#22396)

- Symmetric and Hermitian{<:Real} are invariant to transpose so it is a no-op
- Hermitian and Symmetric{<:Real} are invariant to ctranspose so it is a no-op

Author: fredrikekre

base/linalg/diagonal.jl

@@ -234,6 +234,12 @@ A_mul_B!(out::AbstractMatrix, A::Diagonal, in::AbstractMatrix) = out .= A.diag .
 Ac_mul_B!(out::AbstractMatrix, A::Diagonal, in::AbstractMatrix) = out .= ctranspose.(A.diag) .* in
 At_mul_B!(out::AbstractMatrix, A::Diagonal, in::AbstractMatrix) = out .= transpose.(A.diag) .* in
 
+# ambiguities with Symmetric/Hermitian
+# RealHermSymComplex[Sym]/[Herm] only include Number; invariant to [c]transpose
+A_mul_Bt(A::Diagonal, B::RealHermSymComplexSym) = A*B
+At_mul_B(A::RealHermSymComplexSym, B::Diagonal) = A*B
+A_mul_Bc(A::Diagonal, B::RealHermSymComplexHerm) = A*B
+Ac_mul_B(A::RealHermSymComplexHerm, B::Diagonal) = A*B
 
 (/)(Da::Diagonal, Db::Diagonal) = Diagonal(Da.diag ./ Db.diag)
 function A_ldiv_B!(D::Diagonal{T}, v::AbstractVector{T}) where {T}

base/linalg/symmetric.jl

@@ -109,6 +109,7 @@ end
 
 const HermOrSym{T,S} = Union{Hermitian{T,S}, Symmetric{T,S}}
 const RealHermSymComplexHerm{T<:Real,S} = Union{Hermitian{T,S}, Symmetric{T,S}, Hermitian{Complex{T},S}}
+const RealHermSymComplexSym{T<:Real,S} = Union{Hermitian{T,S}, Symmetric{T,S}, Symmetric{Complex{T},S}}
 
 size(A::HermOrSym, d) = size(A.data, d)
 size(A::HermOrSym) = size(A.data)
@@ -303,6 +304,25 @@ A_mul_B!(C::StridedMatrix{T}, A::StridedMatrix{T}, B::Hermitian{T,<:StridedMatri
 
 *(A::HermOrSym, B::HermOrSym) = A*full(B)
 
+# Fallbacks to avoid generic_matvecmul!/generic_matmatmul!
+## Symmetric{<:Number} and Hermitian{<:Real} are invariant to transpose; peel off the t
+At_mul_B(A::RealHermSymComplexSym, B::AbstractVector) = A*B
+At_mul_B(A::RealHermSymComplexSym, B::AbstractMatrix) = A*B
+A_mul_Bt(A::AbstractMatrix, B::RealHermSymComplexSym) = A*B
+## Hermitian{<:Number} and Symmetric{<:Real} are invariant to ctranspose; peel off the c
+Ac_mul_B(A::RealHermSymComplexHerm, B::AbstractVector) = A*B
+Ac_mul_B(A::RealHermSymComplexHerm, B::AbstractMatrix) = A*B
+A_mul_Bc(A::AbstractMatrix, B::RealHermSymComplexHerm) = A*B
+
+# ambiguities with RowVector
+A_mul_Bt(A::RowVector, B::RealHermSymComplexSym) = A*B
+A_mul_Bc(A::RowVector, B::RealHermSymComplexHerm) = A*B
+# ambiguities with AbstractTriangular
+At_mul_B(A::RealHermSymComplexSym, B::AbstractTriangular) = A*B
+A_mul_Bt(A::AbstractTriangular, B::RealHermSymComplexSym) = A*B
+Ac_mul_B(A::RealHermSymComplexHerm, B::AbstractTriangular) = A*B
+A_mul_Bc(A::AbstractTriangular, B::RealHermSymComplexHerm) = A*B
+
 for T in (:Symmetric, :Hermitian), op in (:+, :-, :*, :/)
     # Deal with an ambiguous case
     @eval ($op)(A::$T, x::Bool) = ($T)(($op)(A.data, x), Symbol(A.uplo))