From 7a6fa699f2aad57a5b4772fc97b2fc60abc8526e Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Tue, 20 Feb 2024 10:50:43 +0000 Subject: [PATCH 001/267] Small GEMM for AArch64 This is a fairly conservative addition of small matrix kernels using SVE. --- Makefile.system | 2 + kernel/arm64/KERNEL.ARMV8SVE | 20 + kernel/arm64/dgemm_small_kernel_nn_sve.c | 657 ++++++++++ kernel/arm64/dgemm_small_kernel_nt_sve.c | 414 ++++++ kernel/arm64/dgemm_small_kernel_tn_sve.c | 709 +++++++++++ kernel/arm64/dgemm_small_kernel_tt_sve.c | 482 +++++++ kernel/arm64/gemm_small_kernel_permit_sve.c | 47 + kernel/arm64/sgemm_small_kernel_nn_sve.c | 1046 ++++++++++++++++ kernel/arm64/sgemm_small_kernel_nt_sve.c | 647 ++++++++++ kernel/arm64/sgemm_small_kernel_tn_sve.c | 1247 +++++++++++++++++++ kernel/arm64/sgemm_small_kernel_tt_sve.c | 574 +++++++++ 11 files changed, 5845 insertions(+) create mode 100644 kernel/arm64/dgemm_small_kernel_nn_sve.c create mode 100644 kernel/arm64/dgemm_small_kernel_nt_sve.c create mode 100644 kernel/arm64/dgemm_small_kernel_tn_sve.c create mode 100644 kernel/arm64/dgemm_small_kernel_tt_sve.c create mode 100644 kernel/arm64/gemm_small_kernel_permit_sve.c create mode 100644 kernel/arm64/sgemm_small_kernel_nn_sve.c create mode 100644 kernel/arm64/sgemm_small_kernel_nt_sve.c create mode 100644 kernel/arm64/sgemm_small_kernel_tn_sve.c create mode 100644 kernel/arm64/sgemm_small_kernel_tt_sve.c diff --git a/Makefile.system b/Makefile.system index aadf3459a1..e95d1529a8 100644 --- a/Makefile.system +++ b/Makefile.system @@ -268,6 +268,8 @@ SMALL_MATRIX_OPT = 1 else ifeq ($(ARCH), power) SMALL_MATRIX_OPT = 1 BUILD_BFLOAT16 = 1 +else ifeq ($(ARCH), arm64) +SMALL_MATRIX_OPT = 1 endif ifeq ($(SMALL_MATRIX_OPT), 1) CCOMMON_OPT += -DSMALL_MATRIX_OPT diff --git a/kernel/arm64/KERNEL.ARMV8SVE b/kernel/arm64/KERNEL.ARMV8SVE index eeb4844bf2..bfadf5cba9 100644 --- a/kernel/arm64/KERNEL.ARMV8SVE +++ b/kernel/arm64/KERNEL.ARMV8SVE @@ -131,6 +131,16 @@ SGEMMITCOPYOBJ = sgemm_itcopy$(TSUFFIX).$(SUFFIX) SGEMMONCOPYOBJ = sgemm_oncopy$(TSUFFIX).$(SUFFIX) SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX) +SGEMM_SMALL_M_PERMIT = gemm_small_kernel_permit_sve.c +SGEMM_SMALL_K_NT = sgemm_small_kernel_nt_sve.c +SGEMM_SMALL_K_B0_NT = sgemm_small_kernel_nt_sve.c +SGEMM_SMALL_K_NN = sgemm_small_kernel_nn_sve.c +SGEMM_SMALL_K_B0_NN = sgemm_small_kernel_nn_sve.c +SGEMM_SMALL_K_TT = sgemm_small_kernel_tt_sve.c +SGEMM_SMALL_K_B0_TT = sgemm_small_kernel_tt_sve.c +SGEMM_SMALL_K_TN = sgemm_small_kernel_tn_sve.c +SGEMM_SMALL_K_B0_TN = sgemm_small_kernel_tn_sve.c + STRMMUNCOPY_M = trmm_uncopy_sve_v1.c STRMMLNCOPY_M = trmm_lncopy_sve_v1.c STRMMUTCOPY_M = trmm_utcopy_sve_v1.c @@ -152,6 +162,16 @@ DGEMMITCOPYOBJ = dgemm_itcopy$(TSUFFIX).$(SUFFIX) DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX) DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX) +DGEMM_SMALL_M_PERMIT = gemm_small_kernel_permit_sve.c +DGEMM_SMALL_K_NT = dgemm_small_kernel_nt_sve.c +DGEMM_SMALL_K_B0_NT = dgemm_small_kernel_nt_sve.c +DGEMM_SMALL_K_NN = dgemm_small_kernel_nn_sve.c +DGEMM_SMALL_K_B0_NN = dgemm_small_kernel_nn_sve.c +DGEMM_SMALL_K_TT = dgemm_small_kernel_tt_sve.c +DGEMM_SMALL_K_B0_TT = dgemm_small_kernel_tt_sve.c +DGEMM_SMALL_K_TN = dgemm_small_kernel_tn_sve.c +DGEMM_SMALL_K_B0_TN = dgemm_small_kernel_tn_sve.c + DTRMMUNCOPY_M = trmm_uncopy_sve_v1.c DTRMMLNCOPY_M = trmm_lncopy_sve_v1.c DTRMMUTCOPY_M = trmm_utcopy_sve_v1.c diff --git a/kernel/arm64/dgemm_small_kernel_nn_sve.c b/kernel/arm64/dgemm_small_kernel_nn_sve.c new file mode 100644 index 0000000000..8baef82775 --- /dev/null +++ b/kernel/arm64/dgemm_small_kernel_nn_sve.c @@ -0,0 +1,657 @@ +/*************************************************************************** +Copyright (c) 2024, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "common.h" + +#include +#include +#ifdef __ARM_NEON_SVE_BRIDGE +#include +#else +#define svdup_neonq_f32(fixed_reg) \ + ({ \ + svfloat32_t scalable_reg; \ + asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \ + scalable_reg; \ + }) +#define svdup_neonq_f64(fixed_reg) \ + ({ \ + svfloat64_t scalable_reg; \ + asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \ + scalable_reg; \ + }) +#endif + +#define A_ELEMENT_K(m, offset_k) A[(i + (m)) + (k + offset_k) * lda] +#define A_ELEMENT(m) A_ELEMENT_K(m, 0) + +#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) + (j + (n)) * ldb] +#define B_ELEMENT(n) B_ELEMENT_K(n, 0) + +#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc] + +#define PACK_ELEMENT_K(n, offset_k) packed_b[(k + offset_k) * 4 + n] +#define PACK_ELEMENT(n) PACK_ELEMENT_K(n, 0) + +// ASIMD +#define DECLARE_RESULT_VECTOR2(m, n) \ + float64x2_t result##m##n = vdupq_n_f64(0.0); +#define DECLARE_RESULT(m, n) float64_t result##m##n = 0.0; +#define BROADCAST_LOAD_A2(m, offset_k) \ + float64x2_t a##m##_k##offset_k = vld1q_dup_f64(&A_ELEMENT_K(m, offset_k)); +#define LOAD_A1(m, offset_k) \ + float64_t a##m##_k##offset_k = A_ELEMENT_K(m, offset_k); +#define VECTOR_LOAD_B_K2(n, offset_k) \ + float64x2_t b##k##n##_k##offset_k = vld1q_f64(&B_ELEMENT_K(n, offset_k)); +#define TRANSPOSE_B2_K2(n0, n1, offset_k0, offset_k1) \ + float64x2_t b##n0##_k##offset_k0 = \ + vzip1q_f64(b##k##n0##_k##offset_k0, b##k##n1##_k##offset_k0); \ + float64x2_t b##n0##_k##offset_k1 = \ + vzip2q_f64(b##k##n0##_k##offset_k0, b##k##n1##_k##offset_k0); + +#define SCALE_B2_K2(n0, offset_k0, offset_k1) \ + svfloat64_t b##s##n0##_k##offset_k0 = svdup_neonq_f64(b##n0##_k##offset_k0); \ + svfloat64_t b##s##n0##_k##offset_k1 = svdup_neonq_f64(b##n0##_k##offset_k1); +#define GATHER_LOAD_B2(n, offset_k) \ + float64x2_t b##n##_k##offset_k = vdupq_n_f64(B_ELEMENT_K(n, offset_k)); \ + b##n##_k##offset_k = \ + vsetq_lane_f64(B_ELEMENT_K(n + 1, offset_k), b##n##_k##offset_k, 1); +#define VECTOR_UNPACK_B2(n, offset_k) \ + float64x2_t b##n##_k##offset_k = vld1q_f64(&PACK_ELEMENT_K(n, offset_k)); +#define VECTOR_PACK_B2(n, offset_k) \ + vst1q_f64(&PACK_ELEMENT_K(n, offset_k), b##n##_k##offset_k); +#define PACK_B0(n, offset_k) \ + PACK_ELEMENT_K(n, offset_k) = vget_lane_f64(b##n##_k##offset_k, 0); +#define UPDATE_RESULT_VECTOR2(m, n, offset_k) \ + result##m##n = \ + vfmaq_f64(result##m##n, a##m##_k##offset_k, b##n##_k##offset_k); +#define UPDATE_RESULT(m, n, offset_k) \ + result##m##n = result##m##n + a##m##_k##offset_k * b##n##_k##offset_k; +#ifdef B0 +#define SCATTER_STORE2(m, n) \ + result##m##n = vmulq_f64(result##m##n, vdupq_n_f64(alpha)); \ + C_ELEMENT(m, n + 0) = vgetq_lane_f64(result##m##n, 0); \ + C_ELEMENT(m, n + 1) = vgetq_lane_f64(result##m##n, 1); +#else +#define SCATTER_STORE2(m, n) \ + result##m##n = vmulq_f64(result##m##n, vdupq_n_f64(alpha)); \ + C_ELEMENT(m, n + 0) = \ + C_ELEMENT(m, n + 0) * beta + vgetq_lane_f64(result##m##n, 0); \ + C_ELEMENT(m, n + 1) = \ + C_ELEMENT(m, n + 1) * beta + vgetq_lane_f64(result##m##n, 1); +#endif + +// SVE +#define DECLARE_RESULT_VECTOR(m, n) svfloat64_t result##m##n = svdup_f64(0.0); +#define BROADCAST_LOAD_A(m, offset_k) \ + svfloat64_t a##s##m##_k##offset_k = svdup_f64(A_ELEMENT_K(m, offset_k)); +#define BROADCAST_LOAD_B(n, offset_k) \ + svfloat64_t b##s##n##_k##offset_k = svdup_f64(B_ELEMENT_K(n, offset_k)); +#define VECTOR_LOAD_A(pg, m, offset_k) \ + svfloat64_t a##s##m##_k##offset_k = \ + svld1(pg, &A_ELEMENT_K(v_size * m, offset_k)); +#define QUADWORD_LOAD_B(n, offset_k) \ + svfloat64_t b##s##n##_k##offset_k = \ + svld1rq(pg_true, &B_ELEMENT_K(n, offset_k)); +#define PACK_B(n, offset_k) \ + svst1(pg_first, &PACK_ELEMENT_K(n, offset_k), b##s##n##_k##offset_k); +#define VECTOR_PACK_B(n, offset_k) \ + svst1(pg_true, &PACK_ELEMENT_K(n* v_size, offset_k), b##s##n##_k##offset_k); +#define QUADWORD_PACK_B(n, offset_k) \ + svst1(pg_quad, &PACK_ELEMENT_K(n, offset_k), b##s##n##_k##offset_k); +#define UNPACK_VECTOR_B(n, offset_k) \ + svfloat64_t b##s##n##_k##offset_k = \ + svld1(pg_true, &PACK_ELEMENT_K(n * v_size, offset_k)); +#define UNPACK_BROADCAST_B(n, offset_k) \ + svfloat64_t b##s##n##_k##offset_k = svdup_f64(PACK_ELEMENT_K(n, offset_k)); +#define UNPACK_QUADWORD_B(n, offset_k) \ + svfloat64_t b##s##n##_k##offset_k = \ + svld1rq(pg_true, &PACK_ELEMENT_K(n, offset_k)); +#define UPDATE_RESULT_VECTOR(pg, m, n, offset_k) \ + result##m##n = \ + svmla_m(pg, result##m##n, a##s##m##_k##offset_k, b##s##n##_k##offset_k); +#define UPDATE_RESULT_VECTOR_QUADWORD(m, n, outer, lane, offset_k) \ + result##m##n = svmla_lane( \ + result##m##n, a##s##m##_k##offset_k, b##s##outer##_k##offset_k, lane); +#ifdef B0 +#define VECTOR_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); +#define SCATTER_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + svst1_scatter_index( \ + pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n); +#else +#define VECTOR_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + result##m##n = \ + svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(v_size * m, n)), beta_vec); \ + svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); +#define SCATTER_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + result##m##n = svmla_m( \ + pg, \ + result##m##n, \ + svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u64(0LL, ldc)), \ + beta_vec); \ + svst1_scatter_index( \ + pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n); +#endif + +#ifndef LIKELY +#ifdef __GNUC__ +#define LIKELY(x) __builtin_expect(!!(x), 1) +#else +#define LIKELY(x) (x) +#endif +#endif +#ifndef UNLIKELY +#ifdef __GNUC__ +#define UNLIKELY(x) __builtin_expect(!!(x), 0) +#else +#define UNLIKELY(x) (x) +#endif +#endif + +#ifdef B0 +int +CNAME(BLASLONG M, + BLASLONG N, + BLASLONG K, + IFLOAT* A, + BLASLONG lda, + FLOAT alpha, + IFLOAT* B, + BLASLONG ldb, + FLOAT* C, + BLASLONG ldc) +#else +int +CNAME(BLASLONG M, + BLASLONG N, + BLASLONG K, + IFLOAT* A, + BLASLONG lda, + FLOAT alpha, + IFLOAT* B, + BLASLONG ldb, + FLOAT beta, + FLOAT* C, + BLASLONG ldc) +#endif +{ + const uint64_t v_size = svcntd(); + const uint64_t v_size2 = v_size * 2; + const svbool_t pg_true = svptrue_b64(); + const svbool_t pg_quad = svwhilelt_b64(0, 2); + const svbool_t pg_first = svwhilelt_b64(0, 1); + const svfloat64_t alpha_vec = svdup_f64(alpha); +#ifndef B0 + const svfloat64_t beta_vec = svdup_f64(beta); +#endif + const BLASLONG n4 = N & -4; + const BLASLONG n2 = N & -2; + const BLASLONG v_m2 = M & -v_size2; + const BLASLONG v_m1 = M & -v_size; + const BLASLONG k2 = K & -2; + + const int pack_b = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0; + FLOAT* packed_b = + (pack_b) ? packed_b = (FLOAT*)malloc(K * 4 * sizeof(FLOAT)) : NULL; + + BLASLONG j = 0; + for (; j < n4; j += 4) { + + BLASLONG i = 0; + for (; i < v_m2; i += v_size2) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(1, 1); + DECLARE_RESULT_VECTOR(1, 2); + DECLARE_RESULT_VECTOR(1, 3); + + if (LIKELY(packed_b != NULL)) { + if (i == 0) { + for (; k < k2; k += 2) { + + VECTOR_LOAD_B_K2(0, 0); + VECTOR_LOAD_B_K2(1, 0); + TRANSPOSE_B2_K2(0, 1, 0, 1); + SCALE_B2_K2(0, 0, 1); + VECTOR_PACK_B2(0, 0); + VECTOR_PACK_B2(0, 1); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + VECTOR_LOAD_A(pg_true, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + VECTOR_LOAD_B_K2(2, 0); + VECTOR_LOAD_B_K2(3, 0); + TRANSPOSE_B2_K2(2, 3, 0, 1); + SCALE_B2_K2(2, 0, 1); + VECTOR_PACK_B2(2, 0); + VECTOR_PACK_B2(2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 1); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0); + VECTOR_LOAD_A(pg_true, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 1); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + PACK_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + PACK_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); + BROADCAST_LOAD_B(2, 0); + PACK_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); + BROADCAST_LOAD_B(3, 0); + PACK_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); + } + } else { + for (; k < K; k++) { + + UNPACK_QUADWORD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UNPACK_QUADWORD_B(2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0); + } + } + } else { + for (; k < k2; k += 2) { + + VECTOR_LOAD_B_K2(0, 0); + VECTOR_LOAD_B_K2(1, 0); + TRANSPOSE_B2_K2(0, 1, 0, 1); + SCALE_B2_K2(0, 0, 1); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + VECTOR_LOAD_A(pg_true, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + VECTOR_LOAD_B_K2(2, 0); + VECTOR_LOAD_B_K2(3, 0); + TRANSPOSE_B2_K2(2, 3, 0, 1); + SCALE_B2_K2(2, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 1); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0); + VECTOR_LOAD_A(pg_true, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 1); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); + } + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + VECTOR_STORE(pg_true, 1, 0); + VECTOR_STORE(pg_true, 1, 1); + VECTOR_STORE(pg_true, 1, 2); + VECTOR_STORE(pg_true, 1, 3); + } + for (; i < v_m1; i += v_size) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + + if (LIKELY(packed_b != NULL)) { + for (; k < K; k++) { + + UNPACK_QUADWORD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UNPACK_QUADWORD_B(2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); + } + } else { + for (; k < k2; k += 2) { + + VECTOR_LOAD_B_K2(0, 0); + VECTOR_LOAD_B_K2(1, 0); + TRANSPOSE_B2_K2(0, 1, 0, 1); + SCALE_B2_K2(0, 0, 1); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + VECTOR_LOAD_A(pg_true, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + VECTOR_LOAD_B_K2(2, 0); + VECTOR_LOAD_B_K2(3, 0); + TRANSPOSE_B2_K2(2, 3, 0, 1); + SCALE_B2_K2(2, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 1); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + } + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + } + for (; i < M; i += v_size) { + const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M)); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + + if (LIKELY(packed_b != NULL)) { + for (; k < K; k++) { + + UNPACK_QUADWORD_B(0, 0); + VECTOR_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UNPACK_QUADWORD_B(2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); + } + } else { + for (; k < k2; k += 2) { + + VECTOR_LOAD_B_K2(0, 0); + VECTOR_LOAD_B_K2(1, 0); + TRANSPOSE_B2_K2(0, 1, 0, 1); + SCALE_B2_K2(0, 0, 1); + VECTOR_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + VECTOR_LOAD_A(pg_tail, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + VECTOR_LOAD_B_K2(2, 0); + VECTOR_LOAD_B_K2(3, 0); + TRANSPOSE_B2_K2(2, 3, 0, 1); + SCALE_B2_K2(2, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 1); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 3, 0); + } + } + VECTOR_STORE(pg_tail, 0, 0); + VECTOR_STORE(pg_tail, 0, 1); + VECTOR_STORE(pg_tail, 0, 2); + VECTOR_STORE(pg_tail, 0, 3); + } + } + for (; j < n2; j += 2) { + + BLASLONG i = 0; + for (; i < v_m2; i += v_size2) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(1, 1); + + for (; k < k2; k += 2) { + + VECTOR_LOAD_B_K2(0, 0); + VECTOR_LOAD_B_K2(1, 0); + TRANSPOSE_B2_K2(0, 1, 0, 1); + SCALE_B2_K2(0, 0, 1); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + VECTOR_LOAD_A(pg_true, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + VECTOR_LOAD_A(pg_true, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 1, 0); + VECTOR_STORE(pg_true, 1, 1); + } + for (; i < v_m1; i += v_size) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + + for (; k < k2; k += 2) { + + VECTOR_LOAD_B_K2(0, 0); + VECTOR_LOAD_B_K2(1, 0); + TRANSPOSE_B2_K2(0, 1, 0, 1); + SCALE_B2_K2(0, 0, 1); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + VECTOR_LOAD_A(pg_true, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + } + for (; i < M; i += v_size) { + const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M)); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + + for (; k < k2; k += 2) { + + VECTOR_LOAD_B_K2(0, 0); + VECTOR_LOAD_B_K2(1, 0); + TRANSPOSE_B2_K2(0, 1, 0, 1); + SCALE_B2_K2(0, 0, 1); + VECTOR_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + VECTOR_LOAD_A(pg_tail, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 1, 0); + } + VECTOR_STORE(pg_tail, 0, 0); + VECTOR_STORE(pg_tail, 0, 1); + } + } + for (; j < N; j++) { + + BLASLONG i = 0; + for (; i < v_m2; i += v_size2) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(1, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 1, 0); + } + for (; i < v_m1; i += v_size) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + } + VECTOR_STORE(pg_true, 0, 0); + } + for (; i < M; i += v_size) { + const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M)); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); + } + VECTOR_STORE(pg_tail, 0, 0); + } + } + + if (pack_b) + free(packed_b); + + return 0; +} diff --git a/kernel/arm64/dgemm_small_kernel_nt_sve.c b/kernel/arm64/dgemm_small_kernel_nt_sve.c new file mode 100644 index 0000000000..982388287f --- /dev/null +++ b/kernel/arm64/dgemm_small_kernel_nt_sve.c @@ -0,0 +1,414 @@ +/*************************************************************************** +Copyright (c) 2024, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "common.h" + +#include +#include +#ifdef __ARM_NEON_SVE_BRIDGE +#include +#else +#define svdup_neonq_f32(fixed_reg) \ + ({ \ + svfloat32_t scalable_reg; \ + asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \ + scalable_reg; \ + }) +#define svdup_neonq_f64(fixed_reg) \ + ({ \ + svfloat64_t scalable_reg; \ + asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \ + scalable_reg; \ + }) +#endif + +#define A_ELEMENT_K(m, offset_k) A[(i + (m)) + (k + offset_k) * lda] +#define A_ELEMENT(m) A_ELEMENT_K(m, 0) + +#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) * ldb + (j + (n))] +#define B_ELEMENT(n) B_ELEMENT_K(n, 0) + +#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc] + +#define PACK_ELEMENT_K(n, offset_k) packed_b[(k + offset_k) * 4 + n] +#define PACK_ELEMENT(n) PACK_ELEMENT_K(n, 0) + +// ASIMD +#define DECLARE_RESULT_VECTOR2(m, n) \ + float64x2_t result##m##n = vdupq_n_f64(0.0); +#define DECLARE_RESULT(m, n) float64_t result##m##n = 0.0; +#define BROADCAST_LOAD_A2(m, offset_k) \ + float64x2_t a##m##_k##offset_k = vld1q_dup_f64(&A_ELEMENT_K(m, offset_k)); +#define LOAD_A1(m, offset_k) \ + float64_t a##m##_k##offset_k = A_ELEMENT_K(m, offset_k); +#define VECTOR_LOAD_B2(n, offset_k) \ + float64x2_t b##n##_k##offset_k = vld1q_f64(&B_ELEMENT_K(n, offset_k)); +#define GATHER_LOAD_B2(n, offset_k) \ + float64x2_t b##n##_k##offset_k = vdupq_n_f64(B_ELEMENT_K(n, offset_k)); \ + b##n##_k##offset_k = \ + vsetq_lane_f64(B_ELEMENT_K(n + 1, offset_k), b##n##_k##offset_k, 1); +#define UPDATE_RESULT_VECTOR2(m, n, offset_k) \ + result##m##n = \ + vfmaq_f64(result##m##n, a##m##_k##offset_k, b##n##_k##offset_k); +#define UPDATE_RESULT(m, n, offset_k) \ + result##m##n = result##m##n + a##m##_k##offset_k * b##n##_k##offset_k; +#ifdef B0 +#define SCATTER_STORE2(m, n) \ + result##m##n = vmulq_f64(result##m##n, vdupq_n_f64(alpha)); \ + C_ELEMENT(m, n + 0) = vgetq_lane_f64(result##m##n, 0); \ + C_ELEMENT(m, n + 1) = vgetq_lane_f64(result##m##n, 1); +#else +#define SCATTER_STORE2(m, n) \ + result##m##n = vmulq_f64(result##m##n, vdupq_n_f64(alpha)); \ + C_ELEMENT(m, n + 0) = \ + C_ELEMENT(m, n + 0) * beta + vgetq_lane_f64(result##m##n, 0); \ + C_ELEMENT(m, n + 1) = \ + C_ELEMENT(m, n + 1) * beta + vgetq_lane_f64(result##m##n, 1); +#endif + +// SVE +#define DECLARE_RESULT_VECTOR(m, n) svfloat64_t result##m##n = svdup_f64(0.0); +#define BROADCAST_LOAD_A(m, offset_k) \ + svfloat64_t a##s##m##_k##offset_k = svdup_f64(A_ELEMENT_K(m, offset_k)); +#define BROADCAST_LOAD_B(n, offset_k) \ + svfloat64_t b##s##n##_k##offset_k = svdup_f64(B_ELEMENT_K(n, offset_k)); +#define VECTOR_LOAD_A(pg, m, offset_k) \ + svfloat64_t a##s##m##_k##offset_k = \ + svld1(pg, &A_ELEMENT_K(v_size * m, offset_k)); +#define QUADWORD_LOAD_B(n, offset_k) \ + svfloat64_t b##s##n##_k##offset_k = \ + svld1rq(pg_true, &B_ELEMENT_K(n, offset_k)); +#define UPDATE_RESULT_VECTOR(pg, m, n, offset_k) \ + result##m##n = \ + svmla_m(pg, result##m##n, a##s##m##_k##offset_k, b##s##n##_k##offset_k); +#define UPDATE_RESULT_VECTOR_QUADWORD(m, n, outer, lane, offset_k) \ + result##m##n = svmla_lane( \ + result##m##n, a##s##m##_k##offset_k, b##s##outer##_k##offset_k, lane); +#ifdef B0 +#define VECTOR_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); +#define SCATTER_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + svst1_scatter_index( \ + pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n); +#else +#define VECTOR_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + result##m##n = \ + svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(v_size * m, n)), beta_vec); \ + svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); +#define SCATTER_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + result##m##n = svmla_m( \ + pg, \ + result##m##n, \ + svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u64(0LL, ldc)), \ + beta_vec); \ + svst1_scatter_index( \ + pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n); +#endif + +#ifndef LIKELY +#ifdef __GNUC__ +#define LIKELY(x) __builtin_expect(!!(x), 1) +#else +#define LIKELY(x) (x) +#endif +#endif +#ifndef UNLIKELY +#ifdef __GNUC__ +#define UNLIKELY(x) __builtin_expect(!!(x), 0) +#else +#define UNLIKELY(x) (x) +#endif +#endif + +#ifdef B0 +int +CNAME(BLASLONG M, + BLASLONG N, + BLASLONG K, + IFLOAT* A, + BLASLONG lda, + FLOAT alpha, + IFLOAT* B, + BLASLONG ldb, + FLOAT* C, + BLASLONG ldc) +#else +int +CNAME(BLASLONG M, + BLASLONG N, + BLASLONG K, + IFLOAT* A, + BLASLONG lda, + FLOAT alpha, + IFLOAT* B, + BLASLONG ldb, + FLOAT beta, + FLOAT* C, + BLASLONG ldc) +#endif +{ + const uint64_t v_size = svcntd(); + const uint64_t v_size32 = v_size * 32; + const uint64_t v_size3 = v_size * 3; + const svbool_t pg_true = svptrue_b64(); + const svbool_t pg_quad = svwhilelt_b64(0, 2); + const svfloat64_t alpha_vec = svdup_f64(alpha); +#ifndef B0 + const svfloat64_t beta_vec = svdup_f64(beta); +#endif + const BLASLONG n4 = N & -4; + const BLASLONG n2 = N & -2; + const BLASLONG v_m3 = M - (M % v_size3); + const BLASLONG v_m1 = M & -v_size; + + BLASLONG j = 0; + for (; j < n4; j += 4) { + + BLASLONG i = 0; + for (; i < v_m3; i += v_size3) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(1, 1); + DECLARE_RESULT_VECTOR(1, 2); + DECLARE_RESULT_VECTOR(1, 3); + DECLARE_RESULT_VECTOR(2, 0); + DECLARE_RESULT_VECTOR(2, 1); + DECLARE_RESULT_VECTOR(2, 2); + DECLARE_RESULT_VECTOR(2, 3); + + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + QUADWORD_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0); + VECTOR_LOAD_A(pg_true, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 3, 2, 1, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + VECTOR_STORE(pg_true, 1, 0); + VECTOR_STORE(pg_true, 1, 1); + VECTOR_STORE(pg_true, 1, 2); + VECTOR_STORE(pg_true, 1, 3); + VECTOR_STORE(pg_true, 2, 0); + VECTOR_STORE(pg_true, 2, 1); + VECTOR_STORE(pg_true, 2, 2); + VECTOR_STORE(pg_true, 2, 3); + } + for (; i < v_m1; i += v_size) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + QUADWORD_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + } + for (; i < M; i += v_size) { + const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M)); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + QUADWORD_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); + } + VECTOR_STORE(pg_tail, 0, 0); + VECTOR_STORE(pg_tail, 0, 1); + VECTOR_STORE(pg_tail, 0, 2); + VECTOR_STORE(pg_tail, 0, 3); + } + } + for (; j < n2; j += 2) { + + BLASLONG i = 0; + for (; i < v_m3; i += v_size3) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(1, 1); + DECLARE_RESULT_VECTOR(2, 0); + DECLARE_RESULT_VECTOR(2, 1); + + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + VECTOR_LOAD_A(pg_true, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 1, 0, 1, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 1, 0); + VECTOR_STORE(pg_true, 1, 1); + VECTOR_STORE(pg_true, 2, 0); + VECTOR_STORE(pg_true, 2, 1); + } + for (; i < v_m1; i += v_size) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + } + for (; i < M; i += v_size) { + const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M)); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + } + VECTOR_STORE(pg_tail, 0, 0); + VECTOR_STORE(pg_tail, 0, 1); + } + } + for (; j < N; j++) { + + BLASLONG i = 0; + for (; i < v_m3; i += v_size3) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(2, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + VECTOR_LOAD_A(pg_true, 2, 0); + UPDATE_RESULT_VECTOR(pg_true, 2, 0, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 1, 0); + VECTOR_STORE(pg_true, 2, 0); + } + for (; i < v_m1; i += v_size) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + } + VECTOR_STORE(pg_true, 0, 0); + } + for (; i < M; i += v_size) { + const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M)); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); + } + VECTOR_STORE(pg_tail, 0, 0); + } + } + + return 0; +} diff --git a/kernel/arm64/dgemm_small_kernel_tn_sve.c b/kernel/arm64/dgemm_small_kernel_tn_sve.c new file mode 100644 index 0000000000..7158851da3 --- /dev/null +++ b/kernel/arm64/dgemm_small_kernel_tn_sve.c @@ -0,0 +1,709 @@ +/*************************************************************************** +Copyright (c) 2024, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "common.h" + +#include +#include +#ifdef __ARM_NEON_SVE_BRIDGE +#include +#else +#define svdup_neonq_f32(fixed_reg) \ + ({ \ + svfloat32_t scalable_reg; \ + asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \ + scalable_reg; \ + }) +#define svdup_neonq_f64(fixed_reg) \ + ({ \ + svfloat64_t scalable_reg; \ + asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \ + scalable_reg; \ + }) +#endif + +#define A_ELEMENT_K(m, offset_k) A[(i + (m)) * lda + (k + offset_k)] +#define A_ELEMENT(m) A_ELEMENT_K(m, 0) + +#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) + (j + (n)) * ldb] +#define B_ELEMENT(n) B_ELEMENT_K(n, 0) + +#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc] + +#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * 2 * v_size + m] +#define PACK_ELEMENT(m) PACK_ELEMENT_K(m, 0) + +// ASIMD +#define DECLARE_RESULT_VECTOR2(m, n) \ + float64x2_t result##m##n = vdupq_n_f64(0.0); +#define DECLARE_RESULT(m, n) float64_t result##m##n = 0.0; +#define BROADCAST_LOAD_A2(m, offset_k) \ + float64x2_t a##m##_k##offset_k = vld1q_dup_f64(&A_ELEMENT_K(m, offset_k)); +#define LOAD_A1(m, offset_k) \ + float64_t a##m##_k##offset_k = A_ELEMENT_K(m, offset_k); +#define VECTOR_LOAD_B_K2(n, offset_k) \ + float64x2_t b##k##n##_k##offset_k = vld1q_f64(&B_ELEMENT_K(n, offset_k)); +#define TRANSPOSE_B2_K2(n0, n1, offset_k0, offset_k1) \ + float64x2_t b##n0##_k##offset_k0 = \ + vzip1q_f64(b##k##n0##_k##offset_k0, b##k##n1##_k##offset_k0); \ + float64x2_t b##n0##_k##offset_k1 = \ + vzip2q_f64(b##k##n0##_k##offset_k0, b##k##n1##_k##offset_k0); + +#define SCALE_B2_K2(n0, offset_k0, offset_k1) \ + svfloat64_t b##s##n0##_k##offset_k0 = svdup_neonq_f64(b##n0##_k##offset_k0); \ + svfloat64_t b##s##n0##_k##offset_k1 = svdup_neonq_f64(b##n0##_k##offset_k1); +#define GATHER_LOAD_B2(n, offset_k) \ + float64x2_t b##n##_k##offset_k = vdupq_n_f64(B_ELEMENT_K(n, offset_k)); \ + b##n##_k##offset_k = \ + vsetq_lane_f64(B_ELEMENT_K(n + 1, offset_k), b##n##_k##offset_k, 1); +#define VECTOR_UNPACK_B2(n, offset_k) \ + float64x2_t b##n##_k##offset_k = vld1q_f64(&PACK_ELEMENT_K(n, offset_k)); +#define VECTOR_PACK_B2(n, offset_k) \ + vst1q_f64(&PACK_ELEMENT_K(n, offset_k), b##n##_k##offset_k); +#define PACK_B0(n, offset_k) \ + PACK_ELEMENT_K(n, offset_k) = vget_lane_f64(b##n##_k##offset_k, 0); +#define UPDATE_RESULT_VECTOR2(m, n, offset_k) \ + result##m##n = \ + vfmaq_f64(result##m##n, a##m##_k##offset_k, b##n##_k##offset_k); +#define UPDATE_RESULT(m, n, offset_k) \ + result##m##n = result##m##n + a##m##_k##offset_k * b##n##_k##offset_k; +#ifdef B0 +#define SCATTER_STORE2(m, n) \ + result##m##n = vmulq_f64(result##m##n, vdupq_n_f64(alpha)); \ + C_ELEMENT(m, n + 0) = vgetq_lane_f64(result##m##n, 0); \ + C_ELEMENT(m, n + 1) = vgetq_lane_f64(result##m##n, 1); +#else +#define SCATTER_STORE2(m, n) \ + result##m##n = vmulq_f64(result##m##n, vdupq_n_f64(alpha)); \ + C_ELEMENT(m, n + 0) = \ + C_ELEMENT(m, n + 0) * beta + vgetq_lane_f64(result##m##n, 0); \ + C_ELEMENT(m, n + 1) = \ + C_ELEMENT(m, n + 1) * beta + vgetq_lane_f64(result##m##n, 1); +#endif + +// SVE +#define DECLARE_RESULT_VECTOR(m, n) svfloat64_t result##m##n = svdup_f64(0.0); +#define BROADCAST_LOAD_A(m, offset_k) \ + svfloat64_t a##s##m##_k##offset_k = svdup_f64(A_ELEMENT_K(m, offset_k)); +#define BROADCAST_LOAD_B(n, offset_k) \ + svfloat64_t b##s##n##_k##offset_k = svdup_f64(B_ELEMENT_K(n, offset_k)); +#define VECTOR_LOAD_A(pg, m, offset_k) \ + svfloat64_t a##s##m##_k##offset_k = \ + svld1(pg, &A_ELEMENT_K(v_size * m, offset_k)); +#define QUADWORD_LOAD_B(n, offset_k) \ + svfloat64_t b##s##n##_k##offset_k = \ + svld1rq(pg_true, &B_ELEMENT_K(n, offset_k)); +#define GATHER_LOAD_A(pg, m, offset_k) \ + svfloat64_t a##s##m##_k##offset_k = \ + svld1_gather_index(pg, &A_ELEMENT_K(v_size * m, offset_k), lda_vec); +#define PACK_A(m, offset_k) \ + svst1(pg_first, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k); +#define VECTOR_PACK_A(m, offset_k) \ + svst1(pg_true, &PACK_ELEMENT_K(m* v_size, offset_k), a##s##m##_k##offset_k); +#define QUADWORD_PACK_A(m, offset_k) \ + svst1(pg_quad, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k); +#define UNPACK_VECTOR_A(m, offset_k) \ + svfloat64_t a##s##m##_k##offset_k = \ + svld1(pg_true, &PACK_ELEMENT_K(m * v_size, offset_k)); +#define UNPACK_BROADCAST_A(m, offset_k) \ + svfloat64_t a##s##m##_k##offset_k = svdup_f64(PACK_ELEMENT_K(m, offset_k)); +#define UNPACK_QUADWORD_A(m, offset_k) \ + svfloat64_t a##s##m##_k##offset_k = \ + svld1rq(pg_true, &PACK_ELEMENT_K(m, offset_k)); +#define UPDATE_RESULT_VECTOR(pg, m, n, offset_k) \ + result##m##n = \ + svmla_m(pg, result##m##n, a##s##m##_k##offset_k, b##s##n##_k##offset_k); +#define UPDATE_RESULT_VECTOR_QUADWORD(m, n, outer, lane, offset_k) \ + result##m##n = svmla_lane( \ + result##m##n, a##s##m##_k##offset_k, b##s##outer##_k##offset_k, lane); +#ifdef B0 +#define VECTOR_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); +#define SCATTER_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + svst1_scatter_index( \ + pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n); +#else +#define VECTOR_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + result##m##n = \ + svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(v_size * m, n)), beta_vec); \ + svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); +#define SCATTER_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + result##m##n = svmla_m( \ + pg, \ + result##m##n, \ + svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u64(0LL, ldc)), \ + beta_vec); \ + svst1_scatter_index( \ + pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n); +#endif + +#ifndef LIKELY +#ifdef __GNUC__ +#define LIKELY(x) __builtin_expect(!!(x), 1) +#else +#define LIKELY(x) (x) +#endif +#endif +#ifndef UNLIKELY +#ifdef __GNUC__ +#define UNLIKELY(x) __builtin_expect(!!(x), 0) +#else +#define UNLIKELY(x) (x) +#endif +#endif + +#ifdef B0 +int +CNAME(BLASLONG M, + BLASLONG N, + BLASLONG K, + IFLOAT* A, + BLASLONG lda, + FLOAT alpha, + IFLOAT* B, + BLASLONG ldb, + FLOAT* C, + BLASLONG ldc) +#else +int +CNAME(BLASLONG M, + BLASLONG N, + BLASLONG K, + IFLOAT* A, + BLASLONG lda, + FLOAT alpha, + IFLOAT* B, + BLASLONG ldb, + FLOAT beta, + FLOAT* C, + BLASLONG ldc) +#endif +{ + const uint64_t v_size = svcntd(); + const uint64_t v_size2 = v_size * 2; + const svbool_t pg_true = svptrue_b64(); + const svbool_t pg_quad = svwhilelt_b64(0, 2); + const svbool_t pg_first = svwhilelt_b64(0, 1); + const svfloat64_t alpha_vec = svdup_f64(alpha); +#ifndef B0 + const svfloat64_t beta_vec = svdup_f64(beta); +#endif + const svuint64_t lda_vec = svindex_u64(0LL, lda); + + const BLASLONG v_m2 = M & -v_size2; + const BLASLONG v_m1 = M & -v_size; + const BLASLONG n4 = N & -4; + const BLASLONG n2 = N & -2; + const BLASLONG k2 = K & -2; + + const int pack_a = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0; + FLOAT* packed_a = + (pack_a) ? packed_a = (FLOAT*)malloc(K * 2 * v_size * sizeof(FLOAT)) : NULL; + + BLASLONG i = 0; + for (; i < v_m2; i += v_size2) { + + BLASLONG j = 0; + for (; j < n4; j += 4) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(1, 1); + DECLARE_RESULT_VECTOR(1, 2); + DECLARE_RESULT_VECTOR(1, 3); + + if (LIKELY(packed_a != NULL)) { + if (j == 0) { + for (; k < k2; k += 2) { + + VECTOR_LOAD_B_K2(0, 0); + VECTOR_LOAD_B_K2(1, 0); + TRANSPOSE_B2_K2(0, 1, 0, 1); + SCALE_B2_K2(0, 0, 1); + GATHER_LOAD_A(pg_true, 0, 0); + VECTOR_PACK_A(0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + GATHER_LOAD_A(pg_true, 0, 1); + VECTOR_PACK_A(0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + VECTOR_LOAD_B_K2(2, 0); + VECTOR_LOAD_B_K2(3, 0); + TRANSPOSE_B2_K2(2, 3, 0, 1); + SCALE_B2_K2(2, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 1); + GATHER_LOAD_A(pg_true, 1, 0); + VECTOR_PACK_A(1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0); + GATHER_LOAD_A(pg_true, 1, 1); + VECTOR_PACK_A(1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 1); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + VECTOR_PACK_A(0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + GATHER_LOAD_A(pg_true, 1, 0); + VECTOR_PACK_A(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); + } + } else { + for (; k < k2; k += 2) { + + VECTOR_LOAD_B_K2(0, 0); + VECTOR_LOAD_B_K2(1, 0); + TRANSPOSE_B2_K2(0, 1, 0, 1); + SCALE_B2_K2(0, 0, 1); + UNPACK_VECTOR_A(0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UNPACK_VECTOR_A(0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + VECTOR_LOAD_B_K2(2, 0); + VECTOR_LOAD_B_K2(3, 0); + TRANSPOSE_B2_K2(2, 3, 0, 1); + SCALE_B2_K2(2, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 1); + UNPACK_VECTOR_A(1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0); + UNPACK_VECTOR_A(1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 1); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + UNPACK_VECTOR_A(0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + UNPACK_VECTOR_A(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); + } + } + } else { + for (; k < k2; k += 2) { + + VECTOR_LOAD_B_K2(0, 0); + VECTOR_LOAD_B_K2(1, 0); + TRANSPOSE_B2_K2(0, 1, 0, 1); + SCALE_B2_K2(0, 0, 1); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + GATHER_LOAD_A(pg_true, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + VECTOR_LOAD_B_K2(2, 0); + VECTOR_LOAD_B_K2(3, 0); + TRANSPOSE_B2_K2(2, 3, 0, 1); + SCALE_B2_K2(2, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 1); + GATHER_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0); + GATHER_LOAD_A(pg_true, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 1); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + GATHER_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); + } + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + VECTOR_STORE(pg_true, 1, 0); + VECTOR_STORE(pg_true, 1, 1); + VECTOR_STORE(pg_true, 1, 2); + VECTOR_STORE(pg_true, 1, 3); + } + for (; j < n2; j += 2) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(1, 1); + + if (LIKELY(packed_a != NULL)) { + for (; k < k2; k += 2) { + + VECTOR_LOAD_B_K2(0, 0); + VECTOR_LOAD_B_K2(1, 0); + TRANSPOSE_B2_K2(0, 1, 0, 1); + SCALE_B2_K2(0, 0, 1); + UNPACK_VECTOR_A(0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UNPACK_VECTOR_A(0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + UNPACK_VECTOR_A(1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UNPACK_VECTOR_A(1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + UNPACK_VECTOR_A(0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + UNPACK_VECTOR_A(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); + } + } else { + for (; k < k2; k += 2) { + + VECTOR_LOAD_B_K2(0, 0); + VECTOR_LOAD_B_K2(1, 0); + TRANSPOSE_B2_K2(0, 1, 0, 1); + SCALE_B2_K2(0, 0, 1); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + GATHER_LOAD_A(pg_true, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + GATHER_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + GATHER_LOAD_A(pg_true, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + GATHER_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); + } + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 1, 0); + VECTOR_STORE(pg_true, 1, 1); + } + for (; j < N; j++) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(1, 0); + + if (LIKELY(packed_a != NULL)) { + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + UNPACK_VECTOR_A(0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + UNPACK_VECTOR_A(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + } + } else { + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + GATHER_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + } + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 1, 0); + } + } + for (; i < v_m1; i += v_size) { + + BLASLONG j = 0; + for (; j < n4; j += 4) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + + for (; k < k2; k += 2) { + + VECTOR_LOAD_B_K2(0, 0); + VECTOR_LOAD_B_K2(1, 0); + TRANSPOSE_B2_K2(0, 1, 0, 1); + SCALE_B2_K2(0, 0, 1); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + GATHER_LOAD_A(pg_true, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + VECTOR_LOAD_B_K2(2, 0); + VECTOR_LOAD_B_K2(3, 0); + TRANSPOSE_B2_K2(2, 3, 0, 1); + SCALE_B2_K2(2, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 1); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + } + for (; j < n2; j += 2) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + + for (; k < k2; k += 2) { + + VECTOR_LOAD_B_K2(0, 0); + VECTOR_LOAD_B_K2(1, 0); + TRANSPOSE_B2_K2(0, 1, 0, 1); + SCALE_B2_K2(0, 0, 1); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + GATHER_LOAD_A(pg_true, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + } + for (; j < N; j++) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + } + VECTOR_STORE(pg_true, 0, 0); + } + } + for (; i < M; i += v_size) { + const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M)); + + BLASLONG j = 0; + for (; j < n4; j += 4) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + + for (; k < k2; k += 2) { + + VECTOR_LOAD_B_K2(0, 0); + VECTOR_LOAD_B_K2(1, 0); + TRANSPOSE_B2_K2(0, 1, 0, 1); + SCALE_B2_K2(0, 0, 1); + GATHER_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + GATHER_LOAD_A(pg_tail, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + VECTOR_LOAD_B_K2(2, 0); + VECTOR_LOAD_B_K2(3, 0); + TRANSPOSE_B2_K2(2, 3, 0, 1); + SCALE_B2_K2(2, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 1); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 3, 0); + } + VECTOR_STORE(pg_tail, 0, 0); + VECTOR_STORE(pg_tail, 0, 1); + VECTOR_STORE(pg_tail, 0, 2); + VECTOR_STORE(pg_tail, 0, 3); + } + for (; j < n2; j += 2) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + + for (; k < k2; k += 2) { + + VECTOR_LOAD_B_K2(0, 0); + VECTOR_LOAD_B_K2(1, 0); + TRANSPOSE_B2_K2(0, 1, 0, 1); + SCALE_B2_K2(0, 0, 1); + GATHER_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + GATHER_LOAD_A(pg_tail, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 1, 0); + } + VECTOR_STORE(pg_tail, 0, 0); + VECTOR_STORE(pg_tail, 0, 1); + } + for (; j < N; j++) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); + } + VECTOR_STORE(pg_tail, 0, 0); + } + } + + if (pack_a) + free(packed_a); + + return 0; +} diff --git a/kernel/arm64/dgemm_small_kernel_tt_sve.c b/kernel/arm64/dgemm_small_kernel_tt_sve.c new file mode 100644 index 0000000000..12fc0b59e4 --- /dev/null +++ b/kernel/arm64/dgemm_small_kernel_tt_sve.c @@ -0,0 +1,482 @@ +/*************************************************************************** +Copyright (c) 2024, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "common.h" + +#include +#include +#ifdef __ARM_NEON_SVE_BRIDGE +#include +#else +#define svdup_neonq_f32(fixed_reg) \ + ({ \ + svfloat32_t scalable_reg; \ + asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \ + scalable_reg; \ + }) +#define svdup_neonq_f64(fixed_reg) \ + ({ \ + svfloat64_t scalable_reg; \ + asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \ + scalable_reg; \ + }) +#endif + +#define A_ELEMENT_K(m, offset_k) A[(i + (m)) * lda + (k + offset_k)] +#define A_ELEMENT(m) A_ELEMENT_K(m, 0) + +#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) * ldb + (j + (n))] +#define B_ELEMENT(n) B_ELEMENT_K(n, 0) + +#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc] + +#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * 2 * v_size + m] +#define PACK_ELEMENT(m) PACK_ELEMENT_K(m, 0) + +// ASIMD +#define DECLARE_RESULT_VECTOR2(m, n) \ + float64x2_t result##m##n = vdupq_n_f64(0.0); +#define DECLARE_RESULT(m, n) float64_t result##m##n = 0.0; +#define BROADCAST_LOAD_A2(m, offset_k) \ + float64x2_t a##m##_k##offset_k = vld1q_dup_f64(&A_ELEMENT_K(m, offset_k)); +#define LOAD_A1(m, offset_k) \ + float64_t a##m##_k##offset_k = A_ELEMENT_K(m, offset_k); +#define VECTOR_LOAD_B2(n, offset_k) \ + float64x2_t b##n##_k##offset_k = vld1q_f64(&B_ELEMENT_K(n, offset_k)); +#define GATHER_LOAD_B2(n, offset_k) \ + float64x2_t b##n##_k##offset_k = vdupq_n_f64(B_ELEMENT_K(n, offset_k)); \ + b##n##_k##offset_k = \ + vsetq_lane_f64(B_ELEMENT_K(n + 1, offset_k), b##n##_k##offset_k, 1); +#define VECTOR_UNPACK_B2(n, offset_k) \ + float64x2_t b##n##_k##offset_k = vld1q_f64(&PACK_ELEMENT_K(n, offset_k)); +#define VECTOR_PACK_B2(n, offset_k) \ + vst1q_f64(&PACK_ELEMENT_K(n, offset_k), b##n##_k##offset_k); +#define PACK_B0(n, offset_k) \ + PACK_ELEMENT_K(n, offset_k) = vget_lane_f64(b##n##_k##offset_k, 0); +#define UPDATE_RESULT_VECTOR2(m, n, offset_k) \ + result##m##n = \ + vfmaq_f64(result##m##n, a##m##_k##offset_k, b##n##_k##offset_k); +#define UPDATE_RESULT(m, n, offset_k) \ + result##m##n = result##m##n + a##m##_k##offset_k * b##n##_k##offset_k; +#ifdef B0 +#define VECTOR_STORE2(m, n) \ + vst1q_f64(&C_ELEMENT(m, n), vmulq_f64(result##m##n, vdupq_n_f64(alpha))); +#define STORE(m, n) C_ELEMENT(m, n) = alpha * result##m##n; +#else +#define VECTOR_STORE2(m, n) \ + result##m##n = vmulq_f64(result##m##n, vdupq_n_f64(alpha)); \ + result##m##n = \ + vfmaq_f64(result##m##n, vld1q_f64(&C_ELEMENT(m, n)), vdupq_n_f64(beta)); \ + vst1q_f64(&C_ELEMENT(m, n), result##m##n); +#define STORE(m, n) \ + C_ELEMENT(m, n) = C_ELEMENT(m, n) * beta + alpha * result##m##n; +#endif + +// SVE +#define DECLARE_RESULT_VECTOR(m, n) svfloat64_t result##m##n = svdup_f64(0.0); +#define BROADCAST_LOAD_A(m, offset_k) \ + svfloat64_t a##s##m##_k##offset_k = svdup_f64(A_ELEMENT_K(m, offset_k)); +#define BROADCAST_LOAD_B(n, offset_k) \ + svfloat64_t b##s##n##_k##offset_k = svdup_f64(B_ELEMENT_K(n, offset_k)); +#define VECTOR_LOAD_A(pg, m, offset_k) \ + svfloat64_t a##s##m##_k##offset_k = \ + svld1(pg, &A_ELEMENT_K(v_size * m, offset_k)); +#define QUADWORD_LOAD_B(n, offset_k) \ + svfloat64_t b##s##n##_k##offset_k = \ + svld1rq(pg_true, &B_ELEMENT_K(n, offset_k)); +#define GATHER_LOAD_A(pg, m, offset_k) \ + svfloat64_t a##s##m##_k##offset_k = \ + svld1_gather_index(pg, &A_ELEMENT_K(v_size * m, offset_k), lda_vec); +#define PACK_A(m, offset_k) \ + svst1(pg_first, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k); +#define VECTOR_PACK_A(m, offset_k) \ + svst1(pg_true, &PACK_ELEMENT_K(m* v_size, offset_k), a##s##m##_k##offset_k); +#define QUADWORD_PACK_A(m, offset_k) \ + svst1(pg_quad, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k); +#define UNPACK_VECTOR_A(m, offset_k) \ + svfloat64_t a##s##m##_k##offset_k = \ + svld1(pg_true, &PACK_ELEMENT_K(m * v_size, offset_k)); +#define UNPACK_BROADCAST_A(m, offset_k) \ + svfloat64_t a##s##m##_k##offset_k = svdup_f64(PACK_ELEMENT_K(m, offset_k)); +#define UNPACK_QUADWORD_A(m, offset_k) \ + svfloat64_t a##s##m##_k##offset_k = \ + svld1rq(pg_true, &PACK_ELEMENT_K(m, offset_k)); +#define UPDATE_RESULT_VECTOR(pg, m, n, offset_k) \ + result##m##n = \ + svmla_m(pg, result##m##n, a##s##m##_k##offset_k, b##s##n##_k##offset_k); +#define UPDATE_RESULT_VECTOR_QUADWORD(m, n, outer, lane, offset_k) \ + result##m##n = svmla_lane( \ + result##m##n, a##s##m##_k##offset_k, b##s##outer##_k##offset_k, lane); +#ifdef B0 +#define VECTOR_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); +#define SCATTER_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + svst1_scatter_index( \ + pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n); +#else +#define VECTOR_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + result##m##n = \ + svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(v_size * m, n)), beta_vec); \ + svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); +#define SCATTER_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + result##m##n = svmla_m( \ + pg, \ + result##m##n, \ + svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u64(0LL, ldc)), \ + beta_vec); \ + svst1_scatter_index( \ + pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n); +#endif + +#ifndef LIKELY +#ifdef __GNUC__ +#define LIKELY(x) __builtin_expect(!!(x), 1) +#else +#define LIKELY(x) (x) +#endif +#endif +#ifndef UNLIKELY +#ifdef __GNUC__ +#define UNLIKELY(x) __builtin_expect(!!(x), 0) +#else +#define UNLIKELY(x) (x) +#endif +#endif + +#ifdef B0 +int +CNAME(BLASLONG M, + BLASLONG N, + BLASLONG K, + IFLOAT* A, + BLASLONG lda, + FLOAT alpha, + IFLOAT* B, + BLASLONG ldb, + FLOAT* C, + BLASLONG ldc) +#else +int +CNAME(BLASLONG M, + BLASLONG N, + BLASLONG K, + IFLOAT* A, + BLASLONG lda, + FLOAT alpha, + IFLOAT* B, + BLASLONG ldb, + FLOAT beta, + FLOAT* C, + BLASLONG ldc) +#endif +{ + const uint64_t v_size = svcntd(); + const uint64_t v_size2 = v_size * 2; + const svbool_t pg_true = svptrue_b64(); + const svbool_t pg_quad = svwhilelt_b64(0, 2); + const svbool_t pg_first = svwhilelt_b64(0, 1); + const svfloat64_t alpha_vec = svdup_f64(alpha); +#ifndef B0 + const svfloat64_t beta_vec = svdup_f64(beta); +#endif + const svuint64_t lda_vec = svindex_u64(0LL, lda); + + const BLASLONG v_m2 = M & -v_size2; + const BLASLONG v_m1 = M & -v_size; + const BLASLONG n4 = N & -4; + const BLASLONG n2 = N & -2; + + const int pack_a = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0; + FLOAT* packed_a = + (pack_a) ? packed_a = (FLOAT*)malloc(K * 2 * v_size * sizeof(FLOAT)) : NULL; + + BLASLONG i = 0; + for (; i < v_m2; i += v_size2) { + + BLASLONG j = 0; + for (; j < n4; j += 4) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(1, 1); + DECLARE_RESULT_VECTOR(1, 2); + DECLARE_RESULT_VECTOR(1, 3); + + if (LIKELY(packed_a != NULL)) { + if (j == 0) { + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + VECTOR_PACK_A(0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + QUADWORD_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); + GATHER_LOAD_A(pg_true, 1, 0); + VECTOR_PACK_A(1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0); + } + } else { + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + UNPACK_VECTOR_A(0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + QUADWORD_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); + UNPACK_VECTOR_A(1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0); + } + } + } else { + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + QUADWORD_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); + GATHER_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0); + } + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + VECTOR_STORE(pg_true, 1, 0); + VECTOR_STORE(pg_true, 1, 1); + VECTOR_STORE(pg_true, 1, 2); + VECTOR_STORE(pg_true, 1, 3); + } + for (; j < n2; j += 2) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(1, 1); + + if (LIKELY(packed_a != NULL)) { + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + UNPACK_VECTOR_A(0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UNPACK_VECTOR_A(1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + } + } else { + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + GATHER_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + } + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 1, 0); + VECTOR_STORE(pg_true, 1, 1); + } + for (; j < N; j++) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(1, 0); + + if (LIKELY(packed_a != NULL)) { + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + UNPACK_VECTOR_A(0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + UNPACK_VECTOR_A(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + } + } else { + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + GATHER_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + } + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 1, 0); + } + } + for (; i < v_m1; i += v_size) { + + BLASLONG j = 0; + for (; j < n4; j += 4) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + QUADWORD_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + } + for (; j < n2; j += 2) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + } + for (; j < N; j++) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + } + VECTOR_STORE(pg_true, 0, 0); + } + } + for (; i < M; i += v_size) { + const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M)); + + BLASLONG j = 0; + for (; j < n4; j += 4) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + GATHER_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + QUADWORD_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); + } + VECTOR_STORE(pg_tail, 0, 0); + VECTOR_STORE(pg_tail, 0, 1); + VECTOR_STORE(pg_tail, 0, 2); + VECTOR_STORE(pg_tail, 0, 3); + } + for (; j < n2; j += 2) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + GATHER_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + } + VECTOR_STORE(pg_tail, 0, 0); + VECTOR_STORE(pg_tail, 0, 1); + } + for (; j < N; j++) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); + } + VECTOR_STORE(pg_tail, 0, 0); + } + } + + if (pack_a) + free(packed_a); + + return 0; +} diff --git a/kernel/arm64/gemm_small_kernel_permit_sve.c b/kernel/arm64/gemm_small_kernel_permit_sve.c new file mode 100644 index 0000000000..9526dbbe22 --- /dev/null +++ b/kernel/arm64/gemm_small_kernel_permit_sve.c @@ -0,0 +1,47 @@ +/*************************************************************************** +Copyright (c) 2024, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "common.h" + +int CNAME(int transa, int transb, BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, FLOAT beta) +{ + BLASLONG MNK = M * N * K; + +#if defined(DOUBLE) // dgemm + // TN prefers full copies much earlier + if (transa && !transb && MNK > 16*16*16) { + return 0; + } +#else // sgemm + // TODO! +#endif + + if (MNK <= 64*64*64) + return 1; + + return 0; +} diff --git a/kernel/arm64/sgemm_small_kernel_nn_sve.c b/kernel/arm64/sgemm_small_kernel_nn_sve.c new file mode 100644 index 0000000000..85c7cfa868 --- /dev/null +++ b/kernel/arm64/sgemm_small_kernel_nn_sve.c @@ -0,0 +1,1046 @@ +/*************************************************************************** +Copyright (c) 2024, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "common.h" + +#include +#include +#ifdef __ARM_NEON_SVE_BRIDGE +#include +#else +#define svdup_neonq_f32(fixed_reg) \ + ({ \ + svfloat32_t scalable_reg; \ + asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \ + scalable_reg; \ + }) +#define svdup_neonq_f64(fixed_reg) \ + ({ \ + svfloat64_t scalable_reg; \ + asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \ + scalable_reg; \ + }) +#endif + +#define A_ELEMENT_K(m, offset_k) A[(i + (m)) + (k + offset_k) * lda] +#define A_ELEMENT(m) A_ELEMENT_K(m, 0) + +#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) + (j + (n)) * ldb] +#define B_ELEMENT(n) B_ELEMENT_K(n, 0) + +#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc] + +#define PACK_ELEMENT_K(n, offset_k) packed_b[(k + offset_k) * 8 + n] +#define PACK_ELEMENT(n) PACK_ELEMENT_K(n, 0) + +// ASIMD +#define DECLARE_RESULT_VECTOR4(m, n) \ + float32x4_t result##m##n = vdupq_n_f32(0.0); +#define DECLARE_RESULT(m, n) float32_t result##m##n = 0.0; +#define BROADCAST_LOAD_A4(m, offset_k) \ + float32x4_t a##m##_k##offset_k = vld1q_dup_f32(&A_ELEMENT_K(m, offset_k)); +#define LOAD_A1(m, offset_k) \ + float32_t a##m##_k##offset_k = A_ELEMENT_K(m, offset_k); +#define VECTOR_LOAD_B_K4(n, offset_k) \ + float32x4_t b##k##n##_k##offset_k = vld1q_f32(&B_ELEMENT_K(n, offset_k)); +#define TRANSPOSE_B4_K4( \ + n0, n1, n2, n3, offset_k0, offset_k1, offset_k2, offset_k3) \ + float32x4_t b##t##n0##_k##offset_k0 = \ + vzip1q_f32(b##k##n0##_k##offset_k0, b##k##n1##_k##offset_k0); \ + float32x4_t b##t##n0##_k##offset_k1 = \ + vzip2q_f32(b##k##n0##_k##offset_k0, b##k##n1##_k##offset_k0); \ + float32x4_t b##t##n0##_k##offset_k2 = \ + vzip1q_f32(b##k##n2##_k##offset_k0, b##k##n3##_k##offset_k0); \ + float32x4_t b##t##n0##_k##offset_k3 = \ + vzip2q_f32(b##k##n2##_k##offset_k0, b##k##n3##_k##offset_k0); \ + float32x4_t b##n0##_k##offset_k0 = vreinterpretq_f32_f64( \ + vzip1q_f64(vreinterpretq_f64_f32(b##t##n0##_k##offset_k0), \ + vreinterpretq_f64_f32(b##t##n0##_k##offset_k2))); \ + float32x4_t b##n0##_k##offset_k1 = vreinterpretq_f32_f64( \ + vzip2q_f64(vreinterpretq_f64_f32(b##t##n0##_k##offset_k0), \ + vreinterpretq_f64_f32(b##t##n0##_k##offset_k2))); \ + float32x4_t b##n0##_k##offset_k2 = vreinterpretq_f32_f64( \ + vzip1q_f64(vreinterpretq_f64_f32(b##t##n0##_k##offset_k1), \ + vreinterpretq_f64_f32(b##t##n0##_k##offset_k3))); \ + float32x4_t b##n0##_k##offset_k3 = vreinterpretq_f32_f64( \ + vzip2q_f64(vreinterpretq_f64_f32(b##t##n0##_k##offset_k1), \ + vreinterpretq_f64_f32(b##t##n0##_k##offset_k3))); + +#define SCALE_B4_K4(n0, offset_k0, offset_k1, offset_k2, offset_k3) \ + svfloat32_t b##s##n0##_k##offset_k0 = svdup_neonq_f32(b##n0##_k##offset_k0); \ + svfloat32_t b##s##n0##_k##offset_k1 = svdup_neonq_f32(b##n0##_k##offset_k1); \ + svfloat32_t b##s##n0##_k##offset_k2 = svdup_neonq_f32(b##n0##_k##offset_k2); \ + svfloat32_t b##s##n0##_k##offset_k3 = svdup_neonq_f32(b##n0##_k##offset_k3); +#define GATHER_LOAD_B4(n, offset_k) \ + float32x4_t b##n##_k##offset_k = vdupq_n_f32(B_ELEMENT_K(n, offset_k)); \ + b##n##_k##offset_k = \ + vsetq_lane_f32(B_ELEMENT_K(n + 1, offset_k), b##n##_k##offset_k, 1); \ + b##n##_k##offset_k = \ + vsetq_lane_f32(B_ELEMENT_K(n + 2, offset_k), b##n##_k##offset_k, 2); \ + b##n##_k##offset_k = \ + vsetq_lane_f32(B_ELEMENT_K(n + 3, offset_k), b##n##_k##offset_k, 3); +#define VECTOR_UNPACK_B4(n, offset_k) \ + float32x4_t b##n##_k##offset_k = vld1q_f32(&PACK_ELEMENT_K(n, offset_k)); +#define VECTOR_PACK_B4(n, offset_k) \ + vst1q_f32(&PACK_ELEMENT_K(n, offset_k), b##n##_k##offset_k); +#define PACK_B0(n, offset_k) \ + PACK_ELEMENT_K(n, offset_k) = vget_lane_f32(b##n##_k##offset_k, 0); +#define UPDATE_RESULT_VECTOR4(m, n, offset_k) \ + result##m##n = \ + vfmaq_f32(result##m##n, a##m##_k##offset_k, b##n##_k##offset_k); +#define UPDATE_RESULT(m, n, offset_k) \ + result##m##n = result##m##n + a##m##_k##offset_k * b##n##_k##offset_k; +#ifdef B0 +#define SCATTER_STORE4(m, n) \ + result##m##n = vmulq_f32(result##m##n, vdupq_n_f32(alpha)); \ + C_ELEMENT(m, n + 0) = vgetq_lane_f32(result##m##n, 0); \ + C_ELEMENT(m, n + 1) = vgetq_lane_f32(result##m##n, 1); \ + C_ELEMENT(m, n + 2) = vgetq_lane_f32(result##m##n, 2); \ + C_ELEMENT(m, n + 3) = vgetq_lane_f32(result##m##n, 3); +#else +#define SCATTER_STORE4(m, n) \ + result##m##n = vmulq_f32(result##m##n, vdupq_n_f32(alpha)); \ + C_ELEMENT(m, n + 0) = \ + C_ELEMENT(m, n + 0) * beta + vgetq_lane_f32(result##m##n, 0); \ + C_ELEMENT(m, n + 1) = \ + C_ELEMENT(m, n + 1) * beta + vgetq_lane_f32(result##m##n, 1); \ + C_ELEMENT(m, n + 2) = \ + C_ELEMENT(m, n + 2) * beta + vgetq_lane_f32(result##m##n, 2); \ + C_ELEMENT(m, n + 3) = \ + C_ELEMENT(m, n + 3) * beta + vgetq_lane_f32(result##m##n, 3); +#endif + +// SVE +#define DECLARE_RESULT_VECTOR(m, n) svfloat32_t result##m##n = svdup_f32(0.0); +#define BROADCAST_LOAD_A(m, offset_k) \ + svfloat32_t a##s##m##_k##offset_k = svdup_f32(A_ELEMENT_K(m, offset_k)); +#define BROADCAST_LOAD_B(n, offset_k) \ + svfloat32_t b##s##n##_k##offset_k = svdup_f32(B_ELEMENT_K(n, offset_k)); +#define VECTOR_LOAD_A(pg, m, offset_k) \ + svfloat32_t a##s##m##_k##offset_k = \ + svld1(pg, &A_ELEMENT_K(v_size * m, offset_k)); +#define QUADWORD_LOAD_B(n, offset_k) \ + svfloat32_t b##s##n##_k##offset_k = \ + svld1rq(pg_true, &B_ELEMENT_K(n, offset_k)); +#define PACK_B(n, offset_k) \ + svst1(pg_first, &PACK_ELEMENT_K(n, offset_k), b##s##n##_k##offset_k); +#define VECTOR_PACK_B(n, offset_k) \ + svst1(pg_true, &PACK_ELEMENT_K(n* v_size, offset_k), b##s##n##_k##offset_k); +#define QUADWORD_PACK_B(n, offset_k) \ + svst1(pg_quad, &PACK_ELEMENT_K(n, offset_k), b##s##n##_k##offset_k); +#define UNPACK_VECTOR_B(n, offset_k) \ + svfloat32_t b##s##n##_k##offset_k = \ + svld1(pg_true, &PACK_ELEMENT_K(n * v_size, offset_k)); +#define UNPACK_BROADCAST_B(n, offset_k) \ + svfloat32_t b##s##n##_k##offset_k = svdup_f32(PACK_ELEMENT_K(n, offset_k)); +#define UNPACK_QUADWORD_B(n, offset_k) \ + svfloat32_t b##s##n##_k##offset_k = \ + svld1rq(pg_true, &PACK_ELEMENT_K(n, offset_k)); +#define UPDATE_RESULT_VECTOR(pg, m, n, offset_k) \ + result##m##n = \ + svmla_m(pg, result##m##n, a##s##m##_k##offset_k, b##s##n##_k##offset_k); +#define UPDATE_RESULT_VECTOR_QUADWORD(m, n, outer, lane, offset_k) \ + result##m##n = svmla_lane( \ + result##m##n, a##s##m##_k##offset_k, b##s##outer##_k##offset_k, lane); +#ifdef B0 +#define VECTOR_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); +#define SCATTER_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + svst1_scatter_index( \ + pg, &C_ELEMENT(v_size* m, n), svindex_u32(0LL, ldc), result##m##n); +#else +#define VECTOR_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + result##m##n = \ + svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(v_size * m, n)), beta_vec); \ + svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); +#define SCATTER_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + result##m##n = svmla_m( \ + pg, \ + result##m##n, \ + svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u32(0LL, ldc)), \ + beta_vec); \ + svst1_scatter_index( \ + pg, &C_ELEMENT(v_size* m, n), svindex_u32(0LL, ldc), result##m##n); +#endif + +#ifndef LIKELY +#ifdef __GNUC__ +#define LIKELY(x) __builtin_expect(!!(x), 1) +#else +#define LIKELY(x) (x) +#endif +#endif +#ifndef UNLIKELY +#ifdef __GNUC__ +#define UNLIKELY(x) __builtin_expect(!!(x), 0) +#else +#define UNLIKELY(x) (x) +#endif +#endif + +#ifdef B0 +int +CNAME(BLASLONG M, + BLASLONG N, + BLASLONG K, + IFLOAT* A, + BLASLONG lda, + FLOAT alpha, + IFLOAT* B, + BLASLONG ldb, + FLOAT* C, + BLASLONG ldc) +#else +int +CNAME(BLASLONG M, + BLASLONG N, + BLASLONG K, + IFLOAT* A, + BLASLONG lda, + FLOAT alpha, + IFLOAT* B, + BLASLONG ldb, + FLOAT beta, + FLOAT* C, + BLASLONG ldc) +#endif +{ + const uint64_t v_size = svcntw(); + const uint64_t v_size2 = v_size * 2; + const svbool_t pg_true = svptrue_b32(); + const svbool_t pg_quad = svwhilelt_b32(0, 4); + const svbool_t pg_first = svwhilelt_b32(0, 1); + const svfloat32_t alpha_vec = svdup_f32(alpha); +#ifndef B0 + const svfloat32_t beta_vec = svdup_f32(beta); +#endif + const BLASLONG n8 = N & -8; + const BLASLONG n4 = N & -4; + const BLASLONG v_m2 = M & -v_size2; + const BLASLONG v_m1 = M & -v_size; + const BLASLONG k4 = K & -4; + + const int pack_b = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0; + FLOAT* packed_b = + (pack_b) ? packed_b = (FLOAT*)malloc(K * 8 * sizeof(FLOAT)) : NULL; + + BLASLONG j = 0; + for (; j < n8; j += 8) { + + BLASLONG i = 0; + for (; i < v_m2; i += v_size2) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(0, 4); + DECLARE_RESULT_VECTOR(0, 5); + DECLARE_RESULT_VECTOR(0, 6); + DECLARE_RESULT_VECTOR(0, 7); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(1, 1); + DECLARE_RESULT_VECTOR(1, 2); + DECLARE_RESULT_VECTOR(1, 3); + DECLARE_RESULT_VECTOR(1, 4); + DECLARE_RESULT_VECTOR(1, 5); + DECLARE_RESULT_VECTOR(1, 6); + DECLARE_RESULT_VECTOR(1, 7); + + if (LIKELY(packed_b != NULL)) { + if (i == 0) { + for (; k < k4; k += 4) { + + VECTOR_LOAD_B_K4(0, 0); + VECTOR_LOAD_B_K4(1, 0); + VECTOR_LOAD_B_K4(2, 0); + VECTOR_LOAD_B_K4(3, 0); + TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); + SCALE_B4_K4(0, 0, 1, 2, 3); + VECTOR_PACK_B4(0, 0); + VECTOR_PACK_B4(0, 1); + VECTOR_PACK_B4(0, 2); + VECTOR_PACK_B4(0, 3); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + VECTOR_LOAD_A(pg_true, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); + VECTOR_LOAD_A(pg_true, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); + VECTOR_LOAD_A(pg_true, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); + VECTOR_LOAD_B_K4(4, 0); + VECTOR_LOAD_B_K4(5, 0); + VECTOR_LOAD_B_K4(6, 0); + VECTOR_LOAD_B_K4(7, 0); + TRANSPOSE_B4_K4(4, 5, 6, 7, 0, 1, 2, 3); + SCALE_B4_K4(4, 0, 1, 2, 3); + VECTOR_PACK_B4(4, 0); + VECTOR_PACK_B4(4, 1); + VECTOR_PACK_B4(4, 2); + VECTOR_PACK_B4(4, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 3); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); + VECTOR_LOAD_A(pg_true, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 1); + VECTOR_LOAD_A(pg_true, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 2); + VECTOR_LOAD_A(pg_true, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 3); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + PACK_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + PACK_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); + BROADCAST_LOAD_B(2, 0); + PACK_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); + BROADCAST_LOAD_B(3, 0); + PACK_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); + BROADCAST_LOAD_B(4, 0); + PACK_B(4, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 4, 0); + BROADCAST_LOAD_B(5, 0); + PACK_B(5, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 5, 0); + BROADCAST_LOAD_B(6, 0); + PACK_B(6, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 6, 0); + BROADCAST_LOAD_B(7, 0); + PACK_B(7, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 7, 0); + } + } else { + for (; k < K; k++) { + + UNPACK_QUADWORD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + UNPACK_QUADWORD_B(4, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); + } + } + } else { + for (; k < k4; k += 4) { + + VECTOR_LOAD_B_K4(0, 0); + VECTOR_LOAD_B_K4(1, 0); + VECTOR_LOAD_B_K4(2, 0); + VECTOR_LOAD_B_K4(3, 0); + TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); + SCALE_B4_K4(0, 0, 1, 2, 3); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + VECTOR_LOAD_A(pg_true, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); + VECTOR_LOAD_A(pg_true, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); + VECTOR_LOAD_A(pg_true, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); + VECTOR_LOAD_B_K4(4, 0); + VECTOR_LOAD_B_K4(5, 0); + VECTOR_LOAD_B_K4(6, 0); + VECTOR_LOAD_B_K4(7, 0); + TRANSPOSE_B4_K4(4, 5, 6, 7, 0, 1, 2, 3); + SCALE_B4_K4(4, 0, 1, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 3); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); + VECTOR_LOAD_A(pg_true, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 1); + VECTOR_LOAD_A(pg_true, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 2); + VECTOR_LOAD_A(pg_true, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 3); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); + BROADCAST_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 4, 0); + BROADCAST_LOAD_B(5, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 5, 0); + BROADCAST_LOAD_B(6, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 6, 0); + BROADCAST_LOAD_B(7, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 7, 0); + } + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + VECTOR_STORE(pg_true, 0, 4); + VECTOR_STORE(pg_true, 0, 5); + VECTOR_STORE(pg_true, 0, 6); + VECTOR_STORE(pg_true, 0, 7); + VECTOR_STORE(pg_true, 1, 0); + VECTOR_STORE(pg_true, 1, 1); + VECTOR_STORE(pg_true, 1, 2); + VECTOR_STORE(pg_true, 1, 3); + VECTOR_STORE(pg_true, 1, 4); + VECTOR_STORE(pg_true, 1, 5); + VECTOR_STORE(pg_true, 1, 6); + VECTOR_STORE(pg_true, 1, 7); + } + for (; i < v_m1; i += v_size) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(0, 4); + DECLARE_RESULT_VECTOR(0, 5); + DECLARE_RESULT_VECTOR(0, 6); + DECLARE_RESULT_VECTOR(0, 7); + + if (LIKELY(packed_b != NULL)) { + for (; k < K; k++) { + + UNPACK_QUADWORD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + UNPACK_QUADWORD_B(4, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + } + } else { + for (; k < k4; k += 4) { + + VECTOR_LOAD_B_K4(0, 0); + VECTOR_LOAD_B_K4(1, 0); + VECTOR_LOAD_B_K4(2, 0); + VECTOR_LOAD_B_K4(3, 0); + TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); + SCALE_B4_K4(0, 0, 1, 2, 3); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + VECTOR_LOAD_A(pg_true, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); + VECTOR_LOAD_A(pg_true, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); + VECTOR_LOAD_A(pg_true, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); + VECTOR_LOAD_B_K4(4, 0); + VECTOR_LOAD_B_K4(5, 0); + VECTOR_LOAD_B_K4(6, 0); + VECTOR_LOAD_B_K4(7, 0); + TRANSPOSE_B4_K4(4, 5, 6, 7, 0, 1, 2, 3); + SCALE_B4_K4(4, 0, 1, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 3); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + BROADCAST_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0); + BROADCAST_LOAD_B(5, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0); + BROADCAST_LOAD_B(6, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0); + BROADCAST_LOAD_B(7, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0); + } + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + VECTOR_STORE(pg_true, 0, 4); + VECTOR_STORE(pg_true, 0, 5); + VECTOR_STORE(pg_true, 0, 6); + VECTOR_STORE(pg_true, 0, 7); + } + for (; i < M; i += v_size) { + const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M)); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(0, 4); + DECLARE_RESULT_VECTOR(0, 5); + DECLARE_RESULT_VECTOR(0, 6); + DECLARE_RESULT_VECTOR(0, 7); + + if (LIKELY(packed_b != NULL)) { + for (; k < K; k++) { + + UNPACK_QUADWORD_B(0, 0); + VECTOR_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + UNPACK_QUADWORD_B(4, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + } + } else { + for (; k < k4; k += 4) { + + VECTOR_LOAD_B_K4(0, 0); + VECTOR_LOAD_B_K4(1, 0); + VECTOR_LOAD_B_K4(2, 0); + VECTOR_LOAD_B_K4(3, 0); + TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); + SCALE_B4_K4(0, 0, 1, 2, 3); + VECTOR_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + VECTOR_LOAD_A(pg_tail, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); + VECTOR_LOAD_A(pg_tail, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); + VECTOR_LOAD_A(pg_tail, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); + VECTOR_LOAD_B_K4(4, 0); + VECTOR_LOAD_B_K4(5, 0); + VECTOR_LOAD_B_K4(6, 0); + VECTOR_LOAD_B_K4(7, 0); + TRANSPOSE_B4_K4(4, 5, 6, 7, 0, 1, 2, 3); + SCALE_B4_K4(4, 0, 1, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 3); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 3, 0); + BROADCAST_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 4, 0); + BROADCAST_LOAD_B(5, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 5, 0); + BROADCAST_LOAD_B(6, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 6, 0); + BROADCAST_LOAD_B(7, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 7, 0); + } + } + VECTOR_STORE(pg_tail, 0, 0); + VECTOR_STORE(pg_tail, 0, 1); + VECTOR_STORE(pg_tail, 0, 2); + VECTOR_STORE(pg_tail, 0, 3); + VECTOR_STORE(pg_tail, 0, 4); + VECTOR_STORE(pg_tail, 0, 5); + VECTOR_STORE(pg_tail, 0, 6); + VECTOR_STORE(pg_tail, 0, 7); + } + } + for (; j < n4; j += 4) { + + BLASLONG i = 0; + for (; i < v_m2; i += v_size2) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(1, 1); + DECLARE_RESULT_VECTOR(1, 2); + DECLARE_RESULT_VECTOR(1, 3); + + for (; k < k4; k += 4) { + + VECTOR_LOAD_B_K4(0, 0); + VECTOR_LOAD_B_K4(1, 0); + VECTOR_LOAD_B_K4(2, 0); + VECTOR_LOAD_B_K4(3, 0); + TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); + SCALE_B4_K4(0, 0, 1, 2, 3); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + VECTOR_LOAD_A(pg_true, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); + VECTOR_LOAD_A(pg_true, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); + VECTOR_LOAD_A(pg_true, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + VECTOR_LOAD_A(pg_true, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1); + VECTOR_LOAD_A(pg_true, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2); + VECTOR_LOAD_A(pg_true, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + VECTOR_STORE(pg_true, 1, 0); + VECTOR_STORE(pg_true, 1, 1); + VECTOR_STORE(pg_true, 1, 2); + VECTOR_STORE(pg_true, 1, 3); + } + for (; i < v_m1; i += v_size) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + + for (; k < k4; k += 4) { + + VECTOR_LOAD_B_K4(0, 0); + VECTOR_LOAD_B_K4(1, 0); + VECTOR_LOAD_B_K4(2, 0); + VECTOR_LOAD_B_K4(3, 0); + TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); + SCALE_B4_K4(0, 0, 1, 2, 3); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + VECTOR_LOAD_A(pg_true, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); + VECTOR_LOAD_A(pg_true, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); + VECTOR_LOAD_A(pg_true, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + } + for (; i < M; i += v_size) { + const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M)); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + + for (; k < k4; k += 4) { + + VECTOR_LOAD_B_K4(0, 0); + VECTOR_LOAD_B_K4(1, 0); + VECTOR_LOAD_B_K4(2, 0); + VECTOR_LOAD_B_K4(3, 0); + TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); + SCALE_B4_K4(0, 0, 1, 2, 3); + VECTOR_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + VECTOR_LOAD_A(pg_tail, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); + VECTOR_LOAD_A(pg_tail, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); + VECTOR_LOAD_A(pg_tail, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 3, 0); + } + VECTOR_STORE(pg_tail, 0, 0); + VECTOR_STORE(pg_tail, 0, 1); + VECTOR_STORE(pg_tail, 0, 2); + VECTOR_STORE(pg_tail, 0, 3); + } + } + for (; j < N; j++) { + + BLASLONG i = 0; + for (; i < v_m2; i += v_size2) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(1, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 1, 0); + } + for (; i < v_m1; i += v_size) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + } + VECTOR_STORE(pg_true, 0, 0); + } + for (; i < M; i += v_size) { + const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M)); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); + } + VECTOR_STORE(pg_tail, 0, 0); + } + } + + if (pack_b) + free(packed_b); + + return 0; +} diff --git a/kernel/arm64/sgemm_small_kernel_nt_sve.c b/kernel/arm64/sgemm_small_kernel_nt_sve.c new file mode 100644 index 0000000000..1c3d324d08 --- /dev/null +++ b/kernel/arm64/sgemm_small_kernel_nt_sve.c @@ -0,0 +1,647 @@ +/*************************************************************************** +Copyright (c) 2024, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "common.h" + +#include +#include +#ifdef __ARM_NEON_SVE_BRIDGE +#include +#else +#define svdup_neonq_f32(fixed_reg) \ + ({ \ + svfloat32_t scalable_reg; \ + asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \ + scalable_reg; \ + }) +#define svdup_neonq_f64(fixed_reg) \ + ({ \ + svfloat64_t scalable_reg; \ + asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \ + scalable_reg; \ + }) +#endif + +#define A_ELEMENT_K(m, offset_k) A[(i + (m)) + (k + offset_k) * lda] +#define A_ELEMENT(m) A_ELEMENT_K(m, 0) + +#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) * ldb + (j + (n))] +#define B_ELEMENT(n) B_ELEMENT_K(n, 0) + +#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc] + +#define PACK_ELEMENT_K(n, offset_k) packed_b[(k + offset_k) * 8 + n] +#define PACK_ELEMENT(n) PACK_ELEMENT_K(n, 0) + +// ASIMD +#define DECLARE_RESULT_VECTOR4(m, n) \ + float32x4_t result##m##n = vdupq_n_f32(0.0); +#define DECLARE_RESULT(m, n) float32_t result##m##n = 0.0; +#define BROADCAST_LOAD_A4(m, offset_k) \ + float32x4_t a##m##_k##offset_k = vld1q_dup_f32(&A_ELEMENT_K(m, offset_k)); +#define LOAD_A1(m, offset_k) \ + float32_t a##m##_k##offset_k = A_ELEMENT_K(m, offset_k); +#define VECTOR_LOAD_B4(n, offset_k) \ + float32x4_t b##n##_k##offset_k = vld1q_f32(&B_ELEMENT_K(n, offset_k)); +#define GATHER_LOAD_B4(n, offset_k) \ + float32x4_t b##n##_k##offset_k = vdupq_n_f32(B_ELEMENT_K(n, offset_k)); \ + b##n##_k##offset_k = \ + vsetq_lane_f32(B_ELEMENT_K(n + 1, offset_k), b##n##_k##offset_k, 1); \ + b##n##_k##offset_k = \ + vsetq_lane_f32(B_ELEMENT_K(n + 2, offset_k), b##n##_k##offset_k, 2); \ + b##n##_k##offset_k = \ + vsetq_lane_f32(B_ELEMENT_K(n + 3, offset_k), b##n##_k##offset_k, 3); +#define VECTOR_UNPACK_B4(n, offset_k) \ + float32x4_t b##n##_k##offset_k = vld1q_f32(&PACK_ELEMENT_K(n, offset_k)); +#define VECTOR_PACK_B4(n, offset_k) \ + vst1q_f32(&PACK_ELEMENT_K(n, offset_k), b##n##_k##offset_k); +#define PACK_B0(n, offset_k) \ + PACK_ELEMENT_K(n, offset_k) = vget_lane_f32(b##n##_k##offset_k, 0); +#define UPDATE_RESULT_VECTOR4(m, n, offset_k) \ + result##m##n = \ + vfmaq_f32(result##m##n, a##m##_k##offset_k, b##n##_k##offset_k); +#define UPDATE_RESULT(m, n, offset_k) \ + result##m##n = result##m##n + a##m##_k##offset_k * b##n##_k##offset_k; +#ifdef B0 +#define SCATTER_STORE4(m, n) \ + result##m##n = vmulq_f32(result##m##n, vdupq_n_f32(alpha)); \ + C_ELEMENT(m, n + 0) = vgetq_lane_f32(result##m##n, 0); \ + C_ELEMENT(m, n + 1) = vgetq_lane_f32(result##m##n, 1); \ + C_ELEMENT(m, n + 2) = vgetq_lane_f32(result##m##n, 2); \ + C_ELEMENT(m, n + 3) = vgetq_lane_f32(result##m##n, 3); +#else +#define SCATTER_STORE4(m, n) \ + result##m##n = vmulq_f32(result##m##n, vdupq_n_f32(alpha)); \ + C_ELEMENT(m, n + 0) = \ + C_ELEMENT(m, n + 0) * beta + vgetq_lane_f32(result##m##n, 0); \ + C_ELEMENT(m, n + 1) = \ + C_ELEMENT(m, n + 1) * beta + vgetq_lane_f32(result##m##n, 1); \ + C_ELEMENT(m, n + 2) = \ + C_ELEMENT(m, n + 2) * beta + vgetq_lane_f32(result##m##n, 2); \ + C_ELEMENT(m, n + 3) = \ + C_ELEMENT(m, n + 3) * beta + vgetq_lane_f32(result##m##n, 3); +#endif + +// SVE +#define DECLARE_RESULT_VECTOR(m, n) svfloat32_t result##m##n = svdup_f32(0.0); +#define BROADCAST_LOAD_A(m, offset_k) \ + svfloat32_t a##s##m##_k##offset_k = svdup_f32(A_ELEMENT_K(m, offset_k)); +#define BROADCAST_LOAD_B(n, offset_k) \ + svfloat32_t b##s##n##_k##offset_k = svdup_f32(B_ELEMENT_K(n, offset_k)); +#define VECTOR_LOAD_A(pg, m, offset_k) \ + svfloat32_t a##s##m##_k##offset_k = \ + svld1(pg, &A_ELEMENT_K(v_size * m, offset_k)); +#define QUADWORD_LOAD_B(n, offset_k) \ + svfloat32_t b##s##n##_k##offset_k = \ + svld1rq(pg_true, &B_ELEMENT_K(n, offset_k)); +#define PACK_B(n, offset_k) \ + svst1(pg_first, &PACK_ELEMENT_K(n, offset_k), b##s##n##_k##offset_k); +#define VECTOR_PACK_B(n, offset_k) \ + svst1(pg_true, &PACK_ELEMENT_K(n* v_size, offset_k), b##s##n##_k##offset_k); +#define QUADWORD_PACK_B(n, offset_k) \ + svst1(pg_quad, &PACK_ELEMENT_K(n, offset_k), b##s##n##_k##offset_k); +#define UNPACK_VECTOR_B(n, offset_k) \ + svfloat32_t b##s##n##_k##offset_k = \ + svld1(pg_true, &PACK_ELEMENT_K(n * v_size, offset_k)); +#define UNPACK_BROADCAST_B(n, offset_k) \ + svfloat32_t b##s##n##_k##offset_k = svdup_f32(PACK_ELEMENT_K(n, offset_k)); +#define UNPACK_QUADWORD_B(n, offset_k) \ + svfloat32_t b##s##n##_k##offset_k = \ + svld1rq(pg_true, &PACK_ELEMENT_K(n, offset_k)); +#define UPDATE_RESULT_VECTOR(pg, m, n, offset_k) \ + result##m##n = \ + svmla_m(pg, result##m##n, a##s##m##_k##offset_k, b##s##n##_k##offset_k); +#define UPDATE_RESULT_VECTOR_QUADWORD(m, n, outer, lane, offset_k) \ + result##m##n = svmla_lane( \ + result##m##n, a##s##m##_k##offset_k, b##s##outer##_k##offset_k, lane); +#ifdef B0 +#define VECTOR_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); +#define SCATTER_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + svst1_scatter_index( \ + pg, &C_ELEMENT(v_size* m, n), svindex_u32(0LL, ldc), result##m##n); +#else +#define VECTOR_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + result##m##n = \ + svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(v_size * m, n)), beta_vec); \ + svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); +#define SCATTER_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + result##m##n = svmla_m( \ + pg, \ + result##m##n, \ + svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u32(0LL, ldc)), \ + beta_vec); \ + svst1_scatter_index( \ + pg, &C_ELEMENT(v_size* m, n), svindex_u32(0LL, ldc), result##m##n); +#endif + +#ifndef LIKELY +#ifdef __GNUC__ +#define LIKELY(x) __builtin_expect(!!(x), 1) +#else +#define LIKELY(x) (x) +#endif +#endif +#ifndef UNLIKELY +#ifdef __GNUC__ +#define UNLIKELY(x) __builtin_expect(!!(x), 0) +#else +#define UNLIKELY(x) (x) +#endif +#endif + +#ifdef B0 +int +CNAME(BLASLONG M, + BLASLONG N, + BLASLONG K, + IFLOAT* A, + BLASLONG lda, + FLOAT alpha, + IFLOAT* B, + BLASLONG ldb, + FLOAT* C, + BLASLONG ldc) +#else +int +CNAME(BLASLONG M, + BLASLONG N, + BLASLONG K, + IFLOAT* A, + BLASLONG lda, + FLOAT alpha, + IFLOAT* B, + BLASLONG ldb, + FLOAT beta, + FLOAT* C, + BLASLONG ldc) +#endif +{ + const uint64_t v_size = svcntw(); + const uint64_t v_size3 = v_size * 3; + const svbool_t pg_true = svptrue_b32(); + const svbool_t pg_quad = svwhilelt_b32(0, 4); + const svbool_t pg_first = svwhilelt_b32(0, 1); + const svfloat32_t alpha_vec = svdup_f32(alpha); +#ifndef B0 + const svfloat32_t beta_vec = svdup_f32(beta); +#endif + const BLASLONG n8 = N & -8; + const BLASLONG n4 = N & -4; + const BLASLONG v_m3 = M - (M % v_size3); + const BLASLONG v_m1 = M & -v_size; + + const int pack_b = M >= v_size3 && N >= 8 && K >= 8 ? 1 : 0; + FLOAT* packed_b = + (pack_b) ? packed_b = (FLOAT*)malloc(K * 8 * sizeof(FLOAT)) : NULL; + + BLASLONG j = 0; + for (; j < n8; j += 8) { + + BLASLONG i = 0; + for (; i < v_m3; i += v_size3) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(0, 4); + DECLARE_RESULT_VECTOR(0, 5); + DECLARE_RESULT_VECTOR(0, 6); + DECLARE_RESULT_VECTOR(0, 7); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(1, 1); + DECLARE_RESULT_VECTOR(1, 2); + DECLARE_RESULT_VECTOR(1, 3); + DECLARE_RESULT_VECTOR(1, 4); + DECLARE_RESULT_VECTOR(1, 5); + DECLARE_RESULT_VECTOR(1, 6); + DECLARE_RESULT_VECTOR(1, 7); + DECLARE_RESULT_VECTOR(2, 0); + DECLARE_RESULT_VECTOR(2, 1); + DECLARE_RESULT_VECTOR(2, 2); + DECLARE_RESULT_VECTOR(2, 3); + DECLARE_RESULT_VECTOR(2, 4); + DECLARE_RESULT_VECTOR(2, 5); + DECLARE_RESULT_VECTOR(2, 6); + DECLARE_RESULT_VECTOR(2, 7); + + if (LIKELY(packed_b != NULL)) { + if (i == 0) { + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + QUADWORD_PACK_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + QUADWORD_LOAD_B(4, 0); + QUADWORD_PACK_B(4, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); + VECTOR_LOAD_A(pg_true, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 3, 0, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 7, 4, 3, 0); + } + } else { + for (; k < K; k++) { + + UNPACK_QUADWORD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + UNPACK_QUADWORD_B(4, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); + VECTOR_LOAD_A(pg_true, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 3, 0, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 7, 4, 3, 0); + } + } + } else { + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + QUADWORD_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); + VECTOR_LOAD_A(pg_true, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 3, 0, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 7, 4, 3, 0); + } + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + VECTOR_STORE(pg_true, 0, 4); + VECTOR_STORE(pg_true, 0, 5); + VECTOR_STORE(pg_true, 0, 6); + VECTOR_STORE(pg_true, 0, 7); + VECTOR_STORE(pg_true, 1, 0); + VECTOR_STORE(pg_true, 1, 1); + VECTOR_STORE(pg_true, 1, 2); + VECTOR_STORE(pg_true, 1, 3); + VECTOR_STORE(pg_true, 1, 4); + VECTOR_STORE(pg_true, 1, 5); + VECTOR_STORE(pg_true, 1, 6); + VECTOR_STORE(pg_true, 1, 7); + VECTOR_STORE(pg_true, 2, 0); + VECTOR_STORE(pg_true, 2, 1); + VECTOR_STORE(pg_true, 2, 2); + VECTOR_STORE(pg_true, 2, 3); + VECTOR_STORE(pg_true, 2, 4); + VECTOR_STORE(pg_true, 2, 5); + VECTOR_STORE(pg_true, 2, 6); + VECTOR_STORE(pg_true, 2, 7); + } + for (; i < v_m1; i += v_size) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(0, 4); + DECLARE_RESULT_VECTOR(0, 5); + DECLARE_RESULT_VECTOR(0, 6); + DECLARE_RESULT_VECTOR(0, 7); + + if (LIKELY(packed_b != NULL)) { + for (; k < K; k++) { + + UNPACK_QUADWORD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + UNPACK_QUADWORD_B(4, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + } + } else { + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + QUADWORD_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + } + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + VECTOR_STORE(pg_true, 0, 4); + VECTOR_STORE(pg_true, 0, 5); + VECTOR_STORE(pg_true, 0, 6); + VECTOR_STORE(pg_true, 0, 7); + } + for (; i < M; i += v_size) { + const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M)); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(0, 4); + DECLARE_RESULT_VECTOR(0, 5); + DECLARE_RESULT_VECTOR(0, 6); + DECLARE_RESULT_VECTOR(0, 7); + + if (LIKELY(packed_b != NULL)) { + for (; k < K; k++) { + + UNPACK_QUADWORD_B(0, 0); + VECTOR_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + UNPACK_QUADWORD_B(4, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + } + } else { + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + QUADWORD_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + } + } + VECTOR_STORE(pg_tail, 0, 0); + VECTOR_STORE(pg_tail, 0, 1); + VECTOR_STORE(pg_tail, 0, 2); + VECTOR_STORE(pg_tail, 0, 3); + VECTOR_STORE(pg_tail, 0, 4); + VECTOR_STORE(pg_tail, 0, 5); + VECTOR_STORE(pg_tail, 0, 6); + VECTOR_STORE(pg_tail, 0, 7); + } + } + for (; j < n4; j += 4) { + + BLASLONG i = 0; + for (; i < v_m3; i += v_size3) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(1, 1); + DECLARE_RESULT_VECTOR(1, 2); + DECLARE_RESULT_VECTOR(1, 3); + DECLARE_RESULT_VECTOR(2, 0); + DECLARE_RESULT_VECTOR(2, 1); + DECLARE_RESULT_VECTOR(2, 2); + DECLARE_RESULT_VECTOR(2, 3); + + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + VECTOR_LOAD_A(pg_true, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(2, 3, 0, 3, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + VECTOR_STORE(pg_true, 1, 0); + VECTOR_STORE(pg_true, 1, 1); + VECTOR_STORE(pg_true, 1, 2); + VECTOR_STORE(pg_true, 1, 3); + VECTOR_STORE(pg_true, 2, 0); + VECTOR_STORE(pg_true, 2, 1); + VECTOR_STORE(pg_true, 2, 2); + VECTOR_STORE(pg_true, 2, 3); + } + for (; i < v_m1; i += v_size) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + } + for (; i < M; i += v_size) { + const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M)); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + } + VECTOR_STORE(pg_tail, 0, 0); + VECTOR_STORE(pg_tail, 0, 1); + VECTOR_STORE(pg_tail, 0, 2); + VECTOR_STORE(pg_tail, 0, 3); + } + } + for (; j < N; j++) { + + BLASLONG i = 0; + for (; i < v_m3; i += v_size3) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(2, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + VECTOR_LOAD_A(pg_true, 2, 0); + UPDATE_RESULT_VECTOR(pg_true, 2, 0, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 1, 0); + VECTOR_STORE(pg_true, 2, 0); + } + for (; i < v_m1; i += v_size) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + } + VECTOR_STORE(pg_true, 0, 0); + } + for (; i < M; i += v_size) { + const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M)); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); + } + VECTOR_STORE(pg_tail, 0, 0); + } + } + + if (pack_b) + free(packed_b); + + return 0; +} diff --git a/kernel/arm64/sgemm_small_kernel_tn_sve.c b/kernel/arm64/sgemm_small_kernel_tn_sve.c new file mode 100644 index 0000000000..6fd3b12a67 --- /dev/null +++ b/kernel/arm64/sgemm_small_kernel_tn_sve.c @@ -0,0 +1,1247 @@ +/*************************************************************************** +Copyright (c) 2024, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "common.h" + +#include +#include +#ifdef __ARM_NEON_SVE_BRIDGE +#include +#else +#define svdup_neonq_f32(fixed_reg) \ + ({ \ + svfloat32_t scalable_reg; \ + asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \ + scalable_reg; \ + }) +#define svdup_neonq_f64(fixed_reg) \ + ({ \ + svfloat64_t scalable_reg; \ + asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \ + scalable_reg; \ + }) +#endif + +#define A_ELEMENT_K(m, offset_k) A[(i + (m)) * lda + (k + offset_k)] +#define A_ELEMENT(m) A_ELEMENT_K(m, 0) + +#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) + (j + (n)) * ldb] +#define B_ELEMENT(n) B_ELEMENT_K(n, 0) + +#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc] + +#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * 2 * v_size + m] +#define PACK_ELEMENT(m) PACK_ELEMENT_K(m, 0) + +// ASIMD +#define DECLARE_RESULT_VECTOR4(m, n) \ + float32x4_t result##m##n = vdupq_n_f32(0.0); +#define DECLARE_RESULT(m, n) float32_t result##m##n = 0.0; +#define BROADCAST_LOAD_A4(m, offset_k) \ + float32x4_t a##m##_k##offset_k = vld1q_dup_f32(&A_ELEMENT_K(m, offset_k)); +#define LOAD_A1(m, offset_k) \ + float32_t a##m##_k##offset_k = A_ELEMENT_K(m, offset_k); +#define VECTOR_LOAD_B_K4(n, offset_k) \ + float32x4_t b##k##n##_k##offset_k = vld1q_f32(&B_ELEMENT_K(n, offset_k)); +#define TRANSPOSE_B4_K4( \ + n0, n1, n2, n3, offset_k0, offset_k1, offset_k2, offset_k3) \ + float32x4_t b##t##n0##_k##offset_k0 = \ + vzip1q_f32(b##k##n0##_k##offset_k0, b##k##n1##_k##offset_k0); \ + float32x4_t b##t##n0##_k##offset_k1 = \ + vzip2q_f32(b##k##n0##_k##offset_k0, b##k##n1##_k##offset_k0); \ + float32x4_t b##t##n0##_k##offset_k2 = \ + vzip1q_f32(b##k##n2##_k##offset_k0, b##k##n3##_k##offset_k0); \ + float32x4_t b##t##n0##_k##offset_k3 = \ + vzip2q_f32(b##k##n2##_k##offset_k0, b##k##n3##_k##offset_k0); \ + float32x4_t b##n0##_k##offset_k0 = vreinterpretq_f32_f64( \ + vzip1q_f64(vreinterpretq_f64_f32(b##t##n0##_k##offset_k0), \ + vreinterpretq_f64_f32(b##t##n0##_k##offset_k2))); \ + float32x4_t b##n0##_k##offset_k1 = vreinterpretq_f32_f64( \ + vzip2q_f64(vreinterpretq_f64_f32(b##t##n0##_k##offset_k0), \ + vreinterpretq_f64_f32(b##t##n0##_k##offset_k2))); \ + float32x4_t b##n0##_k##offset_k2 = vreinterpretq_f32_f64( \ + vzip1q_f64(vreinterpretq_f64_f32(b##t##n0##_k##offset_k1), \ + vreinterpretq_f64_f32(b##t##n0##_k##offset_k3))); \ + float32x4_t b##n0##_k##offset_k3 = vreinterpretq_f32_f64( \ + vzip2q_f64(vreinterpretq_f64_f32(b##t##n0##_k##offset_k1), \ + vreinterpretq_f64_f32(b##t##n0##_k##offset_k3))); + +#define SCALE_B4_K4(n0, offset_k0, offset_k1, offset_k2, offset_k3) \ + svfloat32_t b##s##n0##_k##offset_k0 = svdup_neonq_f32(b##n0##_k##offset_k0); \ + svfloat32_t b##s##n0##_k##offset_k1 = svdup_neonq_f32(b##n0##_k##offset_k1); \ + svfloat32_t b##s##n0##_k##offset_k2 = svdup_neonq_f32(b##n0##_k##offset_k2); \ + svfloat32_t b##s##n0##_k##offset_k3 = svdup_neonq_f32(b##n0##_k##offset_k3); +#define GATHER_LOAD_B4(n, offset_k) \ + float32x4_t b##n##_k##offset_k = vdupq_n_f32(B_ELEMENT_K(n, offset_k)); \ + b##n##_k##offset_k = \ + vsetq_lane_f32(B_ELEMENT_K(n + 1, offset_k), b##n##_k##offset_k, 1); \ + b##n##_k##offset_k = \ + vsetq_lane_f32(B_ELEMENT_K(n + 2, offset_k), b##n##_k##offset_k, 2); \ + b##n##_k##offset_k = \ + vsetq_lane_f32(B_ELEMENT_K(n + 3, offset_k), b##n##_k##offset_k, 3); +#define VECTOR_UNPACK_B4(n, offset_k) \ + float32x4_t b##n##_k##offset_k = vld1q_f32(&PACK_ELEMENT_K(n, offset_k)); +#define VECTOR_PACK_B4(n, offset_k) \ + vst1q_f32(&PACK_ELEMENT_K(n, offset_k), b##n##_k##offset_k); +#define PACK_B0(n, offset_k) \ + PACK_ELEMENT_K(n, offset_k) = vget_lane_f32(b##n##_k##offset_k, 0); +#define UPDATE_RESULT_VECTOR4(m, n, offset_k) \ + result##m##n = \ + vfmaq_f32(result##m##n, a##m##_k##offset_k, b##n##_k##offset_k); +#define UPDATE_RESULT(m, n, offset_k) \ + result##m##n = result##m##n + a##m##_k##offset_k * b##n##_k##offset_k; +#ifdef B0 +#define SCATTER_STORE4(m, n) \ + result##m##n = vmulq_f32(result##m##n, vdupq_n_f32(alpha)); \ + C_ELEMENT(m, n + 0) = vgetq_lane_f32(result##m##n, 0); \ + C_ELEMENT(m, n + 1) = vgetq_lane_f32(result##m##n, 1); \ + C_ELEMENT(m, n + 2) = vgetq_lane_f32(result##m##n, 2); \ + C_ELEMENT(m, n + 3) = vgetq_lane_f32(result##m##n, 3); +#else +#define SCATTER_STORE4(m, n) \ + result##m##n = vmulq_f32(result##m##n, vdupq_n_f32(alpha)); \ + C_ELEMENT(m, n + 0) = \ + C_ELEMENT(m, n + 0) * beta + vgetq_lane_f32(result##m##n, 0); \ + C_ELEMENT(m, n + 1) = \ + C_ELEMENT(m, n + 1) * beta + vgetq_lane_f32(result##m##n, 1); \ + C_ELEMENT(m, n + 2) = \ + C_ELEMENT(m, n + 2) * beta + vgetq_lane_f32(result##m##n, 2); \ + C_ELEMENT(m, n + 3) = \ + C_ELEMENT(m, n + 3) * beta + vgetq_lane_f32(result##m##n, 3); +#endif + +// SVE +#define DECLARE_RESULT_VECTOR(m, n) svfloat32_t result##m##n = svdup_f32(0.0); +#define BROADCAST_LOAD_A(m, offset_k) \ + svfloat32_t a##s##m##_k##offset_k = svdup_f32(A_ELEMENT_K(m, offset_k)); +#define BROADCAST_LOAD_B(n, offset_k) \ + svfloat32_t b##s##n##_k##offset_k = svdup_f32(B_ELEMENT_K(n, offset_k)); +#define VECTOR_LOAD_A(pg, m, offset_k) \ + svfloat32_t a##s##m##_k##offset_k = \ + svld1(pg, &A_ELEMENT_K(v_size * m, offset_k)); +#define QUADWORD_LOAD_B(n, offset_k) \ + svfloat32_t b##s##n##_k##offset_k = \ + svld1rq(pg_true, &B_ELEMENT_K(n, offset_k)); +#define GATHER_LOAD_A(pg, m, offset_k) \ + svfloat32_t a##s##m##_k##offset_k = \ + svld1_gather_index(pg, &A_ELEMENT_K(v_size * m, offset_k), lda_vec); +#define PACK_A(m, offset_k) \ + svst1(pg_first, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k); +#define VECTOR_PACK_A(m, offset_k) \ + svst1(pg_true, &PACK_ELEMENT_K(m* v_size, offset_k), a##s##m##_k##offset_k); +#define QUADWORD_PACK_A(m, offset_k) \ + svst1(pg_quad, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k); +#define UNPACK_VECTOR_A(m, offset_k) \ + svfloat32_t a##s##m##_k##offset_k = \ + svld1(pg_true, &PACK_ELEMENT_K(m * v_size, offset_k)); +#define UNPACK_BROADCAST_A(m, offset_k) \ + svfloat32_t a##s##m##_k##offset_k = svdup_f32(PACK_ELEMENT_K(m, offset_k)); +#define UNPACK_QUADWORD_A(m, offset_k) \ + svfloat32_t a##s##m##_k##offset_k = \ + svld1rq(pg_true, &PACK_ELEMENT_K(m, offset_k)); +#define UPDATE_RESULT_VECTOR(pg, m, n, offset_k) \ + result##m##n = \ + svmla_m(pg, result##m##n, a##s##m##_k##offset_k, b##s##n##_k##offset_k); +#define UPDATE_RESULT_VECTOR_QUADWORD(m, n, outer, lane, offset_k) \ + result##m##n = svmla_lane( \ + result##m##n, a##s##m##_k##offset_k, b##s##outer##_k##offset_k, lane); +#ifdef B0 +#define VECTOR_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); +#define SCATTER_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + svst1_scatter_index( \ + pg, &C_ELEMENT(v_size* m, n), svindex_u32(0LL, ldc), result##m##n); +#else +#define VECTOR_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + result##m##n = \ + svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(v_size * m, n)), beta_vec); \ + svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); +#define SCATTER_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + result##m##n = svmla_m( \ + pg, \ + result##m##n, \ + svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u32(0LL, ldc)), \ + beta_vec); \ + svst1_scatter_index( \ + pg, &C_ELEMENT(v_size* m, n), svindex_u32(0LL, ldc), result##m##n); +#endif + +#ifndef LIKELY +#ifdef __GNUC__ +#define LIKELY(x) __builtin_expect(!!(x), 1) +#else +#define LIKELY(x) (x) +#endif +#endif +#ifndef UNLIKELY +#ifdef __GNUC__ +#define UNLIKELY(x) __builtin_expect(!!(x), 0) +#else +#define UNLIKELY(x) (x) +#endif +#endif + + +#define GATHER_LOAD_A64(pg, m, offset_k) \ + svfloat64_t a##t##m##_k##offset_k = \ + svld1_gather_offset(pg, (double *)&A_ELEMENT_K(v64_size * m, offset_k), lda_vec64); + +#ifdef B0 +int +CNAME(BLASLONG M, + BLASLONG N, + BLASLONG K, + IFLOAT* A, + BLASLONG lda, + FLOAT alpha, + IFLOAT* B, + BLASLONG ldb, + FLOAT* C, + BLASLONG ldc) +#else +int +CNAME(BLASLONG M, + BLASLONG N, + BLASLONG K, + IFLOAT* A, + BLASLONG lda, + FLOAT alpha, + IFLOAT* B, + BLASLONG ldb, + FLOAT beta, + FLOAT* C, + BLASLONG ldc) +#endif +{ + const uint64_t v_size = svcntw(); + const uint64_t v_size2 = v_size * 2; + const uint64_t v64_size = v_size / 2; + const svbool_t pg_true = svptrue_b32(); + const svbool_t pg_quad = svwhilelt_b32(0, 4); + const svbool_t pg_first = svwhilelt_b32(0, 1); + const svfloat32_t alpha_vec = svdup_f32(alpha); +#ifndef B0 + const svfloat32_t beta_vec = svdup_f32(beta); +#endif + const svuint32_t lda_vec = svindex_u32(0LL, lda); + const svuint64_t lda_vec64 = svmul_m(pg_true, svindex_u64(0,sizeof(FLOAT)), lda); + + const BLASLONG v_m2 = M & -v_size2; + const BLASLONG v_m1 = M & -v_size; + const BLASLONG n8 = N & -8; + const BLASLONG n4 = N & -4; + const BLASLONG k4 = K & -4; + + const int pack_a = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0; + FLOAT* packed_a = + (pack_a) ? packed_a = (FLOAT*)malloc(K * 2 * v_size * sizeof(FLOAT)) : NULL; + + BLASLONG i = 0; + for (; i < v_m2; i += v_size2) { + + BLASLONG j = 0; + for (; j < n8; j += 8) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(0, 4); + DECLARE_RESULT_VECTOR(0, 5); + DECLARE_RESULT_VECTOR(0, 6); + DECLARE_RESULT_VECTOR(0, 7); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(1, 1); + DECLARE_RESULT_VECTOR(1, 2); + DECLARE_RESULT_VECTOR(1, 3); + DECLARE_RESULT_VECTOR(1, 4); + DECLARE_RESULT_VECTOR(1, 5); + DECLARE_RESULT_VECTOR(1, 6); + DECLARE_RESULT_VECTOR(1, 7); + + if (LIKELY(packed_a != NULL)) { + if (j == 0) { + for (; k < k4; k += 4) { + + VECTOR_LOAD_B_K4(0, 0); + VECTOR_LOAD_B_K4(1, 0); + VECTOR_LOAD_B_K4(2, 0); + VECTOR_LOAD_B_K4(3, 0); + TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); + SCALE_B4_K4(0, 0, 1, 2, 3); + + GATHER_LOAD_A64(pg_true, 0, 0); + GATHER_LOAD_A64(pg_true, 1, 0); + svfloat32_t as0_k0 = svuzp1(svreinterpret_f32(at0_k0), svreinterpret_f32(at1_k0)); + svfloat32_t as0_k1 = svuzp2(svreinterpret_f32(at0_k0), svreinterpret_f32(at1_k0)); + VECTOR_PACK_A(0, 0); + VECTOR_PACK_A(0, 1); + + // GATHER_LOAD_A(pg_true, 0, 0); + // VECTOR_PACK_A(0, 0); + // GATHER_LOAD_A(pg_true, 0, 1); + // VECTOR_PACK_A(0, 1); + + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); + + GATHER_LOAD_A64(pg_true, 0, 2); + GATHER_LOAD_A64(pg_true, 1, 2); + svfloat32_t as0_k2 = svuzp1(svreinterpret_f32(at0_k2), svreinterpret_f32(at1_k2)); + svfloat32_t as0_k3 = svuzp2(svreinterpret_f32(at0_k2), svreinterpret_f32(at1_k2)); + VECTOR_PACK_A(0, 2); + VECTOR_PACK_A(0, 3); + + // GATHER_LOAD_A(pg_true, 0, 2); + // VECTOR_PACK_A(0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); + // GATHER_LOAD_A(pg_true, 0, 3); + // VECTOR_PACK_A(0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); + VECTOR_LOAD_B_K4(4, 0); + VECTOR_LOAD_B_K4(5, 0); + VECTOR_LOAD_B_K4(6, 0); + VECTOR_LOAD_B_K4(7, 0); + TRANSPOSE_B4_K4(4, 5, 6, 7, 0, 1, 2, 3); + SCALE_B4_K4(4, 0, 1, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 3); + + GATHER_LOAD_A64(pg_true, 2, 0); + GATHER_LOAD_A64(pg_true, 3, 0); + svfloat32_t as1_k0 = svuzp1(svreinterpret_f32(at2_k0), svreinterpret_f32(at3_k0)); + svfloat32_t as1_k1 = svuzp2(svreinterpret_f32(at2_k0), svreinterpret_f32(at3_k0)); + VECTOR_PACK_A(1, 0); + VECTOR_PACK_A(1, 1); + + // GATHER_LOAD_A(pg_true, 1, 0); + // VECTOR_PACK_A(1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); + // GATHER_LOAD_A(pg_true, 1, 1); + // VECTOR_PACK_A(1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 1); + + // 64-bit load x2 then unzip into 32-bit + GATHER_LOAD_A64(pg_true, 2, 2); + GATHER_LOAD_A64(pg_true, 3, 2); + svfloat32_t as1_k2 = svuzp1(svreinterpret_f32(at2_k2), svreinterpret_f32(at3_k2)); + svfloat32_t as1_k3 = svuzp2(svreinterpret_f32(at2_k2), svreinterpret_f32(at3_k2)); + VECTOR_PACK_A(1, 2); + VECTOR_PACK_A(1, 3); + + // GATHER_LOAD_A(pg_true, 1, 2); + // VECTOR_PACK_A(1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 2); + // GATHER_LOAD_A(pg_true, 1, 3); + // VECTOR_PACK_A(1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 3); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + VECTOR_PACK_A(0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + GATHER_LOAD_A(pg_true, 1, 0); + VECTOR_PACK_A(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); + BROADCAST_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 4, 0); + BROADCAST_LOAD_B(5, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 5, 0); + BROADCAST_LOAD_B(6, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 6, 0); + BROADCAST_LOAD_B(7, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 7, 0); + } + } else { + for (; k < k4; k += 4) { + + VECTOR_LOAD_B_K4(0, 0); + VECTOR_LOAD_B_K4(1, 0); + VECTOR_LOAD_B_K4(2, 0); + VECTOR_LOAD_B_K4(3, 0); + TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); + SCALE_B4_K4(0, 0, 1, 2, 3); + UNPACK_VECTOR_A(0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + UNPACK_VECTOR_A(0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); + UNPACK_VECTOR_A(0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); + UNPACK_VECTOR_A(0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); + VECTOR_LOAD_B_K4(4, 0); + VECTOR_LOAD_B_K4(5, 0); + VECTOR_LOAD_B_K4(6, 0); + VECTOR_LOAD_B_K4(7, 0); + TRANSPOSE_B4_K4(4, 5, 6, 7, 0, 1, 2, 3); + SCALE_B4_K4(4, 0, 1, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 3); + UNPACK_VECTOR_A(1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); + UNPACK_VECTOR_A(1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 1); + UNPACK_VECTOR_A(1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 2); + UNPACK_VECTOR_A(1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 3); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + UNPACK_VECTOR_A(0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + UNPACK_VECTOR_A(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); + BROADCAST_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 4, 0); + BROADCAST_LOAD_B(5, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 5, 0); + BROADCAST_LOAD_B(6, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 6, 0); + BROADCAST_LOAD_B(7, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 7, 0); + } + } + } else { + for (; k < k4; k += 4) { + + VECTOR_LOAD_B_K4(0, 0); + VECTOR_LOAD_B_K4(1, 0); + VECTOR_LOAD_B_K4(2, 0); + VECTOR_LOAD_B_K4(3, 0); + TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); + SCALE_B4_K4(0, 0, 1, 2, 3); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + GATHER_LOAD_A(pg_true, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); + GATHER_LOAD_A(pg_true, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); + GATHER_LOAD_A(pg_true, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); + VECTOR_LOAD_B_K4(4, 0); + VECTOR_LOAD_B_K4(5, 0); + VECTOR_LOAD_B_K4(6, 0); + VECTOR_LOAD_B_K4(7, 0); + TRANSPOSE_B4_K4(4, 5, 6, 7, 0, 1, 2, 3); + SCALE_B4_K4(4, 0, 1, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 3); + GATHER_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); + GATHER_LOAD_A(pg_true, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 1); + GATHER_LOAD_A(pg_true, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 2); + GATHER_LOAD_A(pg_true, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 3); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + GATHER_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); + BROADCAST_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 4, 0); + BROADCAST_LOAD_B(5, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 5, 0); + BROADCAST_LOAD_B(6, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 6, 0); + BROADCAST_LOAD_B(7, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 7, 0); + } + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + VECTOR_STORE(pg_true, 0, 4); + VECTOR_STORE(pg_true, 0, 5); + VECTOR_STORE(pg_true, 0, 6); + VECTOR_STORE(pg_true, 0, 7); + VECTOR_STORE(pg_true, 1, 0); + VECTOR_STORE(pg_true, 1, 1); + VECTOR_STORE(pg_true, 1, 2); + VECTOR_STORE(pg_true, 1, 3); + VECTOR_STORE(pg_true, 1, 4); + VECTOR_STORE(pg_true, 1, 5); + VECTOR_STORE(pg_true, 1, 6); + VECTOR_STORE(pg_true, 1, 7); + } + for (; j < n4; j += 4) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(1, 1); + DECLARE_RESULT_VECTOR(1, 2); + DECLARE_RESULT_VECTOR(1, 3); + + if (LIKELY(packed_a != NULL)) { + for (; k < k4; k += 4) { + + VECTOR_LOAD_B_K4(0, 0); + VECTOR_LOAD_B_K4(1, 0); + VECTOR_LOAD_B_K4(2, 0); + VECTOR_LOAD_B_K4(3, 0); + TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); + SCALE_B4_K4(0, 0, 1, 2, 3); + UNPACK_VECTOR_A(0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + UNPACK_VECTOR_A(0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); + UNPACK_VECTOR_A(0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); + UNPACK_VECTOR_A(0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); + UNPACK_VECTOR_A(1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + UNPACK_VECTOR_A(1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1); + UNPACK_VECTOR_A(1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2); + UNPACK_VECTOR_A(1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + UNPACK_VECTOR_A(0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + UNPACK_VECTOR_A(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); + } + } else { + for (; k < k4; k += 4) { + + VECTOR_LOAD_B_K4(0, 0); + VECTOR_LOAD_B_K4(1, 0); + VECTOR_LOAD_B_K4(2, 0); + VECTOR_LOAD_B_K4(3, 0); + TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); + SCALE_B4_K4(0, 0, 1, 2, 3); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + GATHER_LOAD_A(pg_true, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); + GATHER_LOAD_A(pg_true, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); + GATHER_LOAD_A(pg_true, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); + GATHER_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + GATHER_LOAD_A(pg_true, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1); + GATHER_LOAD_A(pg_true, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2); + GATHER_LOAD_A(pg_true, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + GATHER_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); + } + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + VECTOR_STORE(pg_true, 1, 0); + VECTOR_STORE(pg_true, 1, 1); + VECTOR_STORE(pg_true, 1, 2); + VECTOR_STORE(pg_true, 1, 3); + } + for (; j < N; j++) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(1, 0); + + if (LIKELY(packed_a != NULL)) { + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + UNPACK_VECTOR_A(0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + UNPACK_VECTOR_A(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + } + } else { + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + GATHER_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + } + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 1, 0); + } + } + for (; i < v_m1; i += v_size) { + + BLASLONG j = 0; + for (; j < n8; j += 8) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(0, 4); + DECLARE_RESULT_VECTOR(0, 5); + DECLARE_RESULT_VECTOR(0, 6); + DECLARE_RESULT_VECTOR(0, 7); + + for (; k < k4; k += 4) { + + VECTOR_LOAD_B_K4(0, 0); + VECTOR_LOAD_B_K4(1, 0); + VECTOR_LOAD_B_K4(2, 0); + VECTOR_LOAD_B_K4(3, 0); + TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); + SCALE_B4_K4(0, 0, 1, 2, 3); + + GATHER_LOAD_A64(pg_true, 0, 0); + GATHER_LOAD_A64(pg_true, 1, 0); + svfloat32_t as0_k0 = svuzp1(svreinterpret_f32(at0_k0), svreinterpret_f32(at1_k0)); + svfloat32_t as0_k1 = svuzp2(svreinterpret_f32(at0_k0), svreinterpret_f32(at1_k0)); + + // GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + // GATHER_LOAD_A(pg_true, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); + + GATHER_LOAD_A64(pg_true, 0, 2); + GATHER_LOAD_A64(pg_true, 1, 2); + svfloat32_t as0_k2 = svuzp1(svreinterpret_f32(at0_k2), svreinterpret_f32(at1_k2)); + svfloat32_t as0_k3 = svuzp2(svreinterpret_f32(at0_k2), svreinterpret_f32(at1_k2)); + + // GATHER_LOAD_A(pg_true, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); + // GATHER_LOAD_A(pg_true, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); + VECTOR_LOAD_B_K4(4, 0); + VECTOR_LOAD_B_K4(5, 0); + VECTOR_LOAD_B_K4(6, 0); + VECTOR_LOAD_B_K4(7, 0); + TRANSPOSE_B4_K4(4, 5, 6, 7, 0, 1, 2, 3); + SCALE_B4_K4(4, 0, 1, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 3); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + BROADCAST_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0); + BROADCAST_LOAD_B(5, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0); + BROADCAST_LOAD_B(6, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0); + BROADCAST_LOAD_B(7, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + VECTOR_STORE(pg_true, 0, 4); + VECTOR_STORE(pg_true, 0, 5); + VECTOR_STORE(pg_true, 0, 6); + VECTOR_STORE(pg_true, 0, 7); + } + for (; j < n4; j += 4) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + + for (; k < k4; k += 4) { + + VECTOR_LOAD_B_K4(0, 0); + VECTOR_LOAD_B_K4(1, 0); + VECTOR_LOAD_B_K4(2, 0); + VECTOR_LOAD_B_K4(3, 0); + TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); + SCALE_B4_K4(0, 0, 1, 2, 3); + + GATHER_LOAD_A64(pg_true, 0, 0); + GATHER_LOAD_A64(pg_true, 1, 0); + svfloat32_t as0_k0 = svuzp1(svreinterpret_f32(at0_k0), svreinterpret_f32(at1_k0)); + svfloat32_t as0_k1 = svuzp2(svreinterpret_f32(at0_k0), svreinterpret_f32(at1_k0)); + + // GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + // GATHER_LOAD_A(pg_true, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); + + GATHER_LOAD_A64(pg_true, 0, 2); + GATHER_LOAD_A64(pg_true, 1, 2); + svfloat32_t as0_k2 = svuzp1(svreinterpret_f32(at0_k2), svreinterpret_f32(at1_k2)); + svfloat32_t as0_k3 = svuzp2(svreinterpret_f32(at0_k2), svreinterpret_f32(at1_k2)); + + // GATHER_LOAD_A(pg_true, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); + // GATHER_LOAD_A(pg_true, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + } + for (; j < N; j++) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + } + VECTOR_STORE(pg_true, 0, 0); + } + } + for (; i < M; i += v_size) { + const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M)); + + BLASLONG j = 0; + for (; j < n8; j += 8) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(0, 4); + DECLARE_RESULT_VECTOR(0, 5); + DECLARE_RESULT_VECTOR(0, 6); + DECLARE_RESULT_VECTOR(0, 7); + + for (; k < k4; k += 4) { + + VECTOR_LOAD_B_K4(0, 0); + VECTOR_LOAD_B_K4(1, 0); + VECTOR_LOAD_B_K4(2, 0); + VECTOR_LOAD_B_K4(3, 0); + TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); + SCALE_B4_K4(0, 0, 1, 2, 3); + GATHER_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + GATHER_LOAD_A(pg_tail, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); + GATHER_LOAD_A(pg_tail, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); + GATHER_LOAD_A(pg_tail, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); + VECTOR_LOAD_B_K4(4, 0); + VECTOR_LOAD_B_K4(5, 0); + VECTOR_LOAD_B_K4(6, 0); + VECTOR_LOAD_B_K4(7, 0); + TRANSPOSE_B4_K4(4, 5, 6, 7, 0, 1, 2, 3); + SCALE_B4_K4(4, 0, 1, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 3); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 3, 0); + BROADCAST_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 4, 0); + BROADCAST_LOAD_B(5, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 5, 0); + BROADCAST_LOAD_B(6, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 6, 0); + BROADCAST_LOAD_B(7, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 7, 0); + } + VECTOR_STORE(pg_tail, 0, 0); + VECTOR_STORE(pg_tail, 0, 1); + VECTOR_STORE(pg_tail, 0, 2); + VECTOR_STORE(pg_tail, 0, 3); + VECTOR_STORE(pg_tail, 0, 4); + VECTOR_STORE(pg_tail, 0, 5); + VECTOR_STORE(pg_tail, 0, 6); + VECTOR_STORE(pg_tail, 0, 7); + } + for (; j < n4; j += 4) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + + for (; k < k4; k += 4) { + + VECTOR_LOAD_B_K4(0, 0); + VECTOR_LOAD_B_K4(1, 0); + VECTOR_LOAD_B_K4(2, 0); + VECTOR_LOAD_B_K4(3, 0); + TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); + SCALE_B4_K4(0, 0, 1, 2, 3); + GATHER_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + GATHER_LOAD_A(pg_tail, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); + GATHER_LOAD_A(pg_tail, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); + GATHER_LOAD_A(pg_tail, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 3, 0); + } + VECTOR_STORE(pg_tail, 0, 0); + VECTOR_STORE(pg_tail, 0, 1); + VECTOR_STORE(pg_tail, 0, 2); + VECTOR_STORE(pg_tail, 0, 3); + } + for (; j < N; j++) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); + } + VECTOR_STORE(pg_tail, 0, 0); + } + } + + if (pack_a) + free(packed_a); + + return 0; +} diff --git a/kernel/arm64/sgemm_small_kernel_tt_sve.c b/kernel/arm64/sgemm_small_kernel_tt_sve.c new file mode 100644 index 0000000000..894e7fd46a --- /dev/null +++ b/kernel/arm64/sgemm_small_kernel_tt_sve.c @@ -0,0 +1,574 @@ +/*************************************************************************** +Copyright (c) 2024, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "common.h" + +#include +#include +#ifdef __ARM_NEON_SVE_BRIDGE +#include +#else +#define svdup_neonq_f32(fixed_reg) \ + ({ \ + svfloat32_t scalable_reg; \ + asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \ + scalable_reg; \ + }) +#define svdup_neonq_f64(fixed_reg) \ + ({ \ + svfloat64_t scalable_reg; \ + asm("mov %0.q, %q1" : "=w"(scalable_reg) : "w"(fixed_reg) :); \ + scalable_reg; \ + }) +#endif + +#define A_ELEMENT_K(m, offset_k) A[(i + (m)) * lda + (k + offset_k)] +#define A_ELEMENT(m) A_ELEMENT_K(m, 0) + +#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) * ldb + (j + (n))] +#define B_ELEMENT(n) B_ELEMENT_K(n, 0) + +#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc] + +#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * v_size2 + m] +#define PACK_ELEMENT(m) PACK_ELEMENT_K(m, 0) + +// ASIMD +#define DECLARE_RESULT_VECTOR4(m, n) \ + float32x4_t result##m##n = vdupq_n_f32(0.0); +#define DECLARE_RESULT(m, n) float32_t result##m##n = 0.0; +#define BROADCAST_LOAD_A4(m, offset_k) \ + float32x4_t a##m##_k##offset_k = vld1q_dup_f32(&A_ELEMENT_K(m, offset_k)); +#define LOAD_A1(m, offset_k) \ + float32_t a##m##_k##offset_k = A_ELEMENT_K(m, offset_k); +#define VECTOR_LOAD_B4(n, offset_k) \ + float32x4_t b##n##_k##offset_k = vld1q_f32(&B_ELEMENT_K(n, offset_k)); +#define GATHER_LOAD_B4(n, offset_k) \ + float32x4_t b##n##_k##offset_k = vdupq_n_f32(B_ELEMENT_K(n, offset_k)); \ + b##n##_k##offset_k = \ + vsetq_lane_f32(B_ELEMENT_K(n + 1, offset_k), b##n##_k##offset_k, 1); \ + b##n##_k##offset_k = \ + vsetq_lane_f32(B_ELEMENT_K(n + 2, offset_k), b##n##_k##offset_k, 2); \ + b##n##_k##offset_k = \ + vsetq_lane_f32(B_ELEMENT_K(n + 3, offset_k), b##n##_k##offset_k, 3); +#define VECTOR_UNPACK_B4(n, offset_k) \ + float32x4_t b##n##_k##offset_k = vld1q_f32(&PACK_ELEMENT_K(n, offset_k)); +#define VECTOR_PACK_B4(n, offset_k) \ + vst1q_f32(&PACK_ELEMENT_K(n, offset_k), b##n##_k##offset_k); +#define PACK_B0(n, offset_k) \ + PACK_ELEMENT_K(n, offset_k) = vget_lane_f32(b##n##_k##offset_k, 0); +#define UPDATE_RESULT_VECTOR4(m, n, offset_k) \ + result##m##n = \ + vfmaq_f32(result##m##n, a##m##_k##offset_k, b##n##_k##offset_k); +#define UPDATE_RESULT(m, n, offset_k) \ + result##m##n = result##m##n + a##m##_k##offset_k * b##n##_k##offset_k; +#ifdef B0 +#define VECTOR_STORE4(m, n) \ + vst1q_f32(&C_ELEMENT(m, n), vmulq_f32(result##m##n, vdupq_n_f32(alpha))); +#define STORE(m, n) C_ELEMENT(m, n) = alpha * result##m##n; +#else +#define VECTOR_STORE4(m, n) \ + result##m##n = vmulq_f32(result##m##n, vdupq_n_f32(alpha)); \ + result##m##n = \ + vfmaq_f32(result##m##n, vld1q_f32(&C_ELEMENT(m, n)), vdupq_n_f32(beta)); \ + vst1q_f32(&C_ELEMENT(m, n), result##m##n); +#define STORE(m, n) \ + C_ELEMENT(m, n) = C_ELEMENT(m, n) * beta + alpha * result##m##n; +#endif + +// SVE +#define DECLARE_RESULT_VECTOR(m, n) svfloat32_t result##m##n = svdup_f32(0.0); +#define BROADCAST_LOAD_A(m, offset_k) \ + svfloat32_t a##s##m##_k##offset_k = svdup_f32(A_ELEMENT_K(m, offset_k)); +#define BROADCAST_LOAD_B(n, offset_k) \ + svfloat32_t b##s##n##_k##offset_k = svdup_f32(B_ELEMENT_K(n, offset_k)); + +#define QUADWORD_LOAD_B(n, offset_k) \ + svfloat32_t b##s##n##_k##offset_k = \ + svld1rq(pg_true, &B_ELEMENT_K(n, offset_k)); +#define GATHER_LOAD_A(pg, offset_m, m, offset_k) \ + svfloat32_t a##s##m##_k##offset_k = \ + svld1_gather_index(pg, &A_ELEMENT_K(offset_m, offset_k), lda_vec); +#define PACK_A(m, offset_k) \ + svst1(pg_first, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k); +#define VECTOR_PACK_A(offset_m, m, offset_k) \ + svst1(pg_true, &PACK_ELEMENT_K(m* v_size, offset_k), a##s##m##_k##offset_k); +#define QUADWORD_PACK_A(m, offset_k) \ + svst1(pg_quad, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k); +#define UNPACK_VECTOR_A(offset_m, m, offset_k) \ + svfloat32_t a##s##m##_k##offset_k = \ + svld1(pg_true, &PACK_ELEMENT_K(offset_m, offset_k)); +#define UNPACK_BROADCAST_A(m, offset_k) \ + svfloat32_t a##s##m##_k##offset_k = svdup_f32(PACK_ELEMENT_K(m, offset_k)); +#define UNPACK_QUADWORD_A(m, offset_k) \ + svfloat32_t a##s##m##_k##offset_k = \ + svld1rq(pg_true, &PACK_ELEMENT_K(m, offset_k)); +#define UPDATE_RESULT_VECTOR(pg, m, n, offset_k) \ + result##m##n = \ + svmla_m(pg, result##m##n, a##s##m##_k##offset_k, b##s##n##_k##offset_k); +#define UPDATE_RESULT_VECTOR_QUADWORD(m, n, outer, lane, offset_k) \ + result##m##n = svmla_lane( \ + result##m##n, a##s##m##_k##offset_k, b##s##outer##_k##offset_k, lane); +#ifdef B0 +#define VECTOR_STORE(pg, offset_m, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + svst1(pg, &C_ELEMENT(offset_m, n), result##m##n); +#define SCATTER_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + svst1_scatter_index( \ + pg, &C_ELEMENT(v_size* m, n), svindex_u32(0LL, ldc), result##m##n); +#else +#define VECTOR_STORE(pg, offset_m, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + result##m##n = \ + svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(offset_m, n)), beta_vec); \ + svst1(pg, &C_ELEMENT(offset_m, n), result##m##n); +#define SCATTER_STORE(pg, m, n) \ + result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ + result##m##n = svmla_m( \ + pg, \ + result##m##n, \ + svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u32(0LL, ldc)), \ + beta_vec); \ + svst1_scatter_index( \ + pg, &C_ELEMENT(v_size* m, n), svindex_u32(0LL, ldc), result##m##n); +#endif + +#ifndef LIKELY +#ifdef __GNUC__ +#define LIKELY(x) __builtin_expect(!!(x), 1) +#else +#define LIKELY(x) (x) +#endif +#endif +#ifndef UNLIKELY +#ifdef __GNUC__ +#define UNLIKELY(x) __builtin_expect(!!(x), 0) +#else +#define UNLIKELY(x) (x) +#endif +#endif + +#ifdef B0 +int +CNAME(BLASLONG M, + BLASLONG N, + BLASLONG K, + IFLOAT* A, + BLASLONG lda, + FLOAT alpha, + IFLOAT* B, + BLASLONG ldb, + FLOAT* C, + BLASLONG ldc) +#else +int +CNAME(BLASLONG M, + BLASLONG N, + BLASLONG K, + IFLOAT* A, + BLASLONG lda, + FLOAT alpha, + IFLOAT* B, + BLASLONG ldb, + FLOAT beta, + FLOAT* C, + BLASLONG ldc) +#endif +{ + const uint64_t v_size = svcntw(); + const uint64_t v_size2 = v_size * 2; + const svbool_t pg_true = svptrue_b32(); + const svfloat32_t alpha_vec = svdup_f32(alpha); +#ifndef B0 + const svfloat32_t beta_vec = svdup_f32(beta); +#endif + const svuint32_t lda_vec = svindex_u32(0LL, lda); + + const BLASLONG v_m2 = M & -v_size2; + const BLASLONG v_m1 = M & -v_size; + const BLASLONG n8 = N & -8; + const BLASLONG n4 = N & -4; + + const int pack_a = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0; + FLOAT* packed_a = + (pack_a) ? packed_a = (FLOAT*)malloc(K * 2 * v_size * sizeof(FLOAT)) : NULL; + + BLASLONG i = 0; + for (; i < v_m2; i += v_size2) { + + BLASLONG j = 0; + for (; j < n8; j += 8) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(0, 4); + DECLARE_RESULT_VECTOR(0, 5); + DECLARE_RESULT_VECTOR(0, 6); + DECLARE_RESULT_VECTOR(0, 7); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(1, 1); + DECLARE_RESULT_VECTOR(1, 2); + DECLARE_RESULT_VECTOR(1, 3); + DECLARE_RESULT_VECTOR(1, 4); + DECLARE_RESULT_VECTOR(1, 5); + DECLARE_RESULT_VECTOR(1, 6); + DECLARE_RESULT_VECTOR(1, 7); + + if (LIKELY(packed_a != NULL)) { + if (j == 0) { + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0, 0); + VECTOR_PACK_A(0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + QUADWORD_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + GATHER_LOAD_A(pg_true, v_size, 1, 0); + VECTOR_PACK_A(v_size, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); + } + } else { + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + UNPACK_VECTOR_A(0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + QUADWORD_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + UNPACK_VECTOR_A(v_size, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); + } + } + } else { + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + QUADWORD_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + GATHER_LOAD_A(pg_true, v_size, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); + } + } + VECTOR_STORE(pg_true, 0, 0, 0); + VECTOR_STORE(pg_true, 0, 0, 1); + VECTOR_STORE(pg_true, 0, 0, 2); + VECTOR_STORE(pg_true, 0, 0, 3); + VECTOR_STORE(pg_true, 0, 0, 4); + VECTOR_STORE(pg_true, 0, 0, 5); + VECTOR_STORE(pg_true, 0, 0, 6); + VECTOR_STORE(pg_true, 0, 0, 7); + VECTOR_STORE(pg_true, v_size, 1, 0); + VECTOR_STORE(pg_true, v_size, 1, 1); + VECTOR_STORE(pg_true, v_size, 1, 2); + VECTOR_STORE(pg_true, v_size, 1, 3); + VECTOR_STORE(pg_true, v_size, 1, 4); + VECTOR_STORE(pg_true, v_size, 1, 5); + VECTOR_STORE(pg_true, v_size, 1, 6); + VECTOR_STORE(pg_true, v_size, 1, 7); + } + for (; j < n4; j += 4) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(1, 1); + DECLARE_RESULT_VECTOR(1, 2); + DECLARE_RESULT_VECTOR(1, 3); + + if (LIKELY(packed_a != NULL)) { + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + UNPACK_VECTOR_A(0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + UNPACK_VECTOR_A(v_size, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + } + } else { + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + GATHER_LOAD_A(pg_true, v_size, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + } + } + VECTOR_STORE(pg_true, 0, 0, 0); + VECTOR_STORE(pg_true, 0, 0, 1); + VECTOR_STORE(pg_true, 0, 0, 2); + VECTOR_STORE(pg_true, 0, 0, 3); + VECTOR_STORE(pg_true, v_size, 1, 0); + VECTOR_STORE(pg_true, v_size, 1, 1); + VECTOR_STORE(pg_true, v_size, 1, 2); + VECTOR_STORE(pg_true, v_size, 1, 3); + } + for (; j < N; j++) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(1, 0); + + if (LIKELY(packed_a != NULL)) { + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + UNPACK_VECTOR_A(0, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + UNPACK_VECTOR_A(v_size, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + } + } else { + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + GATHER_LOAD_A(pg_true, v_size, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + } + } + VECTOR_STORE(pg_true, 0, 0, 0); + VECTOR_STORE(pg_true, v_size, 1, 0); + } + } + for (; i < v_m1; i += v_size) { + + BLASLONG j = 0; + for (; j < n8; j += 8) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(0, 4); + DECLARE_RESULT_VECTOR(0, 5); + DECLARE_RESULT_VECTOR(0, 6); + DECLARE_RESULT_VECTOR(0, 7); + + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + QUADWORD_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + } + VECTOR_STORE(pg_true, 0, 0, 0); + VECTOR_STORE(pg_true, 0, 0, 1); + VECTOR_STORE(pg_true, 0, 0, 2); + VECTOR_STORE(pg_true, 0, 0, 3); + VECTOR_STORE(pg_true, 0, 0, 4); + VECTOR_STORE(pg_true, 0, 0, 5); + VECTOR_STORE(pg_true, 0, 0, 6); + VECTOR_STORE(pg_true, 0, 0, 7); + } + for (; j < n4; j += 4) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + } + VECTOR_STORE(pg_true, 0, 0, 0); + VECTOR_STORE(pg_true, 0, 0, 1); + VECTOR_STORE(pg_true, 0, 0, 2); + VECTOR_STORE(pg_true, 0, 0, 3); + } + for (; j < N; j++) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + } + VECTOR_STORE(pg_true, 0, 0, 0); + } + } + for (; i < M; i += v_size) { + const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M)); + + BLASLONG j = 0; + for (; j < n8; j += 8) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(0, 4); + DECLARE_RESULT_VECTOR(0, 5); + DECLARE_RESULT_VECTOR(0, 6); + DECLARE_RESULT_VECTOR(0, 7); + + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + GATHER_LOAD_A(pg_tail, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + QUADWORD_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + } + VECTOR_STORE(pg_tail, 0, 0, 0); + VECTOR_STORE(pg_tail, 0, 0, 1); + VECTOR_STORE(pg_tail, 0, 0, 2); + VECTOR_STORE(pg_tail, 0, 0, 3); + VECTOR_STORE(pg_tail, 0, 0, 4); + VECTOR_STORE(pg_tail, 0, 0, 5); + VECTOR_STORE(pg_tail, 0, 0, 6); + VECTOR_STORE(pg_tail, 0, 0, 7); + } + for (; j < n4; j += 4) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + GATHER_LOAD_A(pg_tail, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + } + VECTOR_STORE(pg_tail, 0, 0, 0); + VECTOR_STORE(pg_tail, 0, 0, 1); + VECTOR_STORE(pg_tail, 0, 0, 2); + VECTOR_STORE(pg_tail, 0, 0, 3); + } + for (; j < N; j++) { + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_tail, 0, 0, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); + } + VECTOR_STORE(pg_tail, 0, 0, 0); + } + } + + if (pack_a) + free(packed_a); + + return 0; +} From 0bb973976ba68c3108038431ca5f322094967c3e Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Fri, 4 Aug 2023 08:21:01 +0100 Subject: [PATCH 002/267] Squash commit of GitHub wiki MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Created Installation Guide (markdown) * Updated quick installation (markdown) * Updated Home (markdown) * Updated Document (markdown) * Updated Document (markdown) * Updated Document (markdown) * Created Installation Guide (markdown) * Created Home (markdown) * Init version * Updated OpenBLAS Wiki (markdown) * Updated OpenBLAS Wiki (markdown) * Updated OpenBLAS Wiki (markdown) * Updated Document (markdown) * Updated Installation Guide (markdown) * Updated Installation Guide (markdown) * Created Download (markdown) * Created Faq (markdown) * Updated Faq (markdown) * Updated FAQ * Created How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated Document (markdown) * Updated Faq (markdown) * Updated Faq (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated Faq (markdown) * Updated OpenBLAS Wiki (markdown) * Updated Home (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Created How to generate import library for MingW (markdown) * Updated Document (markdown) * Updated Faq (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Build instrunctions for FreeBSD * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated Installation Guide (markdown) * Updated Faq (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * minor edits * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated Faq (markdown) * Installation instructions for Windows * Updated Faq (markdown) * G77 conventions no longer needed with GCC 4.7+ * Updated Home (markdown) * Document why issue 168 occurred. * Updated Home (markdown) * Created Publications (markdown) * Updated Home (markdown) * Updated Document (markdown) * Updated Faq (markdown) * Updated Download (markdown) * Updated Publications (markdown) * Updated Faq (markdown) * Updated Document (markdown) * Revert 7580d38ffad37e6613e6304707aaaa681f3d78c2 ... b1bd4ff37d2106bbd5c4730a08dbb789cc44e7d4 * Created Mailing List (markdown) * Updated Mailing List (markdown) * Updated Mailing List (markdown) * Updated Home (markdown) * Updated Document (markdown) * Updated Publications (markdown) * Updated Download (markdown) * Updated Faq (markdown) * Updated Home (markdown) * Updated Faq (markdown) * Updated Home (markdown) * Revert b69f1417cdf8820be046cc27a2b96b42a25bc3a3 ... 90a227c317c3572ced943461ac3a252c40790f44 on Home * Updated Home (markdown) * Updated Publications (markdown) * Updated Faq (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * We already ensure the stack alignment in Makefile.system for Win32. * Updated Faq (markdown) * Updated Faq (markdown) * Updated Publications (markdown) * Created Donation (markdown) * Updated Home (markdown) * Updated Document (markdown) * Updated Faq (markdown) * Updated Publications (markdown) * Updated Download (markdown) * Updated Mailing List (markdown) * Updated Donation (markdown) * Updated Download (markdown) * Updated Donation (markdown) * Updated Donation (markdown) * Updated Donation (markdown) * Updated Donation (markdown) * Updated Home (markdown) * Updated Faq (markdown) * Updated Download (markdown) * Updated Home (markdown) * Updated Home (markdown) * Add new entry for static linking and pthread. * Fix named anchors (see http://stackoverflow.com/questions/5319754/cross-reference-named-anchor-in-markdown/7335259#7335259) * Created Related packages that use OpenBLAS (markdown) * Updated Related packages that use OpenBLAS (markdown) * Updated Related packages that use OpenBLAS (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated Document (markdown) * Created To-do List (markdown) * Updated To do List (markdown) * Updated Fixed optimized kernels To do List (markdown) * Fix English idiom * Remove trailing whitespace * Updated Fixed optimized kernels To do List (markdown) * Updated Fixed optimized kernels To do List (markdown) * Updated Fixed optimized kernels To do List (markdown) * Updated Fixed optimized kernels To do List (markdown) * Updated Faq (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated Related packages that use OpenBLAS (markdown) * Updated Related packages that use OpenBLAS (markdown) * Created Machine List (markdown) * Updated Document (markdown) * Updated Installation Guide (markdown) * Created User Manual (markdown) * Updated User Manual (markdown) * Updated Document (markdown) * Updated User Manual (markdown) * Updated User Manual (markdown) * Updated User Manual (markdown) * Updated User Manual (markdown) * Updated Related packages that use OpenBLAS (markdown) * Updated Faq (markdown) * Updated Related packages that use OpenBLAS (markdown) * Updated Machine List (markdown) * Updated Related packages that use OpenBLAS (markdown) * Updated Related packages that use OpenBLAS (markdown) * Add a note about building in QEMU * Updated Home (markdown) * Updated Faq (markdown) * update for allocating too many meory error. * Updated Faq (markdown) * Updated Faq (markdown) * Updated Installation Guide (markdown) * Updated Faq (markdown) * Init function doc * Updated Document (markdown) * Updated User Manual (markdown) * Updated User Manual (markdown) * Created How to build OpenBLAS for Android (markdown) * Updated How to build OpenBLAS for Android (markdown) * Updated Home (markdown) * Part of the description is really no clear, I add some more information, so it would be easier for VS user to fix the problems facing them. * Created Developer manual (markdown) * Updated Document (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * a typo, download ** frome -> download from * Updated Faq (markdown) * English (minor edit) * Updated Developer manual (markdown) * Updated Developer manual (markdown) * Updated Developer manual (markdown) * Updated Machine List (markdown) * Updated Developer manual (markdown) * Updated Developer manual (markdown) * Updated How to build OpenBLAS for Android (markdown) * Updated How to build OpenBLAS for Android (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * issue 842 * Updated How to build OpenBLAS for Android (markdown) * Updated How to build OpenBLAS for Android (markdown) * Updated How to build OpenBLAS for Android (markdown) * Updated How to build OpenBLAS for Android (markdown) * Added FC for building with Fortran * Change link for the Intel MKL documentation * Updated User Manual (markdown) * Updated User Manual (markdown) * Added MIPS build instructions from issue 949 * use TARGET_CFLAGS and TARGET_LDFLAGS instead of CFLAGS and LDFLAGS for linking OpenBLAS on ARMv7 * Add Windows updates (msys2,mingw/w64 merger), Android/MIPS pointers, qemu hint * Building libs & netlib targets to prevent errors in tests * Recipes not targets (for make) * Making only libs, not netlib (which also contains link/run tests...) * Copied from instructions by Ivan Ushakov, originally posted in issue 569 * Updated How to build OpenBLAS for iPhone iOS (markdown) * Updated Faq (markdown) * Created How to build OpenBLAS for iPhone iOS (markdown) * error code (0xc000007b) was missing a character * Updated How to build OpenBLAS for iPhone iOS (ARMv8) (markdown) * Updated How to build OpenBLAS for iPhone iOS (ARMv8) (markdown) * Revert 7e9dd0ebf079e002e3aa831fa671fde3e8cfad81...8d105c7be8cd447482f61e0295c0c146f5314eb5 on How to build OpenBLAS for iPhone iOS * Add guide on how to reversibly supplant Ubuntu LTS libblas.so.3 * typo * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated User Manual (markdown) * Updated Faq (markdown) * Updated Download (markdown) * Add perl to pacman package list * Fixed formatting on general questions * Copied from issue 1136 * Added instructions for building for Windows UWP. * To clear confusions vs super-fat-binaries that dont exist. * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Update for 0.2.20 (full builds, ARMv7 softfp support, newer NDKs using CLANG) * Updated How to build OpenBLAS for Android (markdown) * Fix some formatting issues * Updated How to build OpenBLAS for Android (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated How to build OpenBLAS for Android (markdown) * Created Precompiled installation packages (markdown) * Updated Precompiled installation packages (markdown) * Example - debian? * Mention (and link to) distribution-specific packages * Updated Installation Guide (markdown) * OpenSuSE (13.2, SLE included) * Updated Precompiled installation packages (markdown) * Updated Precompiled installation packages (markdown) * Make it look consistent. * Fedora+EPEL // maybe rpmbuild is too heavy * Updated Precompiled installation packages (markdown) * Updated Precompiled installation packages (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated Precompiled installation packages (markdown) * fix toolchain argument in armv8 clang build as per issue 1337 * add note about stdio.h not found error * Add flang instructions * Use the SVG Travis badge * homebrew option for OSX * Promote native MSVC builds with LLVM * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Direct people to the appropriate instructions * Add link to the Goto paper * Add CMAKE_BUILD_TYPE * Add note about having to specify AR on a Mac, from issue 1435 * Mention requirement to build a standalone toolchain in the clang section as well * added 'perl' to conda install command * homebrew/science was deprecated. This tap is now empty as all its formulae were migrated. * Added hint for "expected identifier" error message to mingw section following issue 1503 * Revert 9161c3b54281131e892dec739d888f35e6c59cf3...03f879be0c9e6a55705bc7efd5ee193299e04029 on How to use OpenBLAS in Microsoft Visual Studio * Revert to recommending mingw-w64 from sf.net and add note about issue 1503 * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Update MSVC installation procedure with info from issue 1521 * Add downgrade option for msys2 mingw compiler issue as suggested by econwang in issue 1503 * Add note about static linking bug with NDK 16 and API>22 * Updated Precompiled installation packages (markdown) * Updated Precompiled installation packages (markdown) * Updated Faq (markdown) * OBS is renamed and deep link format changed. Apparently recent SLE includes rpm by default too. * Add links to Conda-Forge and to staticfloat's builds for Julia * Mention _64 suffix appended to Julia builds with INTERFACE64 (issue 1617) * Fix unwanted markdown italicization * Add instruction to change to the generic sgemmkernel implementation from issue 1531 * Added hint about stack size requirements for running lapack-test from PR 1645; fixed markup of section headings * Add link to RvdG's publications page as a non-paywalled source of the "Goto paper" * Add section about non-suitability of the IBM XL compiler on POWER8 * Mention cmake version requirement in view or recent issues with link failures in utest etc. * Replace outdated entry for Sandybridge support with more general section on AVX512, Ryzen and GPU * Mention Apple Accelerate here as iOS build issue tickets usually die as soon as someone points out this option to the questioner. * Add section about unexpectedly using an older pre-installed version of the shared library (issue 1822) * fix markup of new entry * Mention perl and C compiler as prerequisites on the build host * Save WIP page * Updated Notes on parallelism and OpenBLAS (markdown) * Updated Notes on parallelism and OpenBLAS (markdown) * Updated Notes on parallelism and OpenBLAS (markdown) * Updated [WIP] Notes on parallelism and OpenBLAS (markdown) * Updated [WIP] Notes on parallelism and OpenBLAS (markdown) * Updated [WIP] Notes on parallelism and OpenBLAS (markdown) * Destroyed [WIP] Notes on parallelism and OpenBLAS (markdown) * Updated Faq (markdown) * Add small note on AVX512 for CentOS/RHEL section. * document the extension functions * formatting * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated Download (markdown) * Add brief general usage information from issue 1925 * Add link to Pete Warden blog article on GEMM rather than just deep-linking to a diagram from it * Document some of the less useful parameters from param.h * Updated Installation Guide (markdown) * Done with issue 2089 * Add note about changed library names for update-alternatives on Debian/Ubuntu * Updated Home (markdown) * Add note about using OpenBLAS with CUDA_HPL 2.3 from issue issue 909 * Fix typos in previous commit * Add pdb instructions fir cross-builds * Add note about generic QEMU CPUID clashing with existing P2(MMX) * typo * typo * C code syntax highlight * Updated multithreading section to introduce option USE_LOCKING (issue 2164) * Updated How to build OpenBLAS for iPhone iOS (ARMv8) (markdown) * Updated How to build OpenBLAS for iPhone iOS (ARMv8) (markdown) * Clarify Miniconda/cmake install instructions and redact outdated note about msys2 * Document cmake install step * Updated How to build OpenBLAS for Android (markdown) * Add solution for programs that look for libblas.so/liblapack.so * Add entry for powersaving modes on ARM boards (from issue 2540) * Add suggestion for speed problems on big.little systems from issue 2589 * Convert the ARMV8 big.little tidbit to a separate topic and update it with more details from the issue ticket * Add entry about problems caused by using the raw cblas.h (issue 2593) * complete quote symbol around CPATH environment variable * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Add note about running conda activate when working in a VS window (from issue 2637) * Add note about (not) compiling with -fbounds-check (ticket 2657) * Add entry about compile-time NUM_THREADS setting (issue 2678) * Added some sketchy description of adding cpuids for autodetection, adding targets and architectures * Markup and typo fixes * Add openblas_set_affinity from PR 2547 * Created _Footer (markdown) * Destroyed _Footer (markdown) * Add LAPACK-like SHGEMM to document the "official" status of the SH prefix * fix formatting of latest addition * Move outdated instructions for gcc-based NDK versions to the bottom, add hint about x86 builds * Add help for cpuid recognition failure * Update source tree layout & mention extraneous cpu paramerts * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Explain why pure VS builds are slower, and highlight that they do not support DYNAMIC_ARCH * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Mention fortran requirement and incompatibility of ifort with msvc * preliminary page for understanding the build system, needs a lot more work and input from more knowledgeable people than me * Updated Build system overview (markdown) * Updated WIP Build system overview (community made) (markdown) * add information for HOSTCC, HOST_CFLAGS * Added alternative script which was tested on OSX with latest NDK * added link to targets list * Updated WIP Build system overview (community made) (markdown) * Updated WIP Build system overview (community made) (markdown) * Updated WIP Build system overview (community made) (markdown) * Updated WIP Build system overview (community made) (markdown) * Updated WIP Build system overview (community made) (markdown) * Updated WIP Build system overview (community made) (markdown) * added script for x86_64 architecture * Updated WIP Build system overview (community made) (markdown) * Updated WIP Build system overview (community made) (markdown) * updated link to FLAME publications list * Created How to use OpenBLAS on Cortex-M (markdown) * Updated How to use OpenBLAS on Cortex M (markdown) * Updated Precompiled installation packages (markdown) * Updated How to use OpenBLAS on Cortex M (markdown) * Updated How to use OpenBLAS on Cortex M (markdown) * Updated How to use OpenBLAS on Cortex M (markdown) * Update source layout graph and start a short section on benchmarking to collect various pointers from the issue tracker * Add workaround for building with CMAKE on OSX * Use actual small headings to fix... weird bullet indent shit * Oops * Updated Faq (markdown) * Updated Faq (markdown) * Updated How to generate import library for MingW (markdown) * Updated How to generate import library for MinGW (markdown) * Updated How to generate import library for MinGW (markdown) * Updated How to generate import library for MinGW (markdown) * Updated How to generate import library for MinGW (markdown) * Updated How to generate import library for MinGW (markdown) * Updated How to generate import library for MinGW (markdown) * Updated How to generate import library for MinGW (markdown) * Updated How to generate import library for MinGW (markdown) * explicitly set CMAKE_MT to replace the new cmake default llvm-mt (failing) * Add -Wl,-rpath,/your_path/OpenBLAS/lib option to gcc linker line in "Link shared library" section + explanation for why it is needed/can be omitted. Also make note that -lgfortran not needed if only making LAPACKE calls. * Add note explaining that build flags passed to make should also be passed to make install * give example of install error * Describe how to build openblas library for win/arm64 targets * Add Xen to the existing entry for QEMU/KVM based on issue 3445 * Updated Download (markdown) * Updated Installation Guide (markdown) * Updated Installation Guide (markdown) * Revert b8da0e8523b898a2206d1e2fe99dbfb4ebb0ffa8...bc55aade759d2f925689b000828da249e1fc6a1a on Installation Guide * Revert b0c9a2ee060b8dd0b46b4c58375ef2a743c0363a...cecf8cf67963bd77a0bb97086e3a457a4cee11ff on Download * Revert bc55aade759d2f925689b000828da249e1fc6a1a...134894a0f09a0e92eef1b9a5c9e63f459d2db55e on Installation Guide * Add NDK23B example * Makes iOS build more robust * Double -isysroot * Bump up required devtoolset version for AVX-512 intrinsics. * Updated Installation Guide (markdown) * Updated How to build OpenBLAS for Windows on ARM64 (markdown) * Revert b8da0e8523b898a2206d1e2fe99dbfb4ebb0ffa8...75bba70832f8765faee693931c4a9e3eb6c84d98 on Installation Guide * Revert 75bba70832f8765faee693931c4a9e3eb6c84d98...d171e711a5cd8026b2eb507b249b5e51fa28b2a2 on Installation Guide * restore Windows link after malicious edit * Revert 1bcb03dcef85c675aace7f0a755d5aa36ec46eca...f732906434146b1a1ee82abe944a6d51d8f43b81 on Installation Guide * restore Windows link after malicious edit * Updated Installation Guide (markdown) * Bump up AVX-512 devtoolset because of identified packaging issues * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * n-dash html entity instead of - * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Add the bfloat16 functions * mention AXPBY * Update building for Apple M1 * Updated How to build OpenBLAS for Windows on ARM64 (markdown) * Created How to build OpenBLAS for macOS M1 / arm64 (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Add NO_AVX2 build hint for OSX Docker Desktop/xhyve (issues 2194 and 2244) * Mention the ELF offset/address bug from binutils 2.38 ld * moved issue 665 (sparse matrix/vector support) to a faq entry * Update and simplify based on CI experience and 3741 * Updated Download (markdown) * Updated How to build OpenBLAS for Windows on ARM64 (markdown) * Revert 0dcee87d486028fbd88c603853cdcae810e025c6...bf3d15e74d42b0b01618b4beb7b9d658fb905118 on Download * Revert a02f9e470f8e26eda1b8d8601ad2486557721ccf...c862aeb3492c29b487858d43c93676855b60a1f2 on How to build OpenBLAS for Windows on ARM64 * Updated How to use OpenBLAS in Microsoft Visual Studio (markdown) * Revert 9db97d11d88c801e8c5e9b8d6cc85fb44e5bca61...d2eb48810f3ecc1680900581473005f79c394ca4 on How to use OpenBLAS in Microsoft Visual Studio * start with the smallest configs, Appveyor and Cirrus * Updated CI jobs overview (markdown) * Add Azure CI * Add github workflows * Add the crossbuild parts of the dynamic_arch workflow * remove trailing separator * Add FreeBSD/Cirrus * Add ILP64 jobs on Cirrus * Add C910V and the OSUOSL Jenkins jobs (currently configured for my fork) * Updated Installation Guide (markdown) * Expand section on precompiled windows binaries to mention INTERFACE64=0 option * Remove reference to buildbot (domain reregistered to someone else, issue 4148 * Add OpenMP hints for mixed threads mode from issue 3186 * document NUM_PARALLEL (paraphrased from issue 1735) and expand other entries a bit * Mention use of llvm-ar rather than gcc-ar in recent NDKs and remove perl requirement * Add ?gemmt from -1.3.23/0.3.24 * note that LLVM is an optional install with VS2022 * clarify that all tools for the xbuild come with VS2022 * add instructions for cross-compiling from Windows/x86 (copied from issue #4459) Co-authored-by: Martin Kroeker Co-authored-by: xianyi Co-authored-by: Zhang Xianyi Co-authored-by: Andrew Co-authored-by: Elethom Co-authored-by: Isuru Fernando Co-authored-by: A. Tammy Co-authored-by: Andrew <16061801+brada4@users.noreply.github.com> Co-authored-by: Paul MUSTIÈRE Co-authored-by: TiborGY Co-authored-by: xoviat Co-authored-by: zchothia Co-authored-by: Eric Larson Co-authored-by: xoviat <49173759+xoviat@users.noreply.github.com> Co-authored-by: Kevin Yang Co-authored-by: Mavaddat Javid Co-authored-by: Derek Huang <37860662+phetdam@users.noreply.github.com> Co-authored-by: Iblis Lin Co-authored-by: Niyas Sait Co-authored-by: Roman Nazarevych Co-authored-by: rumiv <100173053+rumiv@users.noreply.github.com> Co-authored-by: Felix Yan Co-authored-by: Matti Picus Co-authored-by: Timothy Gu Co-authored-by: Yubin Wang Co-authored-by: dtidmarsh Co-authored-by: hninhninhtun <30315263+hninhninhtun@users.noreply.github.com> Co-authored-by: masel0 <96305063+masel0@users.noreply.github.com> Co-authored-by: meow464 <70211708+meow464@users.noreply.github.com> Co-authored-by: Ankush Chauhan Co-authored-by: Ashwin Sekhar T K Co-authored-by: Chunde Co-authored-by: Corey Richardson Co-authored-by: CristianAndrade94 <117796497+CristianAndrade94@users.noreply.github.com> Co-authored-by: Dave Liu Co-authored-by: David Hagen Co-authored-by: Gökçen Eraslan Co-authored-by: Hong Co-authored-by: Iarsv <96173089+Iarsv@users.noreply.github.com> Co-authored-by: Isuru Fernando Co-authored-by: Jellby Co-authored-by: Joachim Wagner Co-authored-by: Joseph Shen Co-authored-by: Kevin Ji Co-authored-by: Liming Wang Co-authored-by: Marco Pompili Co-authored-by: Marcus Ottosson Co-authored-by: Musen Co-authored-by: Neil Shipp Co-authored-by: Olivier Grisel Co-authored-by: Randall Bohn Co-authored-by: Tiger <51085070+1149859096@users.noreply.github.com> Co-authored-by: Tillsten Co-authored-by: Tommy Carozzani Co-authored-by: Tyler Co-authored-by: Xingyu Na Co-authored-by: Zhuo Zhang Co-authored-by: brada4 Co-authored-by: ccy022364 Co-authored-by: davmaz Co-authored-by: dkapelyan <30596321+dkapelyan@users.noreply.github.com> Co-authored-by: eolianoe Co-authored-by: fommil Co-authored-by: magras Co-authored-by: neitann <96481461+neitann@users.noreply.github.com> Co-authored-by: raffamaiden Co-authored-by: sogf <93816959+sogf@users.noreply.github.com> Co-authored-by: wernsaar --- .md | 3 + CI-jobs-overview.md | 54 +++ Developer-manual.md | 141 ++++++ Document.md | 27 ++ Donation.md | 31 ++ Download.md | 23 + Faq.md | 432 ++++++++++++++++++ Fixed-optimized-kernels-To-do-List.md | 14 + Home.md | 56 +++ How-to-build-OpenBLAS-for-Android.md | 190 ++++++++ How-to-build-OpenBLAS-for-Windows-on-ARM64.md | 77 ++++ ...o-build-OpenBLAS-for-iPhone-iOS-(ARMv8).md | 14 + How-to-build-OpenBLAS-for-macOS-M1---arm64.md | 1 + How-to-generate-import-library-for-MinGW.md | 36 ++ ...use-OpenBLAS-in-Microsoft-Visual-Studio.md | 162 +++++++ How-to-use-OpenBLAS-on-Cortex-M.md | 37 ++ Installation-Guide.md | 71 +++ Machine-List.md | 9 + Mailing-List.md | 15 + OpenBLAS-Extensions.md | 28 ++ Precompiled-installation-packages.md | 84 ++++ Publications.md | 19 + Related-packages-that-use-OpenBLAS.md | 13 + User-Manual.md | 187 ++++++++ ...-Build-system-overview-(community-made).md | 103 +++++ 25 files changed, 1827 insertions(+) create mode 100644 .md create mode 100644 CI-jobs-overview.md create mode 100644 Developer-manual.md create mode 100644 Document.md create mode 100644 Donation.md create mode 100644 Download.md create mode 100644 Faq.md create mode 100644 Fixed-optimized-kernels-To-do-List.md create mode 100644 Home.md create mode 100644 How-to-build-OpenBLAS-for-Android.md create mode 100644 How-to-build-OpenBLAS-for-Windows-on-ARM64.md create mode 100644 How-to-build-OpenBLAS-for-iPhone-iOS-(ARMv8).md create mode 100644 How-to-build-OpenBLAS-for-macOS-M1---arm64.md create mode 100644 How-to-generate-import-library-for-MinGW.md create mode 100644 How-to-use-OpenBLAS-in-Microsoft-Visual-Studio.md create mode 100644 How-to-use-OpenBLAS-on-Cortex-M.md create mode 100644 Installation-Guide.md create mode 100644 Machine-List.md create mode 100644 Mailing-List.md create mode 100644 OpenBLAS-Extensions.md create mode 100644 Precompiled-installation-packages.md create mode 100644 Publications.md create mode 100644 Related-packages-that-use-OpenBLAS.md create mode 100644 User-Manual.md create mode 100644 WIP---Build-system-overview-(community-made).md diff --git a/.md b/.md new file mode 100644 index 0000000000..e33f6435d6 --- /dev/null +++ b/.md @@ -0,0 +1,3 @@ +# Installation Guide + +test \ No newline at end of file diff --git a/CI-jobs-overview.md b/CI-jobs-overview.md new file mode 100644 index 0000000000..3e2a181921 --- /dev/null +++ b/CI-jobs-overview.md @@ -0,0 +1,54 @@ +| Arch|Target CPU|OS|Build system|XComp to|C Compiler|Fortran Compiler|threading|DYN_ARCH|INT64|Libraries| CI Provider| CPU count| +| ------------|---|---|-----------|-------------|----------|----------------|------|------------|----------|-----------|----------|-------| +| x86_64 |Intel 32bit|Windows|CMAKE/VS2015| -|mingw6.3| - | pthreads | - | - | static | Appveyor| | +| x86_64 |Intel |Windows|CMAKE/VS2015| -|mingw5.3| - | pthreads | - | - | static | Appveyor| | +| x86_64 |Intel |Centos5|gmake | -|gcc 4.8 |gfortran| pthreads | + | - | both | Azure | | +| x86_64 |SDE (SkylakeX)|Ubuntu| CMAKE| - | gcc | gfortran | pthreads | - | - | both | Azure | | +| x86_64 |Haswell/ SkylakeX|Windows|CMAKE/VS2017| - | VS2017| - | | - | - | static | Azure | | +| x86_64 | " | Windows|mingw32-make| - |gcc | gfortran | | list | - | both | Azure | | +| x86_64 | " |Windows|CMAKE/Ninja| - |LLVM | - | | - | - | static | Azure | | +| x86_64 | " |Windows|CMAKE/Ninja| - |LLVM | flang | | - | - | static | Azure | | +| x86_64 | " |Windows|CMAKE/Ninja| - |VS2022| flang* | | - | - | static | Azure | | +| x86_64 | " |macOS11|gmake | - | gcc-10|gfortran| OpenMP | + | - | both | Azure | | +| x86_64 | " |macOS11|gmake | - | gcc-10|gfortran| none | - | - | both | Azure | | +| x86_64 | " |macOS12|gmake | - | gcc-12|gfortran|pthreads| - | - | both | Azure | | +| x86_64 | " |macOS11|gmake | - | llvm | - | OpenMP | + | - | both | Azure | | +| x86_64 | " |macOS11|CMAKE | - | llvm | - | OpenMP | no_avx512 | - | static | Azure | | +| x86_64 | " |macOS11|CMAKE | - | gcc-10| gfortran| pthreads | list | - | shared | Azure | | +| x86_64 | " |macOS11|gmake | - | llvm | ifort | pthreads | - | - | both | Azure | | +| x86_64 | " |macOS11|gmake |arm| AndroidNDK-llvm | - | | - | - | both | Azure | | +| x86_64 | " |macOS11|gmake |arm64| XCode 12.4 | - | | + | - | both | Azure | | +| x86_64 | " |macOS11|gmake |arm | XCode 12.4 | - | | + | - | both | Azure | | +| x86_64 | " |Alpine Linux(musl)|gmake| - | gcc | gfortran | pthreads | + | - | both | Azure | | +| arm64 |Apple M1 |OSX |CMAKE/XCode| - | LLVM | - | OpenMP | - | - | static | Cirrus | | +| arm64 |Apple M1 |OSX |CMAKE/Xcode| - | LLVM | - | OpenMP | - | + | static | Cirrus | | +| arm64 |Apple M1 |OSX |CMAKE/XCode|x86_64| LLVM| - | - | + | - | static | Cirrus | | +| arm64 |Neoverse N1|Linux |gmake | - |gcc10.2| -| pthreads| - | - | both | Cirrus | | +| arm64 |Neoverse N1|Linux |gmake | - |gcc10.2| -| pthreads| - | + | both | Cirrus | | +| arm64 |Neoverse N1|Linux |gmake |- |gcc10.2| -| OpenMP | - | - | both |Cirrus | 8 | +| x86_64 | Ryzen| FreeBSD |gmake | - | gcc12.2|gfortran| pthreads| - | - | both | Cirrus | | +| x86_64 | Ryzen| FreeBSD |gmake | | gcc12.2|gfortran| pthreads| - | + | both | Cirrus | | +| x86_64 |GENERIC |QEMU |gmake| mips64 | gcc | gfortran | pthreads | - | - | static | Github | | +| x86_64 |SICORTEX |QEMU |gmake| mips64 | gcc | gfortran | pthreads | - | - | static | Github | | +| x86_64 |I6400 |QEMU |gmake| mips64 | gcc | gfortran | pthreads | - | - | static | Github | | +| x86_64 |P6600 |QEMU |gmake| mips64 | gcc | gfortran | pthreads | - | - | static | Github | | +| x86_64 |I6500 |QEMU |gmake| mips64 | gcc | gfortran | pthreads | - | - | static | Github | | +| x86_64 |Intel |Ubuntu |CMAKE| - | gcc-11.3 | gfortran | pthreads | + | - | static | Github | | +| x86_64 |Intel |Ubuntu |gmake| - | gcc-11.3 | gfortran | pthreads | + | - | both | Github | | +| x86_64 |Intel |Ubuntu |CMAKE| - | gcc-11.3 | flang-classic | pthreads | + | - | static | Github | | +| x86_64 |Intel |Ubuntu |gmake| - | gcc-11.3 | flang-classic | pthreads | + | - | both | Github | | +| x86_64 |Intel |macOS12 | CMAKE| - | AppleClang 14 | gfortran | pthreads | + | - | static | Github | | +| x86_64 |Intel |macOS12 | gmake| - | AppleClang 14 | gfortran | pthreads | + | - | both | Github | | +| x86_64 |Intel |Windows2022 | CMAKE/Ninja| - | mingw gcc 13 | gfortran | | + | - | static | Github | | +| x86_64 |Intel |Windows2022 | CMAKE/Ninja| - | mingw gcc 13 | gfortran | | + | + | static | Github | | +| x86_64 |Intel 32bit|Windows2022 | CMAKE/Ninja| - | mingw gcc 13 | gfortran | | + | - | static | Github | | +| x86_64 |Intel |Windows2022 | CMAKE/Ninja| - | LLVM 16 | - | | + | - | static | Github | | +| x86_64 |Intel | Windows2022 |CMAKE/Ninja| - | LLVM 16 | - | | + | + | static | Github | | +| x86_64 |Intel | Windows2022 |CMAKE/Ninja| - | gcc 13| - | | + | - | static | Github | | +| x86_64 |Intel| Ubuntu |gmake |mips64|gcc|gfortran|pthreads|+|-|both|Github| | +| x86_64 |generic|Ubuntu |gmake |riscv64|gcc|gfortran|pthreads|-|-|both|Github| | +| x86_64 |Intel|Ubuntu |gmake |mips32|gcc|gfortran|pthreads|-|-|both|Github | | +| x86_64 |Intel|Ubuntu |gmake |ia64|gcc|gfortran|pthreads|-|-|both|Github| | +| x86_64 |C910V|QEmu |gmake |riscv64|gcc|gfortran|pthreads|-|-|both|Github| | +|power |pwr9| Ubuntu |gmake | - |gcc|gfortran|OpenMP|-|-|both|OSUOSL| | +|zarch |z14 | Ubuntu |gmake | - |gcc|gfortran|OpenMP|-|-|both|OSUOSL| | diff --git a/Developer-manual.md b/Developer-manual.md new file mode 100644 index 0000000000..0e04040252 --- /dev/null +++ b/Developer-manual.md @@ -0,0 +1,141 @@ +## Source codes Layout + +``` +OpenBLAS/ +├── benchmark Benchmark codes for BLAS +├── cmake CMakefiles +├── ctest Test codes for CBLAS interfaces +├── driver Implemented in C +│   ├── level2 +│   ├── level3 +│   ├── mapper +│   └── others Memory management, threading, etc +├── exports Generate shared library +├── interface Implement BLAS and CBLAS interfaces (calling driver or kernel) +│   ├── lapack +│   └── netlib +├── kernel Optimized assembly kernels for CPU architectures +│   ├── alpha Original GotoBLAS kernels for DEC Alpha +│   ├── arm ARMV5,V6,V7 kernels (including generic C codes used by other architectures) +│   ├── arm64 ARMV8 +│   ├── generic General kernel codes written in plain C, parts used by many architectures. +│   ├── ia64 Original GotoBLAS kernels for Intel Itanium +│ ├── mips +│   ├── mips64 +│   ├── power +| ├── riscv64 +| ├── simd Common code for Universal Intrinsics, used by some x86_64 and arm64 kernels +│   ├── sparc +│   ├── x86 +│ ├── x86_64 +│   └── zarch +├── lapack Optimized LAPACK codes (replacing those in regular LAPACK) +│   ├── getf2 +│   ├── getrf +│   ├── getrs +│   ├── laswp +│   ├── lauu2 +│   ├── lauum +│   ├── potf2 +│   ├── potrf +│   ├── trti2 +│ ├── trtri +│   └── trtrs +├── lapack-netlib LAPACK codes from netlib reference implementation +├── reference BLAS Fortran reference implementation (unused) +├── relapack Elmar Peise's recursive LAPACK (implemented on top of regular LAPACK) +├── test Test codes for BLAS +└── utest Regression test + +``` + +A call tree for `dgemm` is as following. + +``` +interface/gemm.c + │ +driver/level3/level3.c + │ +gemm assembly kernels at kernel/ +``` + +To find the kernel currently used for a particular supported cpu, please check the corresponding `kernel/$(ARCH)/KERNEL.$(CPU)` file. + +Here is an example for `kernel/x86_64/KERNEL.HASWELL` + +``` +... +DTRMMKERNEL = dtrmm_kernel_4x8_haswell.c +DGEMMKERNEL = dgemm_kernel_4x8_haswell.S +... +``` +According to the above `KERNEL.HASWELL`, OpenBLAS Haswell dgemm kernel file is `dgemm_kernel_4x8_haswell.S`. + +## Optimizing GEMM for a given hardware + +Read the Goto paper to understand the algorithm. + +Goto, Kazushige; van de Geijn, Robert A. (2008). ["Anatomy of High-Performance Matrix Multiplication"](http://delivery.acm.org/10.1145/1360000/1356053/a12-goto.pdf?ip=155.68.162.54&id=1356053&acc=ACTIVE%20SERVICE&key=A79D83B43E50B5B8%2EF070BBE7E45C3F17%2E4D4702B0C3E38B35%2E4D4702B0C3E38B35&__acm__=1517932837_edfe766f1e295d9a7830812371e1d173). ACM Transactions on Mathematical Software 34 (3): Article 12 +(The above link is available only to ACM members, but this and many related papers is also available on the pages +of van de Geijn's FLAME project, http://www.cs.utexas.edu/~flame/web/FLAMEPublications.html ) + +The `driver/level3/level3.c` is the implementation of Goto's algorithm. Meanwhile, you can look at `kernel/generic/gemmkernel_2x2.c`, which is a naive `2x2` register blocking gemm kernel in C. + +Then, +* Write optimized assembly kernels. consider instruction pipeline, available registers, memory/cache accessing +* Tuning cache block size, `Mc`, `Kc`, and `Nc` + +Note that not all of the cpu-specific parameters in param.h are actively used in algorithms. DNUMOPT only appears as a scale factor in profiling output of the level3 syrk interface code, while its counterpart SNUMOPT (aliased as NUMOPT in common.h) is not used anywhere at all. +SYMV_P is only used in the generic kernels for the symv and chemv/zhemv functions - at least some of those are usually overridden by cpu-specific implementations, so if you start by cloning the existing implementation for a related cpu you need to check its KERNEL file to see if tuning SYMV_P would have any effect at all. +GEMV_UNROLL is only used by some older x86_64 kernels, so not all sections in param.h define it. +Similarly, not all of the cpu parameters like L2 or L3 cache sizes are necessarily used in current kernels for a given model - by all indications the cpu identification code was imported from some other project originally. + +## Run OpenBLAS Test + +We use netlib blas test, cblas test, and LAPACK test. Meanwhile, we use [BLAS-Tester](https://github.com/xianyi/BLAS-Tester), a modified test tool from ATLAS. + +* Run `test` and `ctest` at OpenBLAS. e.g. `make test` or `make ctest`. +* Run regression test `utest` at OpenBLAS. +* Run LAPACK test. e.g. `make lapack-test`. +* Clone [BLAS-Tester](https://github.com/xianyi/BLAS-Tester), which can compare the OpenBLAS result with netlib reference BLAS. + +The project makes use of several Continuous Integration (CI) services conveniently interfaced with github to automatically check compilability on a number of platforms. +Lastly, the testsuites included with "numerically heavy" projects like Julia, NumPy, Octave or QuantumEspresso can be used for regression testing. + +## Benchmarking + +Several simple C benchmarks for performance testing individual BLAS functions are available in the `benchmark` folder, and its `scripts` subdirectory contains corresponding versions for Python, Octave and R. +Other options include + +* https://github.com/RoyiAvital/MatlabJuliaMatrixOperationsBenchmark (various matrix operations in Julia and Matlab) +* https://github.com/mmperf/mmperf/ (single-core matrix multiplication) + +## Adding autodetection support for a new revision or variant of a supported cpu + +Especially relevant for x86_64, a new cpu model may be a "refresh" (die shrink and/or different number of cores) within an existing +model family without significant changes to its instruction set. (e.g. Intel Skylake, Kaby Lake etc. still are fundamentally Haswell, +low end Goldmont etc. are Nehalem). In this case, compilation with the appropriate older TARGET will already lead to a satisfactory build. + +To achieve autodetection of the new model, its CPUID (or an equivalent identifier) needs to be added in the `cpuid_.c` +relevant for its general architecture, with the returned name for the new type set appropriately. For x86 which has the most complex +cpuid file, there are two functions that need to be edited - get_cpuname() to return e.g. CPUTYPE_HASWELL and get_corename() for the (broader) +core family returning e.g. CORE_HASWELL. (This information ends up in the Makefile.conf and config.h files generated by `getarch`. Failure to +set either will typically lead to a missing definition of the GEMM_UNROLL parameters later in the build, as `getarch_2nd` will be unable to +find a matching parameter section in param.h.) + +For architectures where "DYNAMIC_ARCH" builds are supported, a similar but simpler code section for the corresponding runtime detection of the cpu exists in `driver/others/dynamic.c` (for x86) and `driver/others/dynamic_.c` for other architectures. +Note that for x86 the CPUID is compared after splitting it into its family, extended family, model and extended model parts, so the single decimal +number returned by Linux in /proc/cpuinfo for the model has to be converted back to hexadecimal before splitting into its constituent +digits, e.g. 142 = 8E , translates to extended model 8, model 14. + +## Adding dedicated support for a new cpu model + +Usually it will be possible to start from an existing model, clone its KERNEL configuration file to the new name to use for this TARGET and eventually replace individual kernels with versions better suited for peculiarities of the new cpu model. In addition, it is necessary to add +(or clone at first) the corresponding section of GEMM_UNROLL parameters in the toplevel param.h, and possibly to add definitions such as USE_TRMM +(governing whether TRMM functions use the respective GEMM kernel or a separate source file) to the Makefiles (and CMakeLists.txt) in the kernel +directory. The new cpu name needs to be added to TargetLists.txt and the cpu autodetection code used by the `getarch` helper program - contained in +the `cpuid_.c` file amended to include the CPUID (or equivalent) information processing required (see preceding section). + +## Adding support for an entirely new architecture + +This endeavour is best started by cloning the entire support structure for 32bit ARM, and within that the ARMV5 cpu in particular as this is implemented through plain C kernels only. An example providing a convenient "shopping list" can be seen in pull request #1526. diff --git a/Document.md b/Document.md new file mode 100644 index 0000000000..68154b2d35 --- /dev/null +++ b/Document.md @@ -0,0 +1,27 @@ +
+
+ [Home] + [Document] + [FAQ] + [Publications] + [Download] +[Mailing List] +[Donation] +
+
+ +[Installation Guide](Installation-Guide) + +[User Manual](User-Manual) + +[Developer Manual](Developer-manual) + +[Use OpenBLAS in MS Visual Studio](How-to-use-OpenBLAS-in-Microsoft-Visual-Studio) + +[Generate import library for MingW](How-to-generate-import-library-for-MingW) + +[OpenBLAS Extensions](OpenBLAS-Extensions) + +[Related packages that use OpenBLAS](Related-packages-that-use-OpenBLAS) + +[Machine List](Machine-List) \ No newline at end of file diff --git a/Donation.md b/Donation.md new file mode 100644 index 0000000000..ca7efe9150 --- /dev/null +++ b/Donation.md @@ -0,0 +1,31 @@ +
+
+ [Home] + [Document] + [FAQ] + [Publications] + [Download] +[Mailing List] +[Donation] +
+
+ +Thank you for the support. + +You can read OpenBLAS statement of receipts and disbursement and cash balance on [google doc](https://docs.google.com/spreadsheet/ccc?key=0AghkTjXe2lDndE1UZml0dGpaUzJmZGhvenBZd1F2R1E&usp=sharing). + +## Fundraiser + +* [2013.8] [Testbed for OpenBLAS project](https://www.bountysource.com/fundraisers/443-testbed-for-openblas-project) + + * completed. + +Here is [Backer list](https://github.com/xianyi/OpenBLAS/blob/develop/BACKERS.md). + +## For Personal Donation + +You can create a bounty for an issue on [bountysource.com](https://www.bountysource.com/trackers/69691-openblas). + +## For Hardware Vendors + +We welcome the hardware donation, including the latest CPU and boards. \ No newline at end of file diff --git a/Download.md b/Download.md new file mode 100644 index 0000000000..cb0f325261 --- /dev/null +++ b/Download.md @@ -0,0 +1,23 @@ +
+
+ [Home] + [Document] + [FAQ] + [Publications] + [Download] +[Mailing List] +[Donation] +
+
+ +## Binary Packages + +We provide binary packages for the following platform. + +* Windows x86/x86_64 +* ARM + +You can download them from [file hosting on sourceforge.net](http://sourceforge.net/projects/openblas/files/). + +## Source +Download the latest [stable version](https://github.com/xianyi/OpenBLAS/releases) from release page. \ No newline at end of file diff --git a/Faq.md b/Faq.md new file mode 100644 index 0000000000..3c30de0f67 --- /dev/null +++ b/Faq.md @@ -0,0 +1,432 @@ +
+
+ [Home] + [Document] + [FAQ] + [Publications] + [Download] +[Mailing List] +[Donation] +
+
+ +### General questions ++ **[What is BLAS? Why is it important?](#whatblas)** ++ **[What functions are there and how can I call them from my C code?](#whatsinblas)** ++ **[What is OpenBLAS? Why did you create this project?](#what)** ++ **[What's the difference between OpenBLAS and GotoBLAS?](#gotoblas)** ++ **[Where do parameters GEMM_P, GEMM_Q, GEMM_R come from?](#gemmpqr)** ++ **[How can I report a bug?](#reportbug)** ++ **[How to reference OpenBLAS.](#publication)** ++ **[How can I use OpenBLAS in multi-threaded applications?](#multi-threaded)** ++ **[Does OpenBLAS support sparse matrices and/or vectors ?](#sparse)** ++ **[What support is there for recent PC hardware ? What about GPU ?](#recent_hardware)** ++ **[How about the level 3 BLAS performance on Intel Sandy Bridge?](#sandybridge_perf)** + +### OS and Compiler ++ **[How can I call an OpenBLAS function in Microsoft Visual Studio?](#MSVC)** ++ **[How can I use CBLAS and LAPACKE without C99 complex number support (e.g. in Visual Studio)?](#C99_complex_number)** ++ **[I get a SEGFAULT with multi-threading on Linux. What's wrong?](#Linux_SEGFAULT)** ++ **[When I make the library, there is no such instruction: `xgetbv' error. What's wrong?](#xgetbv)** ++ **[My build fails due to the linker error "multiple definition of `dlamc3_'". What is the problem?](#patch_missing)** ++ **[My build worked fine and passed all tests, but running `make lapack-test` ends with segfaults](#xeigtst)** ++ **[How could I disable OpenBLAS threading affinity on runtime?](#no_affinity)** ++ **[How to solve undefined reference errors when statically linking against libopenblas.a](#static_link)** ++ **[Building OpenBLAS for Haswell or Dynamic Arch on RHEL-6, CentOS-6, Rocks-6.1](#binutils)** ++ **[Building OpenBLAS in QEMU/KVM/XEN](#qemu)** ++ **[Building OpenBLAS for MIPS](#mips)** ++ **[Building OpenBLAS on POWER fails with the IBM XL compiler](#ppcxl)** ++ **[Replacing system BLAS in Ubuntu/Debian](#debianlts)** ++ **[I built OpenBLAS for use with some other software, but that software cannot find it](#findblas)** ++ **[I included cblas.h in my program, but the compiler complains about a missing common.h or functions from it](#installincludes)** ++ **[Compiling OpenBLAS with gcc's -fbounds-check actually triggers aborts in programs](#boundscheck)** ++ **[Build fails with lots of errors about undefined ?GEMM_UNROLL_M](#newcpu)** ++ **[CMAKE/OSX: Build fails with 'argument list too long'](#cmakeosx)** ++ **[Likely problems with AVX2 support in Docker Desktop for OSX](#xhyve)** +### Usage ++ **[Program is Terminated. Because you tried to allocate too many memory regions](#allocmorebuffers)** ++ **[How to choose TARGET manually at runtime when compiled with DYNAMIC_ARCH](#choose_target_dynamic)** ++ **[After updating the installed OpenBLAS, a program complains about "undefined symbol gotoblas"](#missgoto)** ++ **[How can I find out at runtime what options the library was built wih ?](#buildoptions)** ++ **[After making OpenBLAS, I find that the static library is multithreaded, but the dynamic one is not ?](#wronglibrary)** ++ **[I want to use OpenBLAS with CUDA in the HPL 2.3 benchmark code but it keeps looking for Intel MKL](#cudahpl)** ++ **[Multithreaded OpenBLAS runs no faster or is even slower than singlethreaded on my ARMV7 board](#cpusoffline)** ++ **[Speed varies wildly between individual runs on a typical ARMV8 smartphone processor](#biglittle)** ++ **[I cannot get OpenBLAS to use more than a small subset of available cores on a big system](#numthreads)** ++ **[Getting "ELF load command address/offset not properly aligned" when loading libopenblas.so](#ELFoffset)** ++ **[Using OpenBLAS with OpenMP](#OpenMP)** +
+ +### General questions + +#### What is BLAS? Why is it important? + +[BLAS](https://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms) stands for Basic Linear Algebra Subprograms. BLAS provides standard interfaces for [linear algebra](https://en.wikipedia.org/wiki/Linear_algebra), including BLAS1 (vector-vector operations), BLAS2 (matrix-vector operations), and BLAS3 (matrix-matrix operations). In general, BLAS is the computational kernel ("the bottom of the food chain") in linear algebra or scientific applications. Thus, if BLAS implementation is highly optimized, the whole application can get substantial benefit. + +#### What functions are there and how can I call them from my C code? + +As BLAS is a standardized interface, you can refer to the documentation of its reference implementation at [netlib.org](http://netlib.org/blas/index.html#_blas_routines). Calls from C go through its CBLAS interface, +so your code will need to include the provided cblas.h in addition to linking with -lopenblas. +A single-precision matrix multiplication will look like +``` +#include +... +cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, M, N, K, 1.0, A, K, B, N, 0.0, result, N); +``` +where M,N,K are the dimensions of your data - see https://petewarden.files.wordpress.com/2015/04/gemm_corrected.png + (This image is part of an article on GEMM in the context of deep learning that is well worth reading in full - +https://petewarden.com/2015/04/20/why-gemm-is-at-the-heart-of-deep-learning/) + +#### What is OpenBLAS? Why did you create this project? + +OpenBLAS is an open source BLAS library forked from the GotoBLAS2-1.13 BSD version. Since Mr. Kazushige Goto left TACC, GotoBLAS is no longer being maintained. Thus, we created this project to continue developing OpenBLAS/GotoBLAS. + +#### What's the difference between OpenBLAS and GotoBLAS? + +In OpenBLAS 0.2.0, we optimized level 3 BLAS on the Intel Sandy Bridge 64-bit OS. We obtained a performance comparable with that Intel MKL. + +We optimized level 3 BLAS performance on the [ICT Loongson-3A](http://en.wikipedia.org/wiki/Loongson) CPU. It outperformed GotoBLAS by 135% in a single thread and 120% in 4 threads. + +We fixed some GotoBLAS bugs including a SEGFAULT bug on the new Linux kernel, MingW32/64 bugs, and a ztrmm computing error bug on Intel Nehalem. + +We also added some minor features, e.g. supporting "make install", compiling without LAPACK and upgrading the LAPACK version to 3.4.2. + +You can find the full list of modifications in Changelog.txt. + +#### Where do parameters GEMM_P, GEMM_Q, GEMM_R come from? + +The detailed explanation is probably in the original publication authored by Kazushige Goto - Goto, Kazushige; van de Geijn, Robert A; Anatomy of high-performance matrix multiplication. ACM Transactions on Mathematical Software (TOMS). Volume 34 Issue 3, May 2008 +While this article is paywalled and too old for preprints to be available on arxiv.org, more recent +publications like https://arxiv.org/pdf/1609.00076 contain at least a brief description of the algorithm. +In practice, the values are derived by experimentation to yield the block sizes that give the highest performance. A general rule of thumb for selecting a starting point seems to be that PxQ is about half the size of L2 cache. + +#### How can I report a bug? + +Please file an issue at this [issue page](https://github.com/xianyi/OpenBLAS/issues) or send mail to the [OpenBLAS mailing list](https://groups.google.com/forum/#!forum/openblas-users). + +Please provide the following information: CPU, OS, compiler, and OpenBLAS compiling flags (Makefile.rule). In addition, please describe how to reproduce this bug. + +#### How to reference OpenBLAS. + +You can reference our papers in [this page](Publications). Alternatively, you can cite the OpenBLAS homepage http://www.openblas.net. + +#### How can I use OpenBLAS in multi-threaded applications? + +If your application is already multi-threaded, it will conflict with OpenBLAS multi-threading. Thus, you must set OpenBLAS to use single thread as following. + + * export OPENBLAS_NUM_THREADS=1 in the environment variables. +Or + * Call openblas_set_num_threads(1) in the application on runtime. +Or + * Build OpenBLAS single thread version, e.g. make USE_THREAD=0 USE_LOCKING=1 (see comment below) + +If the application is parallelized by OpenMP, please build OpenBLAS with USE_OPENMP=1 + +With the increased availability of fast multicore hardware it has unfortunately become clear that the thread management provided by OpenMP is not sufficient to prevent race conditions when OpenBLAS was built single-threaded by USE_THREAD=0 and there are concurrent calls from multiple threads to OpenBLAS functions. In this case, +it is vital to also specify USE_LOCKING=1 (introduced with OpenBLAS 0.3.7). + +#### Does OpenBLAS support sparse matrices and/or vectors ? + +OpenBLAS implements only the standard (dense) BLAS and LAPACK functions with a select few extensions popularized by Intel's MKL. Some +cases can probably be made to work using e.g. GEMV or AXPBY, in general using a dedicated package like SuiteSparse (which can make use of OpenBLAS or equivalent for standard operations) is recommended. + +#### What support is there for recent PC hardware ? What about GPU ? + + As OpenBLAS is a volunteer project, it can take some time for the combination of a capable developer, +free time, and particular hardware to come along, even for relatively common processors. Starting from 0.3.1, support +is being added for AVX 512 (TARGET=SKYLAKEX), requiring a compiler that is capable of handling avx512 intrinsics. +While AMD Zen processors should be autodetected by the build system, as of 0.3.2 they are still handled exactly +like Intel Haswell. There once was an effort to build an OpenCL implementation that one can still find at https://github.com/xianyi/clOpenBLAS , but work on this stopped in 2015. + +#### How about the level 3 BLAS performance on Intel Sandy Bridge? + +We obtained a performance comparable with Intel MKL that actually outperformed Intel MKL in some cases. +Here is the result of the DGEMM subroutine's performance on Intel Core i5-2500K Windows 7 SP1 64-bit: +![Single Thread DGEMM Performance on Intel Desktop Sandy Bridge](http://xianyi.github.com/OpenBLAS/dgemm_snb_1thread.png) + +
+ +### OS and Compiler + +#### How can I call an OpenBLAS function in Microsoft Visual Studio? + +Please read [this page](How-to-use-OpenBLAS-in-Microsoft-Visual-Studio). + +#### How can I use CBLAS and LAPACKE without C99 complex number support (e.g. in Visual Studio)? + +Zaheer has fixed this bug. You can now use the structure instead of C99 complex numbers. Please read [this issue page](http://github.com/xianyi/OpenBLAS/issues/95) for details. + +[This issue](https://github.com/xianyi/OpenBLAS/issues/305) is for using LAPACKE in Visual Studio. + +#### I get a SEGFAULT with multi-threading on Linux. What's wrong? + +This may be related to a bug in the Linux kernel 2.6.32 (?). Try applying the patch segaults.patch to disable mbind using + + patch < segfaults.patch + +and see if the crashes persist. Note that this patch will lead to many compiler warnings. + +#### When I make the library, there is no such instruction: `xgetbv' error. What's wrong? + +Please use GCC 4.4 and later version. This version supports xgetbv instruction. If you use the library for Sandy Bridge with AVX instructions, you should use GCC 4.6 and later version. + +On Mac OS X, please use Clang 3.1 and later version. For example, make CC=clang + +For the compatibility with old compilers (GCC < 4.4), you can enable NO_AVX flag. For example, make NO_AVX=1 + +#### My build fails due to the linker error "multiple definition of `dlamc3_'". What is the problem? + +This linker error occurs if GNU patch is missing or if our patch for LAPACK fails to apply. + +Background: OpenBLAS implements optimized versions of some LAPACK functions, so we need to disable the reference versions. If this process fails we end with duplicated implementations of the same function. + +#### My build worked fine and passed all tests, but running `make lapack-test` ends with segfaults + +Some of the LAPACK tests, notably in xeigtstz, try to allocate around 10MB on the stack. You may need to use +`ulimit -s` to change the default limits on your system to allow this. + +#### How could I disable OpenBLAS threading affinity on runtime? + +You can define the OPENBLAS_MAIN_FREE or GOTOBLAS_MAIN_FREE environment variable to disable threading affinity on runtime. For example, before the running, +``` +export OPENBLAS_MAIN_FREE=1 +``` + +Alternatively, you can disable affinity feature with enabling NO_AFFINITY=1 in Makefile.rule. + +#### How to solve undefined reference errors when statically linking against libopenblas.a + +On Linux, if OpenBLAS was compiled with threading support (`USE_THREAD=1` by default), custom programs statically linked against `libopenblas.a` should also link to the pthread library e.g.: + +``` +gcc -static -I/opt/OpenBLAS/include -L/opt/OpenBLAS/lib -o my_program my_program.c -lopenblas -lpthread +``` + +Failing to add the `-lpthread` flag will cause errors such as: + +``` +/opt/OpenBLAS/libopenblas.a(memory.o): In function `_touch_memory': +memory.c:(.text+0x15): undefined reference to `pthread_mutex_lock' +memory.c:(.text+0x41): undefined reference to `pthread_mutex_unlock' +/opt/OpenBLAS/libopenblas.a(memory.o): In function `openblas_fork_handler': +memory.c:(.text+0x440): undefined reference to `pthread_atfork' +/opt/OpenBLAS/libopenblas.a(memory.o): In function `blas_memory_alloc': +memory.c:(.text+0x7a5): undefined reference to `pthread_mutex_lock' +memory.c:(.text+0x825): undefined reference to `pthread_mutex_unlock' +/opt/OpenBLAS/libopenblas.a(memory.o): In function `blas_shutdown': +memory.c:(.text+0x9e1): undefined reference to `pthread_mutex_lock' +memory.c:(.text+0xa6e): undefined reference to `pthread_mutex_unlock' +/opt/OpenBLAS/libopenblas.a(blas_server.o): In function `blas_thread_server': +blas_server.c:(.text+0x273): undefined reference to `pthread_mutex_lock' +blas_server.c:(.text+0x287): undefined reference to `pthread_mutex_unlock' +blas_server.c:(.text+0x33f): undefined reference to `pthread_cond_wait' +/opt/OpenBLAS/libopenblas.a(blas_server.o): In function `blas_thread_init': +blas_server.c:(.text+0x416): undefined reference to `pthread_mutex_lock' +blas_server.c:(.text+0x4be): undefined reference to `pthread_mutex_init' +blas_server.c:(.text+0x4ca): undefined reference to `pthread_cond_init' +blas_server.c:(.text+0x4e0): undefined reference to `pthread_create' +blas_server.c:(.text+0x50f): undefined reference to `pthread_mutex_unlock' +... +``` + +The `-lpthread` is not required when linking dynamically against `libopenblas.so.0`. + +#### Building OpenBLAS for Haswell or Dynamic Arch on RHEL-6, CentOS-6, Rocks-6.1,Scientific Linux 6 + +Minimum requirement to actually run AVX2-enabled software like OpenBLAS is kernel-2.6.32-358, shipped with EL6U4 in 2013 + +The `binutils` package from RHEL6 does not know the instruction `vpermpd` or any other AVX2 instruction. You can download a newer `binutils` package from Enterprise Linux software collections, following instructions here:
+https://www.softwarecollections.org/en/scls/rhscl/devtoolset-3/
+After configuring repository you need to install devtoolset-?-binutils to get later usable binutils package +``` +$ yum search devtoolset-\?-binutils +$ sudo yum install devtoolset-3-binutils +``` +once packages are installed check the correct name for SCL redirection set to enable new version +``` +$ scl --list +devtoolset-3 +rh-python35 +``` +Now just prefix your build commands with respective redirection: +``` +$ scl enable devtoolset-3 -- make DYNAMIC_ARCH=1 +``` +AVX-512 (SKYLAKEX) support requires devtoolset-8-gcc-gfortran (which exceeds formal requirement for AVX-512 because of packaging issues in earlier packages) which dependency-installs respective binutils and gcc or later and kernel 2.6.32-696 aka 6U9 or 3.10.0-327 aka 7U2 or later to run. In absence of abovementioned toolset OpenBLAS will fall back to AVX2 instructions in place of AVX512 sacrificing some performance on SKYLAKE-X platform. + +#### Building OpenBLAS in QEMU/KVM/XEN + +By default, QEMU reports the CPU as "QEMU Virtual CPU version 2.2.0", which shares CPUID with existing 32bit CPU even in 64bit virtual machine, and OpenBLAS recognizes it as PENTIUM2. Depending on the exact combination of CPU features the hypervisor choses to expose, this may not correspond to any CPU that exists, and OpenBLAS will error when trying to build. To fix this, pass `-cpu host` or `-cpu passthough` to QEMU, or another CPU model. +Similarly, the XEN hypervisor may not pass through all features of the host cpu while reporting the cpu type itself correctly, which can +lead to compiler error messages about an "ABI change" when compiling AVX512 code. Again changing the Xen configuration by running e.g. +"xen-cmdline --set-xen cpuid=avx512" should get around this (as would building OpenBLAS for an older cpu lacking that particular feature, e.g. TARGET=HASWELL) + +#### Building OpenBLAS for MIPS + +For mips targets you will need latest toolchains +P5600 - MTI GNU/Linux Toolchain +I6400, P6600 - IMG GNU/Linux Toolchain + +The download link is below +(http://codescape-mips-sdk.imgtec.com/components/toolchain/2016.05-03/downloads.html) + +You can use following commandlines for builds + + + IMG_TOOLCHAIN_DIR={full IMG GNU/Linux Toolchain path including "bin" directory -- for example, /opt/linux_toolchain/bin} + IMG_GCC_PREFIX=mips-img-linux-gnu + IMG_TOOLCHAIN=${IMG_TOOLCHAIN_DIR}/${IMG_GCC_PREFIX} + + I6400 Build (n32): + make BINARY=32 BINARY32=1 CC=$IMG_TOOLCHAIN-gcc AR=$IMG_TOOLCHAIN-ar FC="$IMG_TOOLCHAIN-gfortran -EL -mabi=n32" RANLIB=$IMG_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS=$CFLAGS LDFLAGS=$CFLAGS TARGET=I6400 + + I6400 Build (n64): + make BINARY=64 BINARY64=1 CC=$IMG_TOOLCHAIN-gcc AR=$IMG_TOOLCHAIN-ar FC="$IMG_TOOLCHAIN-gfortran -EL" RANLIB=$IMG_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS=$CFLAGS LDFLAGS=$CFLAGS TARGET=I6400 + + P6600 Build (n32): + make BINARY=32 BINARY32=1 CC=$IMG_TOOLCHAIN-gcc AR=$IMG_TOOLCHAIN-ar FC="$IMG_TOOLCHAIN-gfortran -EL -mabi=n32" RANLIB=$IMG_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS=$CFLAGS LDFLAGS=$CFLAGS TARGET=P6600 + + P6600 Build (n64): + make BINARY=64 BINARY64=1 CC=$IMG_TOOLCHAIN-gcc AR=$IMG_TOOLCHAIN-ar FC="$IMG_TOOLCHAIN-gfortran -EL" RANLIB=$IMG_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS="$CFLAGS" LDFLAGS="$CFLAGS" TARGET=P6600 + + MTI_TOOLCHAIN_DIR={full MTI GNU/Linux Toolchain path including "bin" directory -- for example, /opt/linux_toolchain/bin} + MTI_GCC_PREFIX=mips-mti-linux-gnu + MTI_TOOLCHAIN=${IMG_TOOLCHAIN_DIR}/${IMG_GCC_PREFIX} + + P5600 Build: + + make BINARY=32 BINARY32=1 CC=$MTI_TOOLCHAIN-gcc AR=$MTI_TOOLCHAIN-ar FC="$MTI_TOOLCHAIN-gfortran -EL" RANLIB=$MTI_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS=$CFLAGS LDFLAGS=$CFLAGS TARGET=P5600 + +#### Building OpenBLAS on POWER fails with IBM XL + + Trying to compile OpenBLAS with IBM XL ends with error messages about unknown register names +like "vs32". Working around these by using known alternate names for the vector registers only leads to another assembler error about unsupported constraints. This is a known deficiency in the IBM compiler at least up to and including 16.1.0 (and in the POWER version of clang, from which it is derived) - use gcc instead. (See issues #1078 +and #1699 for related discussions) + +#### Replacing system BLAS/updating APT OpenBLAS in Mint/Ubuntu/Debian + +Debian and Ubuntu LTS versions provide OpenBLAS package which is not updated after initial release, and under circumstances one might want to use more recent version of OpenBLAS e.g. to get support for newer CPUs + +Ubuntu and Debian provides 'alternatives' mechanism to comfortably replace BLAS and LAPACK libraries systemwide. + +After successful build of OpenBLAS (with DYNAMIC_ARCH set to 1) + +``` +$ make clean +$ make DYNAMIC_ARCH=1 +$ sudo make DYNAMIC_ARCH=1 install +``` +One can redirect BLAS and LAPACK alternatives to point to source-built OpenBLAS +First you have to install NetLib LAPACK reference implementation (to have alternatives to replace): +``` +$ sudo apt install libblas-dev liblapack-dev +``` +Then we can set alternative to our freshly-built library: +``` +$ sudo update-alternatives --install /usr/lib/libblas.so.3 libblas.so.3 /opt/OpenBLAS/lib/libopenblas.so.0 41 \ + --slave /usr/lib/liblapack.so.3 liblapack.so.3 /opt/OpenBLAS/lib/libopenblas.so.0 +``` +Or remove redirection and switch back to APT-provided BLAS implementation order: +``` +$ sudo update-alternatives --remove libblas.so.3 /opt/OpenBLAS/lib/libopenblas.so.0 +``` +In recent versions of the distributions, the installation path for the libraries has been changed to include the name of the host architecture, like /usr/lib/x86_64-linux-gnu/blas/libblas.so.3 or libblas.so.3.x86_64-linux-gnu. Use ```$ update-alternatives --display libblas.so.3``` +to find out what layout your system has. + +#### I built OpenBLAS for use with some other software, but that software cannot find it + +Openblas installs as a single library named libopenblas.so, while some programs may be searching for a separate libblas.so and liblapack.so so you may need to create appropriate symbolic links (`ln -s libopenblas.so libblas.so; +ln -s libopenblas.so liblapack.so`) or copies. Also make sure that the installation location (usually /opt/OpenBLAS/lib or /usr/local/lib) is among the library search paths of your system. + +#### I included cblas.h in my program, but the compiler complains about a missing common.h or functions from it + +You probably tried to include a cblas.h that you simply copied from the OpenBLAS source, instead you need to run +`make install` after building OpenBLAS and then use the modified cblas.h that this step builds in the installation +path (usually either /usr/local/include, /opt/OpenBLAS/include or whatever you specified as PREFIX= on the `make install`) + +#### Compiling OpenBLAS with gcc's -fbounds-check actually triggers aborts in programs + +This is due to different interpretations of the (informal) standard for passing characters as arguments between C and FORTRAN functions. As the method for storing text differs in the two languages, when C calls Fortran the text length is passed as an "invisible" additional parameter. +Historically, this has not been required when the text is just a single character, so older code like the Reference-LAPACK bundled with OpenBLAS +does not do it. Recently gcc's checking has changed to require it, but there is no consensus yet if and how the existing LAPACK (and many other codebases) should adapt. (And for actual compilation, gcc has mostly backtracked and provided compatibility options - hence the default build settings in the OpenBLAS Makefiles add -fno-optimize-sibling-calls to the gfortran options to prevent miscompilation with "affected" versions. See ticket 2154 in the issue tracker for more details and links) +
+ +#### Build fails with lots of errors about undefined ?GEMM_UNROLL_M + +Your cpu is apparently too new to be recognized by the build scripts, so they failed to assign appropriate parameters for the block algorithm. +Do a `make clean` and try again with TARGET set to one of the cpu models listed in `TargetList.txt` - for x86_64 this will usually be HASWELL. + +#### CMAKE/OSX: Build fails with 'argument list too long' + +This is a limitation in the maximum length of a command on OSX, coupled with how CMAKE works. You should be able to work around this +by adding the option `-DCMAKE_Fortran_USE_RESPONSE_FILE_FOR_OBJECTS=1` to your CMAKE arguments. + +#### Likely problems with AVX2 support in Docker Desktop for OSX + +There have been a few reports of wrong calculation results and build-time test failures when building in a container environment managed by the OSX version of Docker Desktop, which uses the xhyve virtualizer underneath. Judging from these reports, AVX2 support in xhyve appears to be subtly broken but a corresponding ticket in the xhyve issue tracker has not drawn any reaction or comment since 2019. Therefore it is strongly recommended to build OpenBLAS with the NO_AVX2=1 option when inside a container under (or for later use with) the Docker Desktop environment on Intel-based Apple hardware. + +### Usage + +#### Program is Terminated. Because you tried to allocate too many memory regions + +In OpenBLAS, we mange a pool of memory buffers and allocate the number of buffers as the following. +``` +#define NUM_BUFFERS (MAX_CPU_NUMBER * 2) +``` +This error indicates that the program exceeded the number of buffers. + +Please build OpenBLAS with larger `NUM_THREADS`. For example, `make NUM_THREADS=32` or `make NUM_THREADS=64`. +In `Makefile.system`, we will set `MAX_CPU_NUMBER=NUM_THREADS`. + +#### How to choose TARGET manually at runtime when compiled with DYNAMIC_ARCH + +The environment variable which control the kernel selection is `OPENBLAS_CORETYPE` (see `driver/others/dynamic.c`) +e.g. `export OPENBLAS_CORETYPE=Haswell`. And the function `char* openblas_get_corename()` returns the used target. + +#### After updating the installed OpenBLAS, a program complains about "undefined symbol gotoblas" + +This symbol gets defined only when OpenBLAS is built with "make DYNAMIC_ARCH=1" (which is what distributors will choose to ensure support for more than just one CPU type). + +#### How can I find out at runtime what options the library was built with ? + +OpenBLAS has two utility functions that may come in here: + +openblas_get_parallel() will return 0 for a single-threaded library, 1 if multithreading without OpenMP, 2 if built with USE_OPENMP=1 + +openblas_get_config() will return a string containing settings such as USE64BITINT or DYNAMIC_ARCH that were active at build time, as well as the target cpu (or in case of a dynamic_arch build, the currently detected one). + +#### After making OpenBLAS, I find that the static library is multithreaded, but the dynamic one is not ? + +The shared OpenBLAS library you built is probably working fine as well, but your program may be picking up a different (probably single-threaded) version from one of the standard system paths like /usr/lib on startup. +Running `ldd /path/to/your/program` will tell you which library the linkage loader will actually use. + +Specifying the "correct" library location with the `-L` flag (like `-L /opt/OpenBLAS/lib`) when linking your program only defines which library will be used to see if all symbols _can_ be resolved, you will need to add an rpath entry to the binary (using `-Wl,rpath=/opt/OpenBLAS/lib`) to make it request searching that location. Alternatively, remove the "wrong old" library (if you can), or set LD_LIBRARY_PATH to the desired location before running your program. + +#### I want to use OpenBLAS with CUDA in the HPL 2.3 benchmark code but it keeps looking for Intel MKL + +You need to edit file src/cuda/cuda_dgemm.c in the NVIDIA version of HPL, change the "handle2" and "handle" dlopen calls to use libopenblas.so instead of libmkl_intel_lp64.so, and add an trailing underscore in the dlsym lines for dgemm_mkl and dtrsm_mkl (like `dgemm_mkl = (void(*)())dlsym(handle, “dgemm_”);`) + +#### Multithreaded OpenBLAS runs no faster or is even slower than singlethreaded on my ARMV7 board + +The power saving mechanisms of your board may have shut down some cores, making them invisible to OpenBLAS in its startup phase. Try bringing them online before starting your calculation. + +#### Speed varies wildly between individual runs on a typical ARMV8 smartphone processor + +Check the technical specifications, it could be that the SoC combines fast and slow cpus and threads can end up on either. In that case, binding the process to specific cores e.g. by setting `OMP_PLACES=cores` may help. (You may need to experiment with OpenMP options, it has been reported that using `OMP_NUM_THREADS=2 OMP_PLACES=cores` caused +a huge drop in performance on a 4+4 core chip while `OMP_NUM_THREADS=2 OMP_PLACES=cores(2)` worked as intended - as did OMP_PLACES=cores with 4 threads) + +#### I cannot get OpenBLAS to use more than a small subset of available cores on a big system + +Multithreading support in OpenBLAS requires the use of internal buffers for sharing partial results, the number and size of which is defined at compile time. Unless you specify NUM_THREADS in your make or cmake command, the build scripts try to autodetect the number of cores available in your build host to size the library to match. This unfortunately means that if you move the resulting binary from a small "front-end node" to a larger "compute node" later, it will still be limited to the hardware capabilities of the original system. The solution is to set NUM_THREADS to a number big enough to encompass the biggest systems you expect to run the binary on - at runtime, it will scale down the maximum number of threads it uses to match the number of cores physically available. + +#### Getting "ELF load command address/offset not properly aligned" when loading libopenblas.so + +If you get a message "error while loading shared libraries: libopenblas.so.0: ELF load command address/offset not properly aligned" when starting a program that is (dynamically) linked to OpenBLAS, this is very likely due to a bug in the GNU linker (ld) that is part of the +GNU binutils package. This error was specifically observed on older versions of Ubuntu Linux updated with the (at the time) most recent binutils version 2.38, but an internet search turned up sporadic reports involving various other libraries dating back several years. A bugfix was created by the binutils developers and should be available in later versions of binutils.(See issue 3708 for details) + +#### Using OpenBLAS with OpenMP + +OpenMP provides its own locking mechanisms, so when your code makes BLAS/LAPACK calls from inside OpenMP parallel regions it is imperative +that you use an OpenBLAS that is built with USE_OPENMP=1, as otherwise deadlocks might occur. Furthermore, OpenBLAS will automatically restrict itself to using only a single thread when called from an OpenMP parallel region. When it is certain that calls will only occur +from the main thread of your program (i.e. outside of omp parallel constructs), a standard pthreads build of OpenBLAS can be used as well. In that case it may be useful to tune the linger behaviour of idle threads in both your OpenMP program (e.g. set OMP_WAIT_POLICY=passive) and OpenBLAS (by redefining the THREAD_TIMEOUT variable at build time, or setting the environment variable OPENBLAS_THREAD_TIMEOUT smaller than the default 26) so that the two alternating thread pools do not unnecessarily hog the cpu during the handover. + diff --git a/Fixed-optimized-kernels-To-do-List.md b/Fixed-optimized-kernels-To-do-List.md new file mode 100644 index 0000000000..006f2cfa9f --- /dev/null +++ b/Fixed-optimized-kernels-To-do-List.md @@ -0,0 +1,14 @@ +Werner found a lot of bugs about lapack testing. In 0.2.9 version, we used a work-around,e.g. fallback to the old kernel, replacing the optimized kernel with reference kernel. We must fix them in future release. + +* ~~Nehalem dgemm kernel. Fallback to core2 kernel.~~ +* ~~Sandy bridge sgemm kernel. Fallback to Nehalem kernels.~~ +* ~~Sandy bridge cgemm kernel. Fallback to Nehalem kernels.~~ +* sbmv, zsbmv, smp bug (interface/sbmv.c zsbmv.c). Now, it is only sequential. +* zhbmv, smp bug. +* ~~scal, zscal x86/x86_64 kernels. Fallback to C implementation.~~ +* ~~potri, zpotri. Fallback to LAPACK reference implementation.~~ +* ~~lauu2, zlauu2. Fallback to LAPACK reference implementation.~~ +* ~~trtri, ztrtri. Fallback to LAPACK reference implementation.~~ +* ~~lauum, zlauum. Fallback to LAPACK reference implementation.~~ +* ~~trti2, ztrti2. Fallback to LAPACK reference implementation.~~ +* Disable SMP in ger.c and zger.c \ No newline at end of file diff --git a/Home.md b/Home.md new file mode 100644 index 0000000000..06b6f108db --- /dev/null +++ b/Home.md @@ -0,0 +1,56 @@ +# OpenBLAS Wiki + +
+
+ [Home] + [Document] + [FAQ] + [Publications] + [Download] +[Mailing List] +[Donation] +
+
+ +[![Build Status](https://travis-ci.org/xianyi/OpenBLAS.svg?branch=develop)](https://travis-ci.org/xianyi/OpenBLAS) + +### Introduction +OpenBLAS is an optimized Basic Linear Algebra Subprograms (BLAS) library based on [GotoBLAS2](https://www.tacc.utexas.edu/research-development/tacc-software/gotoblas2) 1.13 BSD version. + +### License + +OpenBLAS is licensed under the 3-clause BSD license. Full license text follows: + +Copyright (c) 2011-2015, The OpenBLAS Project +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. Neither the name of the OpenBLAS project nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +### Acknowledgements + +This work is partially supported by +* Research and Development of Compiler System and Toolchain for Domestic CPU, National S&T Major Projects: Core Electronic Devices, High-end General Chips and Fundamental Software (No.2009ZX01036-001-002) +* National High-tech R&D Program of China (Grant No.2012AA010903) \ No newline at end of file diff --git a/How-to-build-OpenBLAS-for-Android.md b/How-to-build-OpenBLAS-for-Android.md new file mode 100644 index 0000000000..2c2229e37f --- /dev/null +++ b/How-to-build-OpenBLAS-for-Android.md @@ -0,0 +1,190 @@ +## Prerequisites + +In addition to the Android NDK, you will need a C compiler on the build host as this is currently +required by the OpenBLAS build environment. + + +# Building with android NDK using clang compiler +Around version 11 Android NDKs stopped supporting gcc, so you would need to use clang to compile OpenBLAS. clang is supported from OpenBLAS 0.2.20 version onwards. See below sections on how to build with clang for ARMV7 and ARMV8 targets. The same basic principles as described below for ARMV8 should also apply to building an x86 or x86_64 version (substitute something like NEHALEM for the target instead of ARMV8 and replace all the aarch64 in the toolchain paths obviously) +"Historic" notes: +Since version 19 the default toolchain is provided as a standalone toolchain, so building one yourself following [building a standalone toolchain](http://developer.android.com/ndk/guides/standalone_toolchain.html) should no longer be necessary. +If you want to use static linking with an old NDK version older than about r17, you need to choose an API level below 23 currently due to NDK bug 272 (https://github.com/android-ndk/ndk/issues/272 , the libc.a lacks a definition of stderr) that will probably be fixed in r17 of the NDK. + +## Build ARMV7 with clang +``` +# Set path to ndk-bundle +export NDK_BUNDLE_DIR=/path/to/ndk-bundle + +# Set the PATH to contain paths to clang and arm-linux-androideabi-* utilities +export PATH=${NDK_BUNDLE_DIR}/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/bin:${NDK_BUNDLE_DIR}/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH + +# Set LDFLAGS so that the linker finds the appropriate libgcc +export LDFLAGS="-L${NDK_BUNDLE_DIR}/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/lib/gcc/arm-linux-androideabi/4.9.x" + +# Set the clang cross compile flags +export CLANG_FLAGS="-target arm-linux-androideabi -marm -mfpu=vfp -mfloat-abi=softfp --sysroot ${NDK_BUNDLE_DIR}/platforms/android-23/arch-arm -gcc-toolchain ${NDK_BUNDLE_DIR}/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/" + +#OpenBLAS Compile +make TARGET=ARMV7 ONLY_CBLAS=1 AR=ar CC="clang ${CLANG_FLAGS}" HOSTCC=gcc ARM_SOFTFP_ABI=1 -j4 +``` +On a Mac, it may also be necessary to give the complete path to the `ar` utility in the make command above, like so: +``` +AR=${NDK_BUNDLE_DIR}/toolchains/arm-linux-androideabi-4.9/prebuilt/darwin-x86_64/bin/arm-linux-androideabi-gcc-ar +``` +otherwise you may get a linker error complaining about a "malformed archive header name at 8" when the native OSX ar command was invoked instead. Note that with recent NDK versions, the AR tool may be named `llvm-ar` rather than what is assumed above. + +## Build ARMV8 with clang +``` +# Set path to ndk-bundle +export NDK_BUNDLE_DIR=/path/to/ndk-bundle/ + +# Export PATH to contain directories of clang and aarch64-linux-android-* utilities +export PATH=${NDK_BUNDLE_DIR}/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/bin/:${NDK_BUNDLE_DIR}/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH + +# Setup LDFLAGS so that loader can find libgcc and pass -lm for sqrt +export LDFLAGS="-L${NDK_BUNDLE_DIR}/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/lib/gcc/aarch64-linux-android/4.9.x -lm" + +# Setup the clang cross compile options +export CLANG_FLAGS="-target aarch64-linux-android --sysroot ${NDK_BUNDLE_DIR}/platforms/android-23/arch-arm64 -gcc-toolchain ${NDK_BUNDLE_DIR}/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/" + +# Compile +make TARGET=ARMV8 ONLY_CBLAS=1 AR=ar CC="clang ${CLANG_FLAGS}" HOSTCC=gcc -j4 +``` +Note: Using TARGET=CORTEXA57 in place of ARMV8 will pick up better optimized routines. Implementations for CORTEXA57 target is compatible with all other armv8 targets. + +Note: For NDK 23b, something as simple as +``` +export PATH=/opt/android-ndk-r23b/toolchains/llvm/prebuilt/linux-x86_64/bin/:$PATH +make HOSTCC=gcc CC=/opt/android-ndk-r23b/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android31-clang ONLY_CBLAS=1 TARGET=ARMV8 +``` +appears to be sufficient on Linux. On OSX, setting AR to the ar provided in the "bin" path of the NDK (probably llvm-ar) is also necessary. + +## Alternative script which was tested on OSX with NDK(21.3.6528147) +This script will build openblas for 3 architecture (ARMV7,ARMV8,X86) and put them with `sudo make install` to `/opt/OpenBLAS/lib`. +Of course you can also copy only the section that is of interest to you - also notice that the AR= line may need adapting to the +name of the ar tool provided in your $TOOLCHAIN/bin - for example llvm-ar in some recent NDK versions. +``` +export NDK=YOUR_PATH_TO_SDK/Android/sdk/ndk/21.3.6528147 +export TOOLCHAIN=$NDK/toolchains/llvm/prebuilt/darwin-x86_64 + +make clean +make \ + TARGET=ARMV7 \ + ONLY_CBLAS=1 \ + CC="$TOOLCHAIN"/bin/armv7a-linux-androideabi21-clang \ + AR="$TOOLCHAIN"/bin/arm-linux-androideabi-ar \ + HOSTCC=gcc \ + ARM_SOFTFP_ABI=1 \ + -j4 +sudo make install + +make clean +make \ + TARGET=CORTEXA57 \ + ONLY_CBLAS=1 \ + CC=$TOOLCHAIN/bin/aarch64-linux-android21-clang \ + AR=$TOOLCHAIN/bin/aarch64-linux-android-ar \ + HOSTCC=gcc \ + -j4 +sudo make install + +make clean +make \ + TARGET=ATOM \ + ONLY_CBLAS=1 \ + CC="$TOOLCHAIN"/bin/i686-linux-android21-clang \ + AR="$TOOLCHAIN"/bin/i686-linux-android-ar \ + HOSTCC=gcc \ + ARM_SOFTFP_ABI=1 \ + -j4 +sudo make install + +# This will build for x86_64 +make clean +make \ + TARGET=ATOM BINARY=64\ + ONLY_CBLAS=1 \ + CC="$TOOLCHAIN"/bin/x86_64-linux-android21-clang \ + AR="$TOOLCHAIN"/bin/x86_64-linux-android-ar \ + HOSTCC=gcc \ + ARM_SOFTFP_ABI=1 \ + -j4 +sudo make install +``` +Also you can find full list of target architectures in [TargetsList.txt](https://github.com/xianyi/OpenBLAS/blob/develop/TargetList.txt) + +*** +anything below this line should be irrelevant nowadays unless you need to perform software archeology +*** +## Building OpenBLAS with very old gcc-based versions of the NDK, without Fortran + +The prebuilt Android NDK toolchains do not include Fortran, hence parts like LAPACK cannot be built. You can still build OpenBLAS without it. For instructions on how to build OpenBLAS with Fortran, see the [next section](#building-openblas-with-fortran). + +To use easily the prebuilt toolchains, follow [building a standalone toolchain](http://developer.android.com/ndk/guides/standalone_toolchain.html) for your desired architecture. +This would be `arm-linux-androideabi-gcc-4.9` for ARMV7 and `aarch64-linux-android-gcc-4.9` for ARMV8. + +You can build OpenBLAS (0.2.19 and earlier) with: +``` +# Add the toolchain to your path +export PATH=/path/to/standalone-toolchain/bin:$PATH + +# Build without Fortran for ARMV7 +make TARGET=ARMV7 HOSTCC=gcc CC=arm-linux-androideabi-gcc NOFORTRAN=1 libs +# Build without Fortran for ARMV8 +make TARGET=ARMV8 BINARY=64 HOSTCC=gcc CC=aarch64-linux-android-gcc NOFORTRAN=1 libs +``` + +Since we are cross-compiling, we make the `libs` recipe, not `all`. Otherwise you will get errors when trying to link/run tests as versions up to and including 0.2.19 cannot build a shared library for Android. + +From 0.2.20 on, you should leave off the "libs" to get a full build, and you may want to use the softfp ABI instead of the deprecated hardfp one on ARMV7 so you would use +``` +# Add the toolchain to your path +export PATH=/path/to/standalone-toolchain/bin:$PATH + +# Build without Fortran for ARMV7 +make TARGET=ARMV7 ARM_SOFTFP_ABI=1 HOSTCC=gcc CC=arm-linux-androideabi-gcc NOFORTRAN=1 +# Build without Fortran for ARMV8 +make TARGET=ARMV8 BINARY=64 HOSTCC=gcc CC=aarch64-linux-android-gcc NOFORTRAN=1 +``` + +If you get an error about stdio.h not being found, you need to specify your sysroot in the CFLAGS argument to `make` like +```CFLAGS=--sysroot=$NDK/platforms/android-16/arch-arm``` +When you are done, install OpenBLAS into the desired directory. Be sure to also use all command line options +here that you specified for building, otherwise errors may occur as it tries to install things you did not build: +``` +make PREFIX=/path/to/install-dir TARGET=... install +``` + +## Building OpenBLAS with Fortran + +Instructions on how to build the GNU toolchains with Fortran can be found [here](https://github.com/buffer51/android-gfortran). The [Releases section](https://github.com/buffer51/android-gfortran/releases) provides prebuilt versions, use the standalone one. + +You can build OpenBLAS with: +``` +# Add the toolchain to your path +export PATH=/path/to/standalone-toolchain-with-fortran/bin:$PATH + +# Build with Fortran for ARMV7 +make TARGET=ARMV7 HOSTCC=gcc CC=arm-linux-androideabi-gcc FC=arm-linux-androideabi-gfortran libs +# Build with LAPACK for ARMV8 +make TARGET=ARMV8 BINARY=64 HOSTCC=gcc CC=aarch64-linux-android-gcc FC=aarch64-linux-android-gfortran libs +``` + +As mentioned above you can leave off the `libs` argument here when building 0.2.20 and later, and you may want to add ARM_SOFTFP_ABI=1 when building for ARMV7. + +## Linking OpenBLAS (0.2.19 and earlier) for ARMV7 + +If you are using `ndk-build`, you need to set the ABI to hard floating points in your Application.mk: +``` +APP_ABI := armeabi-v7a-hard +``` + +This will set the appropriate flags for you. If you are not using `ndk-build`, you will want to add the following flags: +``` +TARGET_CFLAGS += -mhard-float -D_NDK_MATH_NO_SOFTFP=1 +TARGET_LDFLAGS += -Wl,--no-warn-mismatch -lm_hard +``` + +From 0.2.20 on, it is also possible to build for the softfp ABI by specifying ARM_SOFTFP_ABI=1 during the build. +In that case, also make sure that all your dependencies are compiled with -mfloat-abi=softfp as well, as mixing +"hard" and "soft" floating point ABIs in a program will make it crash. diff --git a/How-to-build-OpenBLAS-for-Windows-on-ARM64.md b/How-to-build-OpenBLAS-for-Windows-on-ARM64.md new file mode 100644 index 0000000000..01c2a84dfb --- /dev/null +++ b/How-to-build-OpenBLAS-for-Windows-on-ARM64.md @@ -0,0 +1,77 @@ +This page describes how to natively build OpenBLAS library for windows on arm64 targets. + +See [below](#xcomp) for how to cross-compile OpenBLAS for WoA on an x86_64 Windows host. + +_We hope that the procedure can be simplified for/by LLVM 17+ and its flang-new compiler for the FORTRAN parts (LAPACK) of the code but we have no means of verifying that as of January 2024._ + + +# Prerequisite + +Following tools needs to be installed + +## 1. Download and install clang for windows on arm + +Find the latest LLVM build for WoA from [LLVM release page](https://releases.llvm.org/) + +E.g: LLVM 12 build for WoA64 can be found [here](https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.0/LLVM-12.0.0-woa64.exe) + +Run the LLVM installer and ensure that LLVM is added to environment PATH. + +## 2. Download and install classic flang for windows on arm + +Classic flang is the only available FORTRAN compiler for windows on arm for now and a pre-release build can be found [here](https://github.com/kaadam/flang/releases/tag/v0.1) + +There is no installer for classic flang and the zip package can be extracted and the path needs to be added to environment PATH. + +E.g: on PowerShell + +`$env:Path += ";C:\flang_woa\bin"` + +# Build + +The following steps describe how to build the static library for OpenBLAS with and without LAPACK + +## 1. Build OpenBLAS static library with BLAS and LAPACK routines with Make + +Following command can be used to build OpenBLAS static library with BLAS and LAPACK routines + +`make CC="clang-cl" HOSTCC="clang-cl" AR="llvm-ar" BUILD_WITHOUT_LAPACK=0 NOFORTRAN=0 DYNAMIC_ARCH=0 TARGET=ARMV8 ARCH=arm64 BINARY=64 USE_OPENMP=0 PARALLEL=1 RANLIB="llvm-ranlib" MAKE=make F_COMPILER=FLANG FC=FLANG FFLAGS_NOOPT="-march=armv8-a -cpp" FFLAGS="-march=armv8-a -cpp" NEED_PIC=0 HOSTARCH=arm64 libs netlib` + +## 2. Build static library with BLAS routines using CMake + +Classic flang has compatibility issues with cmake hence only BLAS routines can be compiled with CMake + +`mkdir build` + +`cd build` + +`cmake .. -G Ninja -DCMAKE_C_COMPILER=clang -DBUILD_WITHOUT_LAPACK=1 -DNOFORTRAN=1 -DDYNAMIC_ARCH=0 -DTARGET=ARMV8 -DARCH=arm64 -DBINARY=64 -DUSE_OPENMP=0 -DCMAKE_SYSTEM_PROCESSOR=ARM64 -DCMAKE_CROSSCOMPILING=1 -DCMAKE_SYSTEM_NAME=Windows` + +`cmake --build . --config Release` + +# Known issue + +## `getarch.exe` execution error + +If you notice that platform-specific headers by `getarch.exe` are not generated correctly, It could be due to a known debug runtime DLL issue for arm64 platforms. Please check out [link](https://linaro.atlassian.net/wiki/spaces/WOAR/pages/28677636097/Debug+run-time+DLL+issue#Workaround) for the workaround. + +# Alternative: cross-compiling on Windows x86 +## Prerequisites: +a working installation of LLVM and Ninja (the versions included in the latest Visual Studio 2022 will do, if you install its optional Llvm.Clang component) + +## Build + 1. Load into the appropriate cross compiling developer shell (in this case powershell, for cmd.exe it would be vcvarsamd64_arm64.bat): + 'C:\Program Files\Microsoft Visual Studio\2022\Professional\Common7\Tools\Launch-VsDevShell.ps1' -Arch arm64 -HostArch amd64 + + 2. Change to the folder where you unpacked the OpenBLAS sources if you haven't already, and create a subfolder for building. + Then change into that folder. + + 3. Invoke the following CMake command + + ``` + cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DTARGET=ARMV8 -DCMAKE_CROSSCOMPILING=ON -DCMAKE_SYSTEM_NAME="Windows" -DARCH=arm -DBINARY=64 -DCMAKE_SYSTEM_PROCESSOR=ARM64 -DCMAKE_C_COMPILER=clang-cl -DCMAKE_C_COMPILER_TARGET=arm64-pc-windows-msvc -DCMAKE_ASM_COMPILER_TARGET=arm64-pc-windows-msvc. + ``` + (Note the use of clang-cl instead of clang, and adding in the _COMPILER_TARGET entries. Without C_COMPILER_TARGET, clang-cl will be trying to compile x64 machine code and not make it past the cmake generation step. Without ASM_COMPILER_TARGET the assembly files in openblas will fail to assemble because the assembler is trying to interpret them as x64 assembly.) + + 4. Invoke Ninja for the actual compilation + diff --git a/How-to-build-OpenBLAS-for-iPhone-iOS-(ARMv8).md b/How-to-build-OpenBLAS-for-iPhone-iOS-(ARMv8).md new file mode 100644 index 0000000000..f0b054e13b --- /dev/null +++ b/How-to-build-OpenBLAS-for-iPhone-iOS-(ARMv8).md @@ -0,0 +1,14 @@ +# How to build OpenBLAS for iPhone/iOS + +As none of the current developers uses iOS, the following instructions are what was found to work in our Azure CI setup, but as far as we know this builds a fully working OpenBLAS for this platform. + +Go to the directory where you unpacked OpenBLAS,and enter the following commands: +``` + CC=/Applications/Xcode_12.4.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang + +CFLAGS= -O2 -Wno-macro-redefined -isysroot /Applications/Xcode_12.4.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS14.4.sdk -arch arm64 -miphoneos-version-min=10.0 + +make TARGET=ARMV8 DYNAMIC_ARCH=1 NUM_THREADS=32 HOSTCC=clang NOFORTRAN=1 +``` +Adjust MIN_IOS_VERSION as necessary for your installation, e.g. change the version number +to the minimum iOS version you want to target and execute this file to build the library. \ No newline at end of file diff --git a/How-to-build-OpenBLAS-for-macOS-M1---arm64.md b/How-to-build-OpenBLAS-for-macOS-M1---arm64.md new file mode 100644 index 0000000000..77e3b9e83b --- /dev/null +++ b/How-to-build-OpenBLAS-for-macOS-M1---arm64.md @@ -0,0 +1 @@ +On newer versions of Xcode and on arm64, you might need to compile with a newer macOS target (11.0) than the default (10.8) with `MACOSX_DEPLOYMENT_TARGET=11.0`, or switch your command-line tools to use an older SDK (e.g., [13.1](https://developer.apple.com/download/all/?q=Xcode%2013)). \ No newline at end of file diff --git a/How-to-generate-import-library-for-MinGW.md b/How-to-generate-import-library-for-MinGW.md new file mode 100644 index 0000000000..a0038c1de9 --- /dev/null +++ b/How-to-generate-import-library-for-MinGW.md @@ -0,0 +1,36 @@ +Microsoft Windows has this thing called "import libraries". You don't need it in MinGW because the `ld` linker from GNU Binutils is smart, but you may still want it for whatever reason. + +## Make the `.def` + +Import libraries are compiled from a list of what symbols to use, `.def`. This should be already in your `exports` directory: `cd OPENBLAS_TOP_DIR/exports`. + +## Making a MinGW import library + +MinGW import libraries have the suffix `.a`, same as static libraries. (It's actually more common to do `.dll.a`...) + +You need to first prepend `libopenblas.def` with a line `LIBRARY libopenblas.dll`: + + cat <(echo "LIBRARY libopenblas.dll") libopenblas.def > libopenblas.def.1 + mv libopenblas.def.1 libopenblas.def + +Now it probably looks like: + + LIBRARY libopenblas.dll + EXPORTS + caxpy=caxpy_ @1 + caxpy_=caxpy_ @2 + ... + +Then, generate the import library: `dlltool -d libopenblas.def -l libopenblas.a` + +Again, there is basically **no point** in making an import library for use in MinGW. It actually slows down linking. + +## Making a MSVC import library + +Unlike MinGW, MSVC absolutely requires an import library. Now the C ABI of MSVC and MinGW are actually identical, so linking is actually okay. (Any incompatibility in the C ABI would be a bug.) + +The import libraries of MSVC have the suffix `.lib`. They are generated from a `.def` file using MSVC's `lib.exe`. See [the MSVC instructions](https://github.com/xianyi/OpenBLAS/wiki/How-to-use-OpenBLAS-in-Microsoft-Visual-Studio#generate-import-library-before-0210-version). + +## Notes +* Always remember that MinGW is **not the same** as MSYS2 or Cygwin. MSYS2 and Cygwin are full POSIX environments with a lot of magic such as `fork()` and its own `malloc()`. MinGW, which builds on the normal Microsoft C Runtime, has none of that. Be clear about which one you are building for. + diff --git a/How-to-use-OpenBLAS-in-Microsoft-Visual-Studio.md b/How-to-use-OpenBLAS-in-Microsoft-Visual-Studio.md new file mode 100644 index 0000000000..fe031383f6 --- /dev/null +++ b/How-to-use-OpenBLAS-in-Microsoft-Visual-Studio.md @@ -0,0 +1,162 @@ +As of OpenBLAS v0.2.15, we support MinGW and Visual Studio (using CMake to generate visual studio solution files – note that you will need at least version 3.11 of CMake for linking to work correctly) to build OpenBLAS on Windows. + +Note that you need a Fortran compiler if you plan to build and use the LAPACK functions included with OpenBLAS. The sections below describe using either `flang` as an add-on to clang/LLVM or `gfortran` as part of MinGW for this purpose. If you want to use the Intel Fortran compiler `ifort` for this, be sure to also use the Intel C compiler `icc` for building the C parts, as the ABI imposed by `ifort` is incompatible with `msvc`. + +## 1. Native (MSVC) ABI + +A fully-optimized OpenBLAS that can be statically or dynamically linked to your application can currently be built for the 64-bit architecture with the LLVM compiler infrastructure. We're going to use Miniconda3 to grab all of the tools we need, since some of them are in an experimental status. Before you begin, you'll need to have Microsoft Visual Studio 2015 or newer installed. + +1. Install Miniconda3 for 64 bits using `winget install --id Anaconda.Miniconda3` or easily download from [conda.io](https://docs.conda.io/en/latest/miniconda.html). +2. Open the "Anaconda Command Prompt," now available in the Start Menu, or at `%USERPROFILE%\miniconda3\shell\condabin\conda-hook.ps1`. +3. In that command prompt window, use `cd` to change to the directory where you want to build OpenBLAS +4. Now install all of the tools we need: + + ``` + conda update -n base conda + conda config --add channels conda-forge + conda install -y cmake flang clangdev perl libflang ninja + ``` + +5. Still in the Anaconda Command Prompt window, activate the MSVC environment for 64 bits with `vcvarsall x64`. On Windows 11 with Visual Studio 2022, this would be done by invoking: + + ```shell + "c:\Program Files\Microsoft Visual Studio\2022\Preview\vc\Auxiliary\Build\vcvars64.bat" + ``` + + With VS2019, the command should be the same – except for the year number, obviously. For other/older versions of MSVC, + the VS documentation or a quick search on the web should turn up the exact wording you need. + + Confirm that the environment is active by typing `link` – this should return a long list of possible options for the `link` command. If it just + returns "command not found" or similar, review and retype the call to vcvars64.bat. + **NOTE:** if you are working from a Visual Studio Command prompt window instead (so that you do not have to do the vcvars call), you need to invoke + `conda activate` so that CONDA_PREFIX etc. get set up correctly before proceeding to step 6. Failing to do so will lead to link errors like + libflangmain.lib not getting found later in the build. + +6. Now configure the project with CMake. Starting in the project directory, execute the following: + + ``` + set "LIB=%CONDA_PREFIX%\Library\lib;%LIB%" + set "CPATH=%CONDA_PREFIX%\Library\include;%CPATH%" + mkdir build + cd build + cmake .. -G "Ninja" -DCMAKE_CXX_COMPILER=clang-cl -DCMAKE_C_COMPILER=clang-cl -DCMAKE_Fortran_COMPILER=flang -DCMAKE_MT=mt -DBUILD_WITHOUT_LAPACK=no -DNOFORTRAN=0 -DDYNAMIC_ARCH=ON -DCMAKE_BUILD_TYPE=Release + ``` + + You may want to add further options in the `cmake` command here – for instance, the default only produces a static .lib version of the library. If you would rather have a DLL, add -DBUILD_SHARED_LIBS=ON above. Note that this step only creates some command files and directories, the actual build happens next. + + +7. Build the project: + + ``` + cmake --build . --config Release + ``` + This step will create the OpenBLAS library in the "lib" directory, and various build-time tests in the `test`, `ctest` and `openblas_utest` directories. However it will not separate the header files you might need for building your own programs from those used internally. To put all relevant files in a more convenient arrangement, run the next step. + +8. Install all relevant files created by the build + + ``` + cmake --install . --prefix c:\opt -v + ``` + This will copy all files that are needed for building and running your own programs with OpenBLAS to the given location, creating appropriate subdirectories for the individual kinds of files. In the case of "C:\opt" as given above, this would be C:\opt\include\openblas for the header files, + C:\opt\bin for the libopenblas.dll and C:\opt\lib for the static library. C:\opt\share holds various support files that enable other cmake-based build scripts to find OpenBLAS automatically. + +### Visual studio 2017+ (C++2017 standard) + +In newer visual studio versions, Microsoft has changed [how it handles complex types](https://docs.microsoft.com/en-us/cpp/c-runtime-library/complex-math-support?view=msvc-170#types-used-in-complex-math). Even when using a precompiled version of OpenBLAS, you might need to define `LAPACK_COMPLEX_CUSTOM` in order to define complex types properly for MSVC. For example, some variant of the following might help: + +``` +#if defined(_MSC_VER) + #include + #define LAPACK_COMPLEX_CUSTOM + #define lapack_complex_float _Fcomplex + #define lapack_complex_double _Dcomplex +#endif +``` + +For reference, see https://github.com/xianyi/OpenBLAS/issues/3661, https://github.com/Reference-LAPACK/lapack/issues/683, and https://stackoverflow.com/questions/47520244/using-openblas-lapacke-in-visual-studio. + +### CMake and Visual Studio + +To build OpenBLAS for the 32-bit architecture, you'll need to use the builtin Visual Studio compilers. + +**[Notice]** This method may produce binaries which demonstrate significantly lower performance than those built with the other methods. (The Visual Studio compiler does not support the dialect of assembly used in the cpu-specific optimized files, so only the "generic" TARGET which is +written in pure C will get built. For the same reason it is not possible (and not necessary) to use -DDYNAMIC_ARCH=ON in a Visual Studio build) You may consider building for the 32-bit architecture using the GNU (MinGW) ABI. + +#### 1. Install CMake at Windows + +#### 2. Use CMake to generate Visual Studio solution files + +``` +# Do this from Powershell so cmake can find visual studio +cmake -G "Visual Studio 14 Win64" -DCMAKE_BUILD_TYPE=Release . +``` + +### Build the solution at Visual Studio + +Note that this step depends on perl, so you'll need to install perl for windows, and put perl on your path so VS can start perl (http://stackoverflow.com/questions/3051049/active-perl-installation-on-windows-operating-system). + +Step 2 will build the OpenBLAS solution, open it in VS, and build the projects. Note that the dependencies do not seem to be automatically configured: if you try to build libopenblas directly, it will fail with a message saying that some .obj files aren't found, but if you build the projects libopenblas depends on before building libopenblas, the build will succeed. + +### Build OpenBLAS for Universal Windows Platform + +OpenBLAS can be built for use on the [Universal Windows Platform](https://en.wikipedia.org/wiki/Universal_Windows_Platform) using a two step process since commit [c66b842](https://github.com/xianyi/OpenBLAS/commit/c66b842d66c5516e52804bf5a0544d18b1da1b44). + +#### 1. Follow steps 1 and 2 above to build the Visual Studio solution files for Windows. This builds the helper executables which are required when building the OpenBLAS Visual Studio solution files for UWP in step 2. + +#### 2. Remove the generated CMakeCache.txt and CMakeFiles directory from the OpenBLAS source directory and re-run CMake with the following options: + +``` +# do this to build UWP compatible solution files +cmake -G "Visual Studio 14 Win64" -DCMAKE_SYSTEM_NAME=WindowsStore -DCMAKE_SYSTEM_VERSION="10.0" -DCMAKE_SYSTEM_PROCESSOR=AMD64 -DVS_WINRT_COMPONENT=TRUE -DCMAKE_BUILD_TYPE=Release . +``` + +#### Build the solution with Visual Studio + +This will build the OpenBLAS binaries with the required settings for use with UWP. + +## 2. GNU (MinGW) ABI + +The resulting library can be used in Visual Studio, but it can only be linked dynamically. This configuration has not been thoroughly tested and should be considered experimental. + +### Incompatible x86 calling conventions + +Due to incompatibilities between the calling conventions of MinGW and Visual Studio you will need to make the following modifications ( **32-bit only** ): + +1. Use the newer GCC 4.7.0. The older GCC (<4.7.0) has an ABI incompatibility for returning aggregate structures larger than 8 bytes with MSVC. + + +### Build OpenBLAS on Windows OS +1. Install the MinGW (GCC) compiler suite, either 32-bit (http://www.mingw.org/) or 64-bit (http://mingw-w64.sourceforge.net/). Be sure to install its gfortran package as well (unless you really want to build the BLAS part of OpenBLAS only) and check that gcc and gfortran are the same version – mixing compilers from different sources or release versions can lead to strange error messages in the linking stage. In addition, please install MSYS with MinGW. +1. Build OpenBLAS in the MSYS shell. Usually, you can just type "make". OpenBLAS will detect the compiler and CPU automatically. +1. After the build is complete, OpenBLAS will generate the static library "libopenblas.a" and the shared dll library "libopenblas.dll" in the folder. You can type "make PREFIX=/your/installation/path install" to install the library to a certain location. + +**[Notice]** We suggest using official MinGW or MinGW-w64 compilers. A user reported that s/he met `Unhandled exception` by other compiler suite. https://groups.google.com/forum/#!topic/openblas-users/me2S4LkE55w + +Note also that older versions of the alternative builds of mingw-w64 available through http://www.msys2.org may contain a defect that leads to a compilation failure accompanied by the error message +``` +:0:4: error: expected identifier or '(' before numeric constant +``` +If you encounter this, please upgrade your msys2 setup or see https://github.com/xianyi/OpenBLAS/issues/1503 for a workaround. + +### Generate import library (before 0.2.10 version) + +1. First, you will need to have the `lib.exe` tool in the Visual Studio command prompt. +1. Open the command prompt and type `cd OPENBLAS_TOP_DIR/exports`, where OPENBLAS_TOP_DIR is the main folder of your OpenBLAS installation. +1. For a 32-bit library, type `lib /machine:i386 /def:libopenblas.def`. For 64-bit, type `lib /machine:X64 /def:libopenblas.def`. +1. This will generate the import library "libopenblas.lib" and the export library "libopenblas.exp" in OPENBLAS_TOP_DIR/exports. Although these two files have the same name, they are totally different. + +### Generate import library (0.2.10 and after version) +1. OpenBLAS already generated the import library "libopenblas.dll.a" for "libopenblas.dll". + +### generate windows native PDB files from gcc/gfortran build +Tool to do so is available at https://github.com/rainers/cv2pdb + +### Use OpenBLAS .dll library in Visual Studio +1. Copy the import library (before 0.2.10: "OPENBLAS_TOP_DIR/exports/libopenblas.lib", 0.2.10 and after: "OPENBLAS_TOP_DIR/libopenblas.dll.a") and .dll library "libopenblas.dll" into the same folder(The folder of your project that is going to use the BLAS library. You may need to add the libopenblas.dll.a to the linker input list: properties->Linker->Input). +1. Please follow the documentation about using third-party .dll libraries in MS Visual Studio 2008 or 2010. Make sure to link against a library for the correct architecture. For example, you may receive an error such as "The application was unable to start correctly (0xc000007b)" which typically indicates a mismatch between 32/64-bit libraries. + +**[Notice]** If you need CBLAS, you should include cblas.h in /your/installation/path/include in Visual Studio. Please read [this page](http://github.com/xianyi/OpenBLAS/issues/95). + +### Limitations +* Both static and dynamic linking are supported with MinGW. With Visual Studio, however, only dynamic linking is supported and so you should use the import library. +* Debugging from Visual Studio does not work because MinGW and Visual Studio have incompatible formats for debug information (PDB vs. DWARF/STABS). You should either debug with GDB on the command-line or with a visual frontend, for instance [Eclipse](http://www.eclipse.org/cdt/) or [Qt Creator](http://qt.nokia.com/products/developer-tools/). diff --git a/How-to-use-OpenBLAS-on-Cortex-M.md b/How-to-use-OpenBLAS-on-Cortex-M.md new file mode 100644 index 0000000000..a90b1324f2 --- /dev/null +++ b/How-to-use-OpenBLAS-on-Cortex-M.md @@ -0,0 +1,37 @@ +Cortex-M is a widely used microcontroller that is present in a variety of industrial and consumer electronics. +A common variant of the Cortex-M is the STM32F4xx series. Here, we will give instructions for building for +the STM32F4xx. + +First, install the embedded arm gcc compiler from the arm website. Then, create the following toolchain file and build as follows. + +```cmake +# cmake .. -G Ninja -DCMAKE_C_COMPILER=arm-none-eabi-gcc -DCMAKE_TOOLCHAIN_FILE:PATH="toolchain.cmake" -DNOFORTRAN=1 -DTARGET=ARMV5 -DEMBEDDED=1 + +set(CMAKE_SYSTEM_NAME Generic) +set(CMAKE_SYSTEM_PROCESSOR arm) + +set(CMAKE_C_COMPILER "arm-none-eabi-gcc.exe") +set(CMAKE_CXX_COMPILER "arm-none-eabi-g++.exe") + +set(CMAKE_EXE_LINKER_FLAGS "--specs=nosys.specs" CACHE INTERNAL "") + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) +``` + +In your embedded application, the following functions need to be provided for OpenBLAS to work correctly: + +```C +void free(void* ptr); +void* malloc(size_t size); +``` + +Note: if you are developing for an embedded platform, it is your responsibility to make sure that the device +has sufficient memory for malloc calls. [Libmemory][1] provides one implementation of malloc for embedded +platforms. + + +[1]: https://github.com/embeddedartistry/libmemory + diff --git a/Installation-Guide.md b/Installation-Guide.md new file mode 100644 index 0000000000..84c8e67998 --- /dev/null +++ b/Installation-Guide.md @@ -0,0 +1,71 @@ +## Quick Installation +[Precompiled packages](https://github.com/xianyi/OpenBLAS/wiki/Precompiled-installation-packages) have recently become available for a number of platforms through their normal installation procedures, so for users of desktop devices at least, the instructions below are mostly relevant when you want to try the most recent development snapshot from git. + +### Linux + +Just type `make` to compile the library. + +Notes +* OpenBLAS doesn't support g77. Please use gfortran or other Fortran compilers. e.g. `make FC=gfortran`. +* When building in an emulator (KVM,QEMU etc.) make sure that the combination of CPU features exposed to + the virtual environment matches that of an existing CPU to allow detection of the cpu model to succeed. + (With qemu, this can be done by passing `-cpu host` or a supported model name at invocation) + +### Windows + +See [[How-to-use-OpenBLAS-in-Microsoft-Visual-Studio]]. + +The precompiled binaries available with each release (in https://github.com/xianyi/OpenBLAS/releases) are +created with MinGW (as described in Section 2 of the wiki page mentioned above) using an option list of +"NUM_THREADS=64 TARGET=GENERIC DYNAMIC_ARCH=1 DYNAMIC_OLDER=1 CONSISTENT_FPCSR=1" - they should work on +any x86 or x86_64 computer. The zip archive contains the include files, static and dll libraries as well +as configuration files for getting them found via CMAKE or pkgconfig - just create a suitable folder for +your OpenBLAS installation and unzip it there. (Note that you will need to edit the provided openblas.pc +and OpenBLASConfig.cmake to reflect the installation path on your computer, as distributed they have "win" +or "win64" reflecting the local paths on the system they were built on). Some programs will expect the DLL +name to be lapack.dll, blas.dll, or (in the case of the statistics package "R") even Rblas.dll to act as a +direct replacement for whatever other implementation of BLAS and LAPACK they use by default. Just copy the +openblas.dll to the desired name(s). +Note that the provided binaries are built with INTERFACE64=0, meaning they use standard 32bit integers for +array indexing and the like (as is the default for most if not all BLAS and LAPACK implementations). If the +documentation of whatever program you are using with OpenBLAS mentions 64bit integers (INTERFACE64=1) for +addressing huge matrix sizes, you will need to build OpenBLAS from source (or open an issue ticket to make +the demand for such a precompiled build known). + +### Mac OSX + +If your CPU is Sandy Bridge, please use Clang version 3.1 and above. The Clang 3.0 will generate the wrong AVX binary code of OpenBLAS. + +#### Build on Apple M1 + +* without Fortran compiler （cannot build LAPACK) +``` + $ make CC=cc NOFORTRAN=1 +``` +* with Fortran compiler (you could `brew install gfortran`) https://github.com/xianyi/OpenBLAS/issues/3032 +``` + $ export MACOSX_DEPLOYMENT_TARGET=11.0 + $ make CC=cc FC=gfortran +``` +### FreeBSD + +You will need to install the following tools from the FreeBSD ports tree: +* lang/gcc [1] +* lang/perl5.12 +* ftp/curl +* devel/gmake +* devel/patch + +To compile run the command: + + $ gmake CC=gcc46 FC=gfortran46 + +Note that you need to build with GNU make and manually specify the compiler, otherwhise gcc 4.2 from the base system would be used. + +[1]: [Removal of Fortran from the FreeBSD base system](http://www.bsdunix.ch/serendipity/index.php?/archives/345-Removal-of-Fortran-from-the-FreeBSD-base-system.html) + +### Android +See [this page](https://github.com/xianyi/OpenBLAS/wiki/How-to-build-OpenBLAS-for-Android) + +### MIPS +See [this page](https://github.com/xianyi/OpenBLAS/wiki/Faq#mips) \ No newline at end of file diff --git a/Machine-List.md b/Machine-List.md new file mode 100644 index 0000000000..2039b3a8fa --- /dev/null +++ b/Machine-List.md @@ -0,0 +1,9 @@ +We deployed the following machine to support developing OpenBLAS. The interested developer can apply the remote access by sending the mail to [OpenBLAS-dev mailing list](https://groups.google.com/forum/#!forum/openblas-dev). + +* 2-way Intel Xeon E5-2680 (2.7GHz, Intel Sandy Bridge), 32GB memory, 1TB hard disk, Ubuntu 14.04 64-bit. +* 1-way Intel Core i7-4770 (3.4GHz, Intel Haswell), 8GB memory, 512GB hard disk, Ubuntu 14.04 64-bit. +* 1-way AMD FX-8320 (3.5GHz, AMD Piledriver), 8GB memory, 512GB hard disk, Ubuntu 14.04 64-bit. +* [Odroid-U2 board](http://www.hardkernel.com/main/products/prdt_info.php?g_code=G135341370451). 1-way Exynos4412 Prime 1.7Ghz ARM Cortex-A9 Quad Cores, 2GB memory, 16GB SD card, Ubuntu 14.04 32-bit. +* AMD A10-7850K APU (Kaveri, Steamroller+GCN GPU), 8GB memory, 1TB hard disk, Ubuntu 14.04.1 64-bit +* NVIDIA TK1 (ARM Cortex-A15), ARM 32-bit, Ubuntu +* NVIDIA TX1 (ARM Cortex-A57), AArch64, Ubuntu \ No newline at end of file diff --git a/Mailing-List.md b/Mailing-List.md new file mode 100644 index 0000000000..8de323e639 --- /dev/null +++ b/Mailing-List.md @@ -0,0 +1,15 @@ +
+
+ [Home] + [Document] + [FAQ] + [Publications] + [Download] +[Mailing List] +[Donation] +
+
+ +OpenBLAS users: https://groups.google.com/forum/#!forum/openblas-users + +OpenBLAS developers: https://groups.google.com/forum/#!forum/openblas-dev \ No newline at end of file diff --git a/OpenBLAS-Extensions.md b/OpenBLAS-Extensions.md new file mode 100644 index 0000000000..3502f6487a --- /dev/null +++ b/OpenBLAS-Extensions.md @@ -0,0 +1,28 @@ +* BLAS-like extensions + +| Routine | Data Types | Description | +| ------------- |:------------- | :---------------| +| ?axpby | s,d,c,z | like axpy with a multiplier for y | +| ?gemm3m | c,z | gemm3m | +| ?imatcopy | s,d,c,z | in-place transpositon/copying | +| ?omatcopy | s,d,c,z | out-of-place transpositon/copying | +| ?geadd | s,d,c,z | matrix add | +| ?gemmt | s,d,c,z | gemm but only a triangular part updated| + +* BLAS-like and Conversion functions for bfloat16 (available when OpenBLAS was compiled with BUILD_BFLOAT16=1) + * `void cblas_sbstobf16` converts a float array to an array of bfloat16 values by rounding + * `void cblas_sbdtobf16` converts a double array to an array of bfloat16 values by rounding + * `void cblas_sbf16tos` converts a bfloat16 array to an array of floats + * `void cblas_dbf16tod` converts a bfloat16 array to an array of doubles + * `float cblas_sbdot` computes the dot product of two bfloat16 arrays + * `void cblas_sbgemv` performs the matrix-vector operations of GEMV with the input matrix and X vector as bfloat16 + * `void cblas_sbgemm` performs the matrix-matrix operations of GEMM with both input arrays containing bfloat16 + +* Utility functions + * openblas_get_num_threads + * openblas_set_num_threads + * `int openblas_get_num_procs(void)` returns the number of processors available on the system (may include "hyperthreading cores") + * `int openblas_get_parallel(void)` returns 0 for sequential use, 1 for platform-based threading and 2 for OpenMP-based threading + * `char * openblas_get_config()` returns the options OpenBLAS was built with, something like `NO_LAPACKE DYNAMIC_ARCH NO_AFFINITY Haswell` + * `int openblas_set_affinity(int thread_index, size_t cpusetsize, cpu_set_t *cpuset)` sets the cpu affinity mask of the given thread to the provided cpuset. (Only available under Linux, with semantics identical to pthread_setaffinity_np) + diff --git a/Precompiled-installation-packages.md b/Precompiled-installation-packages.md new file mode 100644 index 0000000000..d7796db14d --- /dev/null +++ b/Precompiled-installation-packages.md @@ -0,0 +1,84 @@ +# Precompiled installation packages +_note this list is not comprehensive, is not meant to validate nor endorse a particular third-party build over others and may not always lead to the newest version_ + +## Multiple operating systems +Binaries designed to be compatible with [Julia](https://github.com/JuliaLang/julia) are regularly provided in +https://github.com/staticfloat/OpenBLASBuilder/releases . Note that their 64bit builds with INTERFACE64=1 have +`_64` appended to the function symbol names, so your code needs to refer to e.g. gemm_64_ rather than gemm_ + +the [Conda-Forge](https://github.com/conda-forge) project maintains packages for the conda package manager at +https://github.com/conda-forge/openblas-feedstock +## FreeBSD + > pkg install openblas + + see https://www.freebsd.org/ports/index.html +## Linux +### Debian/Ubuntu/Mint/Kali + OpenBLAS package is available in default repositories and can act as default BLAS in system + +Example installation commands: +``` + $ sudo apt update + $ apt search openblas + $ sudo apt install libopenblas-dev + $ sudo update-alternatives --config libblas.so.3 +``` + Alternatively, if distributor's package proves unsatisfactory, you may try latest version of OpenBLAS, [Following guide in OpenBLAS FAQ](https://github.com/xianyi/OpenBLAS/wiki/faq#debianlts) + +### openSuSE/SLE: + Recent OpenSUSE versions include OpenBLAS in default repositories and also permit OpenBLAS to act as replacement of system-wide BLAS. + + Example installation commands: +``` + $ sudo zypper ref + $ zypper se openblas + $ sudo zypper in openblas-devel + $ sudo update-alternatives --config libblas.so.3 +``` +Should you be using older OpenSUSE or SLE that provides no OpenBLAS, you can attach optional or experimental openSUSE repository as a new package source to acquire recent build of OpenBLAS following [instructions on openSUSE software site](https://software.opensuse.org/package/openblas) + +### Fedora/CentOS/RHEL +Fedora provides OpenBLAS in default installation repositories. + +To install it try following: +``` + $ dnf search openblas + $ dnf install openblas-devel +``` +For CentOS/RHEL/Scientific Linux packages are provided via [Fedora EPEL repository](https://fedoraproject.org/wiki/EPEL) + +After adding repository and repository keys installation is pretty straightforward: +``` + $ yum search openblas + $ yum install openblas-devel +``` +No alternatives mechanism is provided for BLAS, and packages in system repositories are linked against NetLib BLAS or ATLAS BLAS libraries. You may wish to re-package RPMs to use OpenBLAS instead [as described here](https://fedoraproject.org/wiki/How_to_create_an_RPM_package) + +### Mageia +Mageia offers ATLAS and NetLIB LAPACK in base repositories. +You can build your own OpenBLAS replacement, and once installed in /opt +TODO: populate /usr/lib64 /usr/include accurately to replicate netlib with update-alternatives + +### Arch/Manjaro/Antergos +``` + $ sudo pacman -S openblas +``` + +## Solaris (via pkgsrc) + + +## OSX +https://www.macports.org/ports.php?by=name&substr=openblas + +`brew install openblas` + +or using the conda package manager from +https://github.com/conda-forge/miniforge#download +(which also has packages for the new M1 cpu) + + `conda install openblas` + +## Windows +http://sourceforge.net/projects/openblas/files + +https://www.nuget.org/packages?q=openblas diff --git a/Publications.md b/Publications.md new file mode 100644 index 0000000000..4161fd4443 --- /dev/null +++ b/Publications.md @@ -0,0 +1,19 @@ +
+
+ [Home] + [Document] + [FAQ] + [Publications] + [Download] +[Mailing List] +[Donation] +
+
+ +### 2013 + +* Wang Qian, Zhang Xianyi, Zhang Yunquan, Qing Yi, **AUGEM: Automatically Generate High Performance Dense Linear Algebra Kernels on x86 CPUs**, In the International Conference for High Performance Computing, Networking, Storage and Analysis (SC'13), Denver CO, November 2013. [[pdf](http://xianyi.github.io/paper/augem_SC13.pdf)] + +### 2012 + +* Zhang Xianyi, Wang Qian, Zhang Yunquan, **Model-driven Level 3 BLAS Performance Optimization on Loongson 3A Processor**, 2012 IEEE 18th International Conference on Parallel and Distributed Systems (ICPADS), 17-19 Dec. 2012. \ No newline at end of file diff --git a/Related-packages-that-use-OpenBLAS.md b/Related-packages-that-use-OpenBLAS.md new file mode 100644 index 0000000000..d667f6c405 --- /dev/null +++ b/Related-packages-that-use-OpenBLAS.md @@ -0,0 +1,13 @@ +## Softwares + +* Julia - a high-level, high-performance dynamic programming language for technical computing
+* Ceemple v1.0.3 (C++ technical computing environment), including OpenBLAS, Qt, Boost, OpenCV and others. The only solution with immediate-recompilation of C++ code. Available from Ceemple C++ Technical Computing. +* [netlib-java](https://github.com/fommil/netlib-java) and various upstream libraries, allowing OpenBLAS to be used from languages on the Java Virtual Machine. + +## Academia Users + +## Industry Users + +## HPC Centers Deployed OpenBLAS + +## Others diff --git a/User-Manual.md b/User-Manual.md new file mode 100644 index 0000000000..2efb0f16d8 --- /dev/null +++ b/User-Manual.md @@ -0,0 +1,187 @@ +##### Table of Contents +[Compile the library](#compile-the-library) +[Link the library](#link-the-library) +[Code examples](#code-examples) +[Troubleshooting](#troubleshooting) +[BLAS reference manual](#blas-reference-manual) + + +## Compile the library +### Normal compile + * type `make` to detect the CPU automatically. + or + * type `make TARGET=xxx` to set target CPU, e.g. `make TARGET=NEHALEM`. The full target list is in file TargetList.txt. + +### Cross compile +Please set `CC` and `FC` with the cross toolchains. Then, set `HOSTCC` with your host C compiler. At last, set `TARGET` explicitly. + +Examples: + +* On x86 box, compile the library for ARM Cortex-A9 linux. + +Install only gnueabihf versions. Please check https://github.com/xianyi/OpenBLAS/issues/936#issuecomment-237596847 + + make CC=arm-linux-gnueabihf-gcc FC=arm-linux-gnueabihf-gfortran HOSTCC=gcc TARGET=CORTEXA9 + +* On X86 box, compile this library for loongson3a CPU. + +``` +make BINARY=64 CC=mips64el-unknown-linux-gnu-gcc FC=mips64el-unknown-linux-gnu-gfortran HOSTCC=gcc TARGET=LOONGSON3A +``` + +* On X86 box, compile this library for loongson3a CPU with loongcc (based on Open64) compiler. + +``` +make CC=loongcc FC=loongf95 HOSTCC=gcc TARGET=LOONGSON3A CROSS=1 CROSS_SUFFIX=mips64el-st-linux-gnu- NO_LAPACKE=1 NO_SHARED=1 BINARY=32 +``` + +### Debug version + + make DEBUG=1 + +### Install to the directory (optional) + +Example: + + make install PREFIX=your_installation_directory + +The default directory is /opt/OpenBLAS. Note that any flags passed to `make` during build should also be passed to `make install` to circumvent any install errors, i.e. some headers not being copied over correctly. + +For more information, please read [Installation Guide](Installation-Guide). + +## Link the library + +* Link shared library + +``` +gcc -o test test.c -I/your_path/OpenBLAS/include/ -L/your_path/OpenBLAS/lib -Wl,-rpath,/your_path/OpenBLAS/lib -lopenblas +``` + +The `-Wl,-rpath,/your_path/OpenBLAS/lib` option to linker can be omitted if you ran `ldconfig` to update linker cache, put `/your_path/OpenBLAS/lib` in `/etc/ld.so.conf` or a file in `/etc/ld.so.conf.d`, or installed OpenBLAS in a location part of `ld.so` default search path. Otherwise, linking at runtime will fail. + +If the library is multithreaded, please add `-lpthread`. If the library contains LAPACK functions, please add `-lgfortran` or other Fortran libs, although if you only make calls to LAPACKE routines, i.e. your code has `#include "lapacke.h"` and makes calls to methods like `LAPACKE_dgeqrf`, `-lgfortran` is not needed. + +* Link static library + +``` +gcc -o test test.c /your/path/libopenblas.a +``` + +You can download `test.c` from https://gist.github.com/xianyi/5780018 + +## Code examples + +### Call CBLAS interface +This example shows calling cblas_dgemm in C. https://gist.github.com/xianyi/6930656 +```c +#include +#include + +void main() +{ + int i=0; + double A[6] = {1.0,2.0,1.0,-3.0,4.0,-1.0}; + double B[6] = {1.0,2.0,1.0,-3.0,4.0,-1.0}; + double C[9] = {.5,.5,.5,.5,.5,.5,.5,.5,.5}; + cblas_dgemm(CblasColMajor, CblasNoTrans, CblasTrans,3,3,2,1,A, 3, B, 3,2,C,3); + + for(i=0; i<9; i++) + printf("%lf ", C[i]); + printf("\n"); +} +``` + +``` +gcc -o test_cblas_open test_cblas_dgemm.c -I /your_path/OpenBLAS/include/ -L/your_path/OpenBLAS/lib -lopenblas -lpthread -lgfortran +``` + +### Call BLAS Fortran interface + +This example shows calling dgemm Fortran interface in C. https://gist.github.com/xianyi/5780018 + +```c +#include "stdio.h" +#include "stdlib.h" +#include "sys/time.h" +#include "time.h" + +extern void dgemm_(char*, char*, int*, int*,int*, double*, double*, int*, double*, int*, double*, double*, int*); + +int main(int argc, char* argv[]) +{ + int i; + printf("test!\n"); + if(argc<4){ + printf("Input Error\n"); + return 1; + } + + int m = atoi(argv[1]); + int n = atoi(argv[2]); + int k = atoi(argv[3]); + int sizeofa = m * k; + int sizeofb = k * n; + int sizeofc = m * n; + char ta = 'N'; + char tb = 'N'; + double alpha = 1.2; + double beta = 0.001; + + struct timeval start,finish; + double duration; + + double* A = (double*)malloc(sizeof(double) * sizeofa); + double* B = (double*)malloc(sizeof(double) * sizeofb); + double* C = (double*)malloc(sizeof(double) * sizeofc); + + srand((unsigned)time(NULL)); + + for (i=0; i +``` + +## Troubleshooting + +* Please read [Faq](https://github.com/xianyi/OpenBLAS/wiki/Faq) at first. +* Please use gcc version 4.6 and above to compile Sandy Bridge AVX kernels on Linux/MingW/BSD. +* Please use Clang version 3.1 and above to compile the library on Sandy Bridge microarchitecture. The Clang 3.0 will generate the wrong AVX binary code. +* The number of CPUs/Cores should less than or equal to 256. On Linux x86_64(amd64), there is experimental support for up to 1024 CPUs/Cores and 128 numa nodes if you build the library with BIGNUMA=1. +* OpenBLAS does not set processor affinity by default. On Linux, you can enable processor affinity by commenting the line NO_AFFINITY=1 in Makefile.rule. But this may cause [the conflict with R parallel](https://stat.ethz.ch/pipermail/r-sig-hpc/2012-April/001348.html). +* On Loongson 3A. make test would be failed because of pthread_create error. The error code is EAGAIN. However, it will be OK when you run the same testcase on shell. + +## BLAS reference manual + +If you want to understand every BLAS function and definition, please read [Intel MKL reference manual](https://software.intel.com/en-us/intel-mkl/documentation) or [netlib.org](http://netlib.org/blas/) + +Here are [OpenBLAS extension functions](OpenBLAS-Extensions) \ No newline at end of file diff --git a/WIP---Build-system-overview-(community-made).md b/WIP---Build-system-overview-(community-made).md new file mode 100644 index 0000000000..06242082d1 --- /dev/null +++ b/WIP---Build-system-overview-(community-made).md @@ -0,0 +1,103 @@ +**This page is made by someone who is not the developer and should not be considered as an official documentation of the build system. For getting the full picture, it is best to read the Makefiles and understand them yourself.** + +### Makefile dep graph + +``` +Makefile +| +|----- Makefile.system # !!! this is included by many of the Makefiles in the subdirectories !!! +| | +| |===== Makefile.prebuild # This is triggered (not included) once by Makefile.system +| | | # and runs before any of the actual library code is built. +| | | # (builds and runs the "getarch" tool for cpu identification, +| | | # runs the compiler detection scripts c_check and f_check) +| | | +| | ----- (Makefile.conf) [ either this or Makefile_kernel.conf is generated ] +| | | { Makefile.system#L243 } +| | ----- (Makefile_kernel.conf) [ temporary Makefile.conf during DYNAMIC_ARCH builds ] +| | +| |----- Makefile.rule # defaults for build options that can be given on the make command line +| | +| |----- Makefile.$(ARCH) # architecture-specific compiler options and OpenBLAS buffer size values +| +|~~~~~ exports/ +| +|~~~~~ test/ +| +|~~~~~ utest/ +| +|~~~~~ ctest/ +| +|~~~~~ cpp_thread_test/ +| +|~~~~~ kernel/ +| +|~~~~~ ${SUBDIRS} +| +|~~~~~ ${BLASDIRS} +| +|~~~~~ ${NETLIB_LAPACK_DIR}{,/timing,/testing/{EIG,LIN}} +| +|~~~~~ relapack/ +``` + +### Important Variables + +Most of the tunable variables are found in [Makefile.rule](https://github.com/xianyi/OpenBLAS/blob/develop/Makefile.rule), along with their detailed descriptions.
+Most of the variables are detected automatically in [Makefile.prebuild](https://github.com/xianyi/OpenBLAS/blob/develop/Makefile.prebuild), if they are not set in the environment. + +#### CPU related +``` +ARCH - Target architecture (eg. x86_64) +TARGET - Target CPU architecture, in case of DYNAMIC_ARCH=1 means library will not be usable on less capable CPUs +TARGET_CORE - TARGET_CORE will override TARGET internally during each cpu-specific cycle of the build for DYNAMIC_ARCH +DYNAMIC_ARCH - For building library for multiple TARGETs (does not lose any optimizations, but increases library size) +DYNAMIC_LIST - optional user-provided subset of the DYNAMIC_CORE list in Makefile.system +``` + +#### Toolchain related +``` +CC - TARGET C compiler used for compilation (can be cross-toolchains) +FC - TARGET Fortran compiler used for compilation (can be cross-toolchains, set NOFORTRAN=1 if used cross-toolchain has no fortran compiler) +AR, AS, LD, RANLIB - TARGET toolchain helpers used for compilation (can be cross-toolchains) + +HOSTCC - compiler of build machine, needed to create proper config files for target architecture +HOST_CFLAGS - flags for build machine compiler +``` + +#### Library related +``` +BINARY - 32/64 bit library + +BUILD_SHARED - Create shared library +BUILD_STATIC - Create static library + +QUAD_PRECISION - enable support for IEEE quad precision [ largely unimplemented leftover from GotoBLAS, do not use ] +EXPRECISION - Obsolete option to use float80 of SSE on BSD-like systems +INTERFACE64 - Build with 64bit integer representations to support large array index values [ incompatible with standard API ] + +BUILD_SINGLE - build the single-precision real functions of BLAS [and optionally LAPACK] +BUILD_DOUBLE - build the double-precision real functions +BUILD_COMPLEX - build the single-precision complex functions +BUILD_COMPLEX16 - build the double-precision complex functions +(all four types are included in the build by default when none was specifically selected) + +BUILD_BFLOAT16 - build the "half precision brainfloat" real functions + +USE_THREAD - Use a multithreading backend (default to pthread) +USE_LOCKING - implement locking for thread safety even when USE_THREAD is not set (so that the singlethreaded library can + safely be called from multithreaded programs) +USE_OPENMP - Use OpenMP as multithreading backend +NUM_THREADS - define this to the maximum number of parallel threads you expect to need (defaults to the number of cores in the build cpu) +NUM_PARALLEL - define this to the number of OpenMP instances that your code may use for parallel calls into OpenBLAS (default 1,see below) + +``` + + +OpenBLAS uses a fixed set of memory buffers internally, used for communicating and compiling partial results from individual threads. +For efficiency, the management array structure for these buffers is sized at build time - this makes it necessary to know in advance how +many threads need to be supported on the target system(s). +With OpenMP, there is an additional level of complexity as there may be calls originating from a parallel region in the calling program. If OpenBLAS gets called from a single parallel region, it runs single-threaded automatically to avoid overloading the system by fanning out its own set of threads. +In the case that an OpenMP program makes multiple calls from independent regions or instances in parallel, this default serialization is not +sufficient as the additional caller(s) would compete for the original set of buffers already in use by the first call. +So if multiple OpenMP runtimes call into OpenBLAS at the same time, then only one of them will be able to make progress while all the rest of them spin-wait for the one available buffer. Setting NUM_PARALLEL to the upper bound on the number of OpenMP runtimes that you can have in a process ensures that there are a sufficient number of buffer sets available \ No newline at end of file From 2aee9a8a7da86c934a0a7641d6523a0983d0ec3c Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Fri, 4 Aug 2023 08:34:20 +0100 Subject: [PATCH 003/267] Move wiki contents to `docs/` --- .md | 3 --- Document.md | 27 ------------------- .../build_android.md | 16 +++++------ .../build_ios.md | 0 .../build_m1.md | 0 .../build_system.md | 0 .../build_win_arm.md | 0 CI-jobs-overview.md => docs/ci.md | 0 Developer-manual.md => docs/developers.md | 0 Donation.md => docs/donate.md | 0 Download.md => docs/download.md | 0 .../ecosystem.md | 0 OpenBLAS-Extensions.md => docs/extensions.md | 0 Faq.md => docs/faq.md | 0 .../import_mingw.md | 0 Home.md => docs/index.md | 0 Installation-Guide.md => docs/install.md | 0 Machine-List.md => docs/machine_list.md | 0 Mailing-List.md => docs/mailing_list.md | 0 .../packages.md | 0 Publications.md => docs/publications.md | 0 .../todo.md | 0 .../use_cortex.md | 0 .../use_visual_studio.md | 0 User-Manual.md => docs/user_manual.md | 0 25 files changed, 7 insertions(+), 39 deletions(-) delete mode 100644 .md delete mode 100644 Document.md rename How-to-build-OpenBLAS-for-Android.md => docs/build_android.md (96%) rename How-to-build-OpenBLAS-for-iPhone-iOS-(ARMv8).md => docs/build_ios.md (100%) rename How-to-build-OpenBLAS-for-macOS-M1---arm64.md => docs/build_m1.md (100%) rename WIP---Build-system-overview-(community-made).md => docs/build_system.md (100%) rename How-to-build-OpenBLAS-for-Windows-on-ARM64.md => docs/build_win_arm.md (100%) rename CI-jobs-overview.md => docs/ci.md (100%) rename Developer-manual.md => docs/developers.md (100%) rename Donation.md => docs/donate.md (100%) rename Download.md => docs/download.md (100%) rename Related-packages-that-use-OpenBLAS.md => docs/ecosystem.md (100%) rename OpenBLAS-Extensions.md => docs/extensions.md (100%) rename Faq.md => docs/faq.md (100%) rename How-to-generate-import-library-for-MinGW.md => docs/import_mingw.md (100%) rename Home.md => docs/index.md (100%) rename Installation-Guide.md => docs/install.md (100%) rename Machine-List.md => docs/machine_list.md (100%) rename Mailing-List.md => docs/mailing_list.md (100%) rename Precompiled-installation-packages.md => docs/packages.md (100%) rename Publications.md => docs/publications.md (100%) rename Fixed-optimized-kernels-To-do-List.md => docs/todo.md (100%) rename How-to-use-OpenBLAS-on-Cortex-M.md => docs/use_cortex.md (100%) rename How-to-use-OpenBLAS-in-Microsoft-Visual-Studio.md => docs/use_visual_studio.md (100%) rename User-Manual.md => docs/user_manual.md (100%) diff --git a/.md b/.md deleted file mode 100644 index e33f6435d6..0000000000 --- a/.md +++ /dev/null @@ -1,3 +0,0 @@ -# Installation Guide - -test \ No newline at end of file diff --git a/Document.md b/Document.md deleted file mode 100644 index 68154b2d35..0000000000 --- a/Document.md +++ /dev/null @@ -1,27 +0,0 @@ -
-
- [Home] - [Document] - [FAQ] - [Publications] - [Download] -[Mailing List] -[Donation] -
-
- -[Installation Guide](Installation-Guide) - -[User Manual](User-Manual) - -[Developer Manual](Developer-manual) - -[Use OpenBLAS in MS Visual Studio](How-to-use-OpenBLAS-in-Microsoft-Visual-Studio) - -[Generate import library for MingW](How-to-generate-import-library-for-MingW) - -[OpenBLAS Extensions](OpenBLAS-Extensions) - -[Related packages that use OpenBLAS](Related-packages-that-use-OpenBLAS) - -[Machine List](Machine-List) \ No newline at end of file diff --git a/How-to-build-OpenBLAS-for-Android.md b/docs/build_android.md similarity index 96% rename from How-to-build-OpenBLAS-for-Android.md rename to docs/build_android.md index 2c2229e37f..b6bc377359 100644 --- a/How-to-build-OpenBLAS-for-Android.md +++ b/docs/build_android.md @@ -1,16 +1,14 @@ -## Prerequisites - In addition to the Android NDK, you will need a C compiler on the build host as this is currently required by the OpenBLAS build environment. -# Building with android NDK using clang compiler +### Building with android NDK using clang compiler Around version 11 Android NDKs stopped supporting gcc, so you would need to use clang to compile OpenBLAS. clang is supported from OpenBLAS 0.2.20 version onwards. See below sections on how to build with clang for ARMV7 and ARMV8 targets. The same basic principles as described below for ARMV8 should also apply to building an x86 or x86_64 version (substitute something like NEHALEM for the target instead of ARMV8 and replace all the aarch64 in the toolchain paths obviously) "Historic" notes: Since version 19 the default toolchain is provided as a standalone toolchain, so building one yourself following [building a standalone toolchain](http://developer.android.com/ndk/guides/standalone_toolchain.html) should no longer be necessary. If you want to use static linking with an old NDK version older than about r17, you need to choose an API level below 23 currently due to NDK bug 272 (https://github.com/android-ndk/ndk/issues/272 , the libc.a lacks a definition of stderr) that will probably be fixed in r17 of the NDK. -## Build ARMV7 with clang +#### Build ARMV7 with clang ``` # Set path to ndk-bundle export NDK_BUNDLE_DIR=/path/to/ndk-bundle @@ -33,7 +31,7 @@ AR=${NDK_BUNDLE_DIR}/toolchains/arm-linux-androideabi-4.9/prebuilt/darwin-x86_64 ``` otherwise you may get a linker error complaining about a "malformed archive header name at 8" when the native OSX ar command was invoked instead. Note that with recent NDK versions, the AR tool may be named `llvm-ar` rather than what is assumed above. -## Build ARMV8 with clang +#### Build ARMV8 with clang ``` # Set path to ndk-bundle export NDK_BUNDLE_DIR=/path/to/ndk-bundle/ @@ -59,7 +57,7 @@ make HOSTCC=gcc CC=/opt/android-ndk-r23b/toolchains/llvm/prebuilt/linux-x86_64/b ``` appears to be sufficient on Linux. On OSX, setting AR to the ar provided in the "bin" path of the NDK (probably llvm-ar) is also necessary. -## Alternative script which was tested on OSX with NDK(21.3.6528147) +#### Alternative script which was tested on OSX with NDK(21.3.6528147) This script will build openblas for 3 architecture (ARMV7,ARMV8,X86) and put them with `sudo make install` to `/opt/OpenBLAS/lib`. Of course you can also copy only the section that is of interest to you - also notice that the AR= line may need adapting to the name of the ar tool provided in your $TOOLCHAIN/bin - for example llvm-ar in some recent NDK versions. @@ -116,7 +114,7 @@ Also you can find full list of target architectures in [TargetsList.txt](https:/ *** anything below this line should be irrelevant nowadays unless you need to perform software archeology *** -## Building OpenBLAS with very old gcc-based versions of the NDK, without Fortran +#### Building OpenBLAS with very old gcc-based versions of the NDK, without Fortran The prebuilt Android NDK toolchains do not include Fortran, hence parts like LAPACK cannot be built. You can still build OpenBLAS without it. For instructions on how to build OpenBLAS with Fortran, see the [next section](#building-openblas-with-fortran). @@ -155,7 +153,7 @@ here that you specified for building, otherwise errors may occur as it tries to make PREFIX=/path/to/install-dir TARGET=... install ``` -## Building OpenBLAS with Fortran +#### Building OpenBLAS with Fortran Instructions on how to build the GNU toolchains with Fortran can be found [here](https://github.com/buffer51/android-gfortran). The [Releases section](https://github.com/buffer51/android-gfortran/releases) provides prebuilt versions, use the standalone one. @@ -172,7 +170,7 @@ make TARGET=ARMV8 BINARY=64 HOSTCC=gcc CC=aarch64-linux-android-gcc FC=aarch64-l As mentioned above you can leave off the `libs` argument here when building 0.2.20 and later, and you may want to add ARM_SOFTFP_ABI=1 when building for ARMV7. -## Linking OpenBLAS (0.2.19 and earlier) for ARMV7 +#### Linking OpenBLAS (0.2.19 and earlier) for ARMV7 If you are using `ndk-build`, you need to set the ABI to hard floating points in your Application.mk: ``` diff --git a/How-to-build-OpenBLAS-for-iPhone-iOS-(ARMv8).md b/docs/build_ios.md similarity index 100% rename from How-to-build-OpenBLAS-for-iPhone-iOS-(ARMv8).md rename to docs/build_ios.md diff --git a/How-to-build-OpenBLAS-for-macOS-M1---arm64.md b/docs/build_m1.md similarity index 100% rename from How-to-build-OpenBLAS-for-macOS-M1---arm64.md rename to docs/build_m1.md diff --git a/WIP---Build-system-overview-(community-made).md b/docs/build_system.md similarity index 100% rename from WIP---Build-system-overview-(community-made).md rename to docs/build_system.md diff --git a/How-to-build-OpenBLAS-for-Windows-on-ARM64.md b/docs/build_win_arm.md similarity index 100% rename from How-to-build-OpenBLAS-for-Windows-on-ARM64.md rename to docs/build_win_arm.md diff --git a/CI-jobs-overview.md b/docs/ci.md similarity index 100% rename from CI-jobs-overview.md rename to docs/ci.md diff --git a/Developer-manual.md b/docs/developers.md similarity index 100% rename from Developer-manual.md rename to docs/developers.md diff --git a/Donation.md b/docs/donate.md similarity index 100% rename from Donation.md rename to docs/donate.md diff --git a/Download.md b/docs/download.md similarity index 100% rename from Download.md rename to docs/download.md diff --git a/Related-packages-that-use-OpenBLAS.md b/docs/ecosystem.md similarity index 100% rename from Related-packages-that-use-OpenBLAS.md rename to docs/ecosystem.md diff --git a/OpenBLAS-Extensions.md b/docs/extensions.md similarity index 100% rename from OpenBLAS-Extensions.md rename to docs/extensions.md diff --git a/Faq.md b/docs/faq.md similarity index 100% rename from Faq.md rename to docs/faq.md diff --git a/How-to-generate-import-library-for-MinGW.md b/docs/import_mingw.md similarity index 100% rename from How-to-generate-import-library-for-MinGW.md rename to docs/import_mingw.md diff --git a/Home.md b/docs/index.md similarity index 100% rename from Home.md rename to docs/index.md diff --git a/Installation-Guide.md b/docs/install.md similarity index 100% rename from Installation-Guide.md rename to docs/install.md diff --git a/Machine-List.md b/docs/machine_list.md similarity index 100% rename from Machine-List.md rename to docs/machine_list.md diff --git a/Mailing-List.md b/docs/mailing_list.md similarity index 100% rename from Mailing-List.md rename to docs/mailing_list.md diff --git a/Precompiled-installation-packages.md b/docs/packages.md similarity index 100% rename from Precompiled-installation-packages.md rename to docs/packages.md diff --git a/Publications.md b/docs/publications.md similarity index 100% rename from Publications.md rename to docs/publications.md diff --git a/Fixed-optimized-kernels-To-do-List.md b/docs/todo.md similarity index 100% rename from Fixed-optimized-kernels-To-do-List.md rename to docs/todo.md diff --git a/How-to-use-OpenBLAS-on-Cortex-M.md b/docs/use_cortex.md similarity index 100% rename from How-to-use-OpenBLAS-on-Cortex-M.md rename to docs/use_cortex.md diff --git a/How-to-use-OpenBLAS-in-Microsoft-Visual-Studio.md b/docs/use_visual_studio.md similarity index 100% rename from How-to-use-OpenBLAS-in-Microsoft-Visual-Studio.md rename to docs/use_visual_studio.md diff --git a/User-Manual.md b/docs/user_manual.md similarity index 100% rename from User-Manual.md rename to docs/user_manual.md From 27b5192ade74ce1e086b1ad4a7bbbd5b7845947d Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Fri, 4 Aug 2023 08:35:26 +0100 Subject: [PATCH 004/267] Bare `mkdocs` support with material theme --- mkdocs.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 mkdocs.yml diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000000..fa486d208f --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,3 @@ +site_name: OpenBLAS +theme: + name: material From 6c8c060c7598ef82ebace762ef5819e46039bcea Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Fri, 4 Aug 2023 10:51:50 +0100 Subject: [PATCH 005/267] Build docs workflow --- .github/workflows/docs.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000000..1091824e04 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,16 @@ +name: Publish docs via GitHub Pages +on: + push: + branches: + - develop +jobs: + build: + name: Deploy docs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: "3.10" + - run: pip install mkdocs mkdocs-material + - run: mkdocs gh-deploy --force --clean --verbose From 9cfcee131e7dacfe951278ff0062fd7978460501 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Fri, 4 Aug 2023 18:48:33 +0100 Subject: [PATCH 006/267] Doc improvements squish * Fix more broken links * Move FAQ last * Fold all install-y content into `install.md` * Fix header levels * Fold `download.md` contents into `install.md` * Fix cross-references * Ordered navigation * Add logo to docs * Make doc pages more consistent --- docs/build_android.md | 188 ----------- docs/build_ios.md | 14 - docs/build_m1.md | 1 - docs/build_system.md | 13 +- docs/build_win_arm.md | 77 ----- docs/ci.md | 2 + docs/developers.md | 2 + docs/distributing.md | 13 +- docs/donate.md | 16 +- docs/download.md | 23 -- docs/ecosystem.md | 2 + docs/faq.md | 143 +++------ docs/import_mingw.md | 36 --- docs/index.md | 25 +- docs/install.md | 658 +++++++++++++++++++++++++++++++++++++- docs/logo.svg | 450 ++++++++++++++++++++++++++ docs/mailing_list.md | 14 +- docs/packages.md | 84 ----- docs/publications.md | 14 +- docs/todo.md | 24 +- docs/use_cortex.md | 37 --- docs/use_visual_studio.md | 162 ---------- docs/user_manual.md | 14 +- mkdocs.yml | 26 ++ 24 files changed, 1213 insertions(+), 825 deletions(-) delete mode 100644 docs/build_android.md delete mode 100644 docs/build_ios.md delete mode 100644 docs/build_m1.md delete mode 100644 docs/build_win_arm.md delete mode 100644 docs/download.md delete mode 100644 docs/import_mingw.md create mode 100644 docs/logo.svg delete mode 100644 docs/packages.md delete mode 100644 docs/use_cortex.md delete mode 100644 docs/use_visual_studio.md diff --git a/docs/build_android.md b/docs/build_android.md deleted file mode 100644 index b6bc377359..0000000000 --- a/docs/build_android.md +++ /dev/null @@ -1,188 +0,0 @@ -In addition to the Android NDK, you will need a C compiler on the build host as this is currently -required by the OpenBLAS build environment. - - -### Building with android NDK using clang compiler -Around version 11 Android NDKs stopped supporting gcc, so you would need to use clang to compile OpenBLAS. clang is supported from OpenBLAS 0.2.20 version onwards. See below sections on how to build with clang for ARMV7 and ARMV8 targets. The same basic principles as described below for ARMV8 should also apply to building an x86 or x86_64 version (substitute something like NEHALEM for the target instead of ARMV8 and replace all the aarch64 in the toolchain paths obviously) -"Historic" notes: -Since version 19 the default toolchain is provided as a standalone toolchain, so building one yourself following [building a standalone toolchain](http://developer.android.com/ndk/guides/standalone_toolchain.html) should no longer be necessary. -If you want to use static linking with an old NDK version older than about r17, you need to choose an API level below 23 currently due to NDK bug 272 (https://github.com/android-ndk/ndk/issues/272 , the libc.a lacks a definition of stderr) that will probably be fixed in r17 of the NDK. - -#### Build ARMV7 with clang -``` -# Set path to ndk-bundle -export NDK_BUNDLE_DIR=/path/to/ndk-bundle - -# Set the PATH to contain paths to clang and arm-linux-androideabi-* utilities -export PATH=${NDK_BUNDLE_DIR}/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/bin:${NDK_BUNDLE_DIR}/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH - -# Set LDFLAGS so that the linker finds the appropriate libgcc -export LDFLAGS="-L${NDK_BUNDLE_DIR}/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/lib/gcc/arm-linux-androideabi/4.9.x" - -# Set the clang cross compile flags -export CLANG_FLAGS="-target arm-linux-androideabi -marm -mfpu=vfp -mfloat-abi=softfp --sysroot ${NDK_BUNDLE_DIR}/platforms/android-23/arch-arm -gcc-toolchain ${NDK_BUNDLE_DIR}/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/" - -#OpenBLAS Compile -make TARGET=ARMV7 ONLY_CBLAS=1 AR=ar CC="clang ${CLANG_FLAGS}" HOSTCC=gcc ARM_SOFTFP_ABI=1 -j4 -``` -On a Mac, it may also be necessary to give the complete path to the `ar` utility in the make command above, like so: -``` -AR=${NDK_BUNDLE_DIR}/toolchains/arm-linux-androideabi-4.9/prebuilt/darwin-x86_64/bin/arm-linux-androideabi-gcc-ar -``` -otherwise you may get a linker error complaining about a "malformed archive header name at 8" when the native OSX ar command was invoked instead. Note that with recent NDK versions, the AR tool may be named `llvm-ar` rather than what is assumed above. - -#### Build ARMV8 with clang -``` -# Set path to ndk-bundle -export NDK_BUNDLE_DIR=/path/to/ndk-bundle/ - -# Export PATH to contain directories of clang and aarch64-linux-android-* utilities -export PATH=${NDK_BUNDLE_DIR}/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/bin/:${NDK_BUNDLE_DIR}/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH - -# Setup LDFLAGS so that loader can find libgcc and pass -lm for sqrt -export LDFLAGS="-L${NDK_BUNDLE_DIR}/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/lib/gcc/aarch64-linux-android/4.9.x -lm" - -# Setup the clang cross compile options -export CLANG_FLAGS="-target aarch64-linux-android --sysroot ${NDK_BUNDLE_DIR}/platforms/android-23/arch-arm64 -gcc-toolchain ${NDK_BUNDLE_DIR}/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/" - -# Compile -make TARGET=ARMV8 ONLY_CBLAS=1 AR=ar CC="clang ${CLANG_FLAGS}" HOSTCC=gcc -j4 -``` -Note: Using TARGET=CORTEXA57 in place of ARMV8 will pick up better optimized routines. Implementations for CORTEXA57 target is compatible with all other armv8 targets. - -Note: For NDK 23b, something as simple as -``` -export PATH=/opt/android-ndk-r23b/toolchains/llvm/prebuilt/linux-x86_64/bin/:$PATH -make HOSTCC=gcc CC=/opt/android-ndk-r23b/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android31-clang ONLY_CBLAS=1 TARGET=ARMV8 -``` -appears to be sufficient on Linux. On OSX, setting AR to the ar provided in the "bin" path of the NDK (probably llvm-ar) is also necessary. - -#### Alternative script which was tested on OSX with NDK(21.3.6528147) -This script will build openblas for 3 architecture (ARMV7,ARMV8,X86) and put them with `sudo make install` to `/opt/OpenBLAS/lib`. -Of course you can also copy only the section that is of interest to you - also notice that the AR= line may need adapting to the -name of the ar tool provided in your $TOOLCHAIN/bin - for example llvm-ar in some recent NDK versions. -``` -export NDK=YOUR_PATH_TO_SDK/Android/sdk/ndk/21.3.6528147 -export TOOLCHAIN=$NDK/toolchains/llvm/prebuilt/darwin-x86_64 - -make clean -make \ - TARGET=ARMV7 \ - ONLY_CBLAS=1 \ - CC="$TOOLCHAIN"/bin/armv7a-linux-androideabi21-clang \ - AR="$TOOLCHAIN"/bin/arm-linux-androideabi-ar \ - HOSTCC=gcc \ - ARM_SOFTFP_ABI=1 \ - -j4 -sudo make install - -make clean -make \ - TARGET=CORTEXA57 \ - ONLY_CBLAS=1 \ - CC=$TOOLCHAIN/bin/aarch64-linux-android21-clang \ - AR=$TOOLCHAIN/bin/aarch64-linux-android-ar \ - HOSTCC=gcc \ - -j4 -sudo make install - -make clean -make \ - TARGET=ATOM \ - ONLY_CBLAS=1 \ - CC="$TOOLCHAIN"/bin/i686-linux-android21-clang \ - AR="$TOOLCHAIN"/bin/i686-linux-android-ar \ - HOSTCC=gcc \ - ARM_SOFTFP_ABI=1 \ - -j4 -sudo make install - -# This will build for x86_64 -make clean -make \ - TARGET=ATOM BINARY=64\ - ONLY_CBLAS=1 \ - CC="$TOOLCHAIN"/bin/x86_64-linux-android21-clang \ - AR="$TOOLCHAIN"/bin/x86_64-linux-android-ar \ - HOSTCC=gcc \ - ARM_SOFTFP_ABI=1 \ - -j4 -sudo make install -``` -Also you can find full list of target architectures in [TargetsList.txt](https://github.com/xianyi/OpenBLAS/blob/develop/TargetList.txt) - -*** -anything below this line should be irrelevant nowadays unless you need to perform software archeology -*** -#### Building OpenBLAS with very old gcc-based versions of the NDK, without Fortran - -The prebuilt Android NDK toolchains do not include Fortran, hence parts like LAPACK cannot be built. You can still build OpenBLAS without it. For instructions on how to build OpenBLAS with Fortran, see the [next section](#building-openblas-with-fortran). - -To use easily the prebuilt toolchains, follow [building a standalone toolchain](http://developer.android.com/ndk/guides/standalone_toolchain.html) for your desired architecture. -This would be `arm-linux-androideabi-gcc-4.9` for ARMV7 and `aarch64-linux-android-gcc-4.9` for ARMV8. - -You can build OpenBLAS (0.2.19 and earlier) with: -``` -# Add the toolchain to your path -export PATH=/path/to/standalone-toolchain/bin:$PATH - -# Build without Fortran for ARMV7 -make TARGET=ARMV7 HOSTCC=gcc CC=arm-linux-androideabi-gcc NOFORTRAN=1 libs -# Build without Fortran for ARMV8 -make TARGET=ARMV8 BINARY=64 HOSTCC=gcc CC=aarch64-linux-android-gcc NOFORTRAN=1 libs -``` - -Since we are cross-compiling, we make the `libs` recipe, not `all`. Otherwise you will get errors when trying to link/run tests as versions up to and including 0.2.19 cannot build a shared library for Android. - -From 0.2.20 on, you should leave off the "libs" to get a full build, and you may want to use the softfp ABI instead of the deprecated hardfp one on ARMV7 so you would use -``` -# Add the toolchain to your path -export PATH=/path/to/standalone-toolchain/bin:$PATH - -# Build without Fortran for ARMV7 -make TARGET=ARMV7 ARM_SOFTFP_ABI=1 HOSTCC=gcc CC=arm-linux-androideabi-gcc NOFORTRAN=1 -# Build without Fortran for ARMV8 -make TARGET=ARMV8 BINARY=64 HOSTCC=gcc CC=aarch64-linux-android-gcc NOFORTRAN=1 -``` - -If you get an error about stdio.h not being found, you need to specify your sysroot in the CFLAGS argument to `make` like -```CFLAGS=--sysroot=$NDK/platforms/android-16/arch-arm``` -When you are done, install OpenBLAS into the desired directory. Be sure to also use all command line options -here that you specified for building, otherwise errors may occur as it tries to install things you did not build: -``` -make PREFIX=/path/to/install-dir TARGET=... install -``` - -#### Building OpenBLAS with Fortran - -Instructions on how to build the GNU toolchains with Fortran can be found [here](https://github.com/buffer51/android-gfortran). The [Releases section](https://github.com/buffer51/android-gfortran/releases) provides prebuilt versions, use the standalone one. - -You can build OpenBLAS with: -``` -# Add the toolchain to your path -export PATH=/path/to/standalone-toolchain-with-fortran/bin:$PATH - -# Build with Fortran for ARMV7 -make TARGET=ARMV7 HOSTCC=gcc CC=arm-linux-androideabi-gcc FC=arm-linux-androideabi-gfortran libs -# Build with LAPACK for ARMV8 -make TARGET=ARMV8 BINARY=64 HOSTCC=gcc CC=aarch64-linux-android-gcc FC=aarch64-linux-android-gfortran libs -``` - -As mentioned above you can leave off the `libs` argument here when building 0.2.20 and later, and you may want to add ARM_SOFTFP_ABI=1 when building for ARMV7. - -#### Linking OpenBLAS (0.2.19 and earlier) for ARMV7 - -If you are using `ndk-build`, you need to set the ABI to hard floating points in your Application.mk: -``` -APP_ABI := armeabi-v7a-hard -``` - -This will set the appropriate flags for you. If you are not using `ndk-build`, you will want to add the following flags: -``` -TARGET_CFLAGS += -mhard-float -D_NDK_MATH_NO_SOFTFP=1 -TARGET_LDFLAGS += -Wl,--no-warn-mismatch -lm_hard -``` - -From 0.2.20 on, it is also possible to build for the softfp ABI by specifying ARM_SOFTFP_ABI=1 during the build. -In that case, also make sure that all your dependencies are compiled with -mfloat-abi=softfp as well, as mixing -"hard" and "soft" floating point ABIs in a program will make it crash. diff --git a/docs/build_ios.md b/docs/build_ios.md deleted file mode 100644 index f0b054e13b..0000000000 --- a/docs/build_ios.md +++ /dev/null @@ -1,14 +0,0 @@ -# How to build OpenBLAS for iPhone/iOS - -As none of the current developers uses iOS, the following instructions are what was found to work in our Azure CI setup, but as far as we know this builds a fully working OpenBLAS for this platform. - -Go to the directory where you unpacked OpenBLAS,and enter the following commands: -``` - CC=/Applications/Xcode_12.4.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang - -CFLAGS= -O2 -Wno-macro-redefined -isysroot /Applications/Xcode_12.4.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS14.4.sdk -arch arm64 -miphoneos-version-min=10.0 - -make TARGET=ARMV8 DYNAMIC_ARCH=1 NUM_THREADS=32 HOSTCC=clang NOFORTRAN=1 -``` -Adjust MIN_IOS_VERSION as necessary for your installation, e.g. change the version number -to the minimum iOS version you want to target and execute this file to build the library. \ No newline at end of file diff --git a/docs/build_m1.md b/docs/build_m1.md deleted file mode 100644 index 77e3b9e83b..0000000000 --- a/docs/build_m1.md +++ /dev/null @@ -1 +0,0 @@ -On newer versions of Xcode and on arm64, you might need to compile with a newer macOS target (11.0) than the default (10.8) with `MACOSX_DEPLOYMENT_TARGET=11.0`, or switch your command-line tools to use an older SDK (e.g., [13.1](https://developer.apple.com/download/all/?q=Xcode%2013)). \ No newline at end of file diff --git a/docs/build_system.md b/docs/build_system.md index 06242082d1..98808fdfab 100644 --- a/docs/build_system.md +++ b/docs/build_system.md @@ -1,6 +1,7 @@ -**This page is made by someone who is not the developer and should not be considered as an official documentation of the build system. For getting the full picture, it is best to read the Makefiles and understand them yourself.** +!!! warning + This page is made by someone who is not the developer and should not be considered as an official documentation of the build system. For getting the full picture, it is best to read the Makefiles and understand them yourself. -### Makefile dep graph +## Makefile dep graph ``` Makefile @@ -41,12 +42,12 @@ Makefile |~~~~~ relapack/ ``` -### Important Variables +## Important Variables Most of the tunable variables are found in [Makefile.rule](https://github.com/xianyi/OpenBLAS/blob/develop/Makefile.rule), along with their detailed descriptions.
Most of the variables are detected automatically in [Makefile.prebuild](https://github.com/xianyi/OpenBLAS/blob/develop/Makefile.prebuild), if they are not set in the environment. -#### CPU related +### CPU related ``` ARCH - Target architecture (eg. x86_64) TARGET - Target CPU architecture, in case of DYNAMIC_ARCH=1 means library will not be usable on less capable CPUs @@ -55,7 +56,7 @@ DYNAMIC_ARCH - For building library for multiple TARGETs (does not lose any opti DYNAMIC_LIST - optional user-provided subset of the DYNAMIC_CORE list in Makefile.system ``` -#### Toolchain related +### Toolchain related ``` CC - TARGET C compiler used for compilation (can be cross-toolchains) FC - TARGET Fortran compiler used for compilation (can be cross-toolchains, set NOFORTRAN=1 if used cross-toolchain has no fortran compiler) @@ -65,7 +66,7 @@ HOSTCC - compiler of build machine, needed to create proper config f HOST_CFLAGS - flags for build machine compiler ``` -#### Library related +### Library related ``` BINARY - 32/64 bit library diff --git a/docs/build_win_arm.md b/docs/build_win_arm.md deleted file mode 100644 index 01c2a84dfb..0000000000 --- a/docs/build_win_arm.md +++ /dev/null @@ -1,77 +0,0 @@ -This page describes how to natively build OpenBLAS library for windows on arm64 targets. - -See [below](#xcomp) for how to cross-compile OpenBLAS for WoA on an x86_64 Windows host. - -_We hope that the procedure can be simplified for/by LLVM 17+ and its flang-new compiler for the FORTRAN parts (LAPACK) of the code but we have no means of verifying that as of January 2024._ - - -# Prerequisite - -Following tools needs to be installed - -## 1. Download and install clang for windows on arm - -Find the latest LLVM build for WoA from [LLVM release page](https://releases.llvm.org/) - -E.g: LLVM 12 build for WoA64 can be found [here](https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.0/LLVM-12.0.0-woa64.exe) - -Run the LLVM installer and ensure that LLVM is added to environment PATH. - -## 2. Download and install classic flang for windows on arm - -Classic flang is the only available FORTRAN compiler for windows on arm for now and a pre-release build can be found [here](https://github.com/kaadam/flang/releases/tag/v0.1) - -There is no installer for classic flang and the zip package can be extracted and the path needs to be added to environment PATH. - -E.g: on PowerShell - -`$env:Path += ";C:\flang_woa\bin"` - -# Build - -The following steps describe how to build the static library for OpenBLAS with and without LAPACK - -## 1. Build OpenBLAS static library with BLAS and LAPACK routines with Make - -Following command can be used to build OpenBLAS static library with BLAS and LAPACK routines - -`make CC="clang-cl" HOSTCC="clang-cl" AR="llvm-ar" BUILD_WITHOUT_LAPACK=0 NOFORTRAN=0 DYNAMIC_ARCH=0 TARGET=ARMV8 ARCH=arm64 BINARY=64 USE_OPENMP=0 PARALLEL=1 RANLIB="llvm-ranlib" MAKE=make F_COMPILER=FLANG FC=FLANG FFLAGS_NOOPT="-march=armv8-a -cpp" FFLAGS="-march=armv8-a -cpp" NEED_PIC=0 HOSTARCH=arm64 libs netlib` - -## 2. Build static library with BLAS routines using CMake - -Classic flang has compatibility issues with cmake hence only BLAS routines can be compiled with CMake - -`mkdir build` - -`cd build` - -`cmake .. -G Ninja -DCMAKE_C_COMPILER=clang -DBUILD_WITHOUT_LAPACK=1 -DNOFORTRAN=1 -DDYNAMIC_ARCH=0 -DTARGET=ARMV8 -DARCH=arm64 -DBINARY=64 -DUSE_OPENMP=0 -DCMAKE_SYSTEM_PROCESSOR=ARM64 -DCMAKE_CROSSCOMPILING=1 -DCMAKE_SYSTEM_NAME=Windows` - -`cmake --build . --config Release` - -# Known issue - -## `getarch.exe` execution error - -If you notice that platform-specific headers by `getarch.exe` are not generated correctly, It could be due to a known debug runtime DLL issue for arm64 platforms. Please check out [link](https://linaro.atlassian.net/wiki/spaces/WOAR/pages/28677636097/Debug+run-time+DLL+issue#Workaround) for the workaround. - -# Alternative: cross-compiling on Windows x86 -## Prerequisites: -a working installation of LLVM and Ninja (the versions included in the latest Visual Studio 2022 will do, if you install its optional Llvm.Clang component) - -## Build - 1. Load into the appropriate cross compiling developer shell (in this case powershell, for cmd.exe it would be vcvarsamd64_arm64.bat): - 'C:\Program Files\Microsoft Visual Studio\2022\Professional\Common7\Tools\Launch-VsDevShell.ps1' -Arch arm64 -HostArch amd64 - - 2. Change to the folder where you unpacked the OpenBLAS sources if you haven't already, and create a subfolder for building. - Then change into that folder. - - 3. Invoke the following CMake command - - ``` - cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DTARGET=ARMV8 -DCMAKE_CROSSCOMPILING=ON -DCMAKE_SYSTEM_NAME="Windows" -DARCH=arm -DBINARY=64 -DCMAKE_SYSTEM_PROCESSOR=ARM64 -DCMAKE_C_COMPILER=clang-cl -DCMAKE_C_COMPILER_TARGET=arm64-pc-windows-msvc -DCMAKE_ASM_COMPILER_TARGET=arm64-pc-windows-msvc. - ``` - (Note the use of clang-cl instead of clang, and adding in the _COMPILER_TARGET entries. Without C_COMPILER_TARGET, clang-cl will be trying to compile x64 machine code and not make it past the cmake generation step. Without ASM_COMPILER_TARGET the assembly files in openblas will fail to assemble because the assembler is trying to interpret them as x64 assembly.) - - 4. Invoke Ninja for the actual compilation - diff --git a/docs/ci.md b/docs/ci.md index 3e2a181921..e36fb633bc 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -1,3 +1,5 @@ +# CI jobs overview + | Arch|Target CPU|OS|Build system|XComp to|C Compiler|Fortran Compiler|threading|DYN_ARCH|INT64|Libraries| CI Provider| CPU count| | ------------|---|---|-----------|-------------|----------|----------------|------|------------|----------|-----------|----------|-------| | x86_64 |Intel 32bit|Windows|CMAKE/VS2015| -|mingw6.3| - | pthreads | - | - | static | Appveyor| | diff --git a/docs/developers.md b/docs/developers.md index 0e04040252..08443b7e47 100644 --- a/docs/developers.md +++ b/docs/developers.md @@ -1,3 +1,5 @@ +# Developer manual + ## Source codes Layout ``` diff --git a/docs/distributing.md b/docs/distributing.md index 1e6372a288..2f3828d08a 100644 --- a/docs/distributing.md +++ b/docs/distributing.md @@ -1,11 +1,12 @@ # Guidance for redistributing OpenBLAS -*We note that this document contains recommendations only - packagers and other -redistributors are in charge of how OpenBLAS is built and distributed in their -systems, and may have good reasons to deviate from the guidance given on this -page. These recommendations are aimed at general packaging systems, with a user -base that typically is large, open source (or freely available at least), and -doesn't behave uniformly or that the packager is directly connected with.* +!!! note + This document contains recommendations only - packagers and other + redistributors are in charge of how OpenBLAS is built and distributed in their + systems, and may have good reasons to deviate from the guidance given on this + page. These recommendations are aimed at general packaging systems, with a user + base that typically is large, open source (or freely available at least), and + doesn't behave uniformly or that the packager is directly connected with.* OpenBLAS has a large number of build-time options which can be used to change how it behaves at runtime, how artifacts or symbols are named, etc. Variation diff --git a/docs/donate.md b/docs/donate.md index ca7efe9150..94d73bab53 100644 --- a/docs/donate.md +++ b/docs/donate.md @@ -1,14 +1,4 @@ -
-
- [Home] - [Document] - [FAQ] - [Publications] - [Download] -[Mailing List] -[Donation] -
-
+# Donations Thank you for the support. @@ -18,8 +8,6 @@ You can read OpenBLAS statement of receipts and disbursement and cash balance on * [2013.8] [Testbed for OpenBLAS project](https://www.bountysource.com/fundraisers/443-testbed-for-openblas-project) - * completed. - Here is [Backer list](https://github.com/xianyi/OpenBLAS/blob/develop/BACKERS.md). ## For Personal Donation @@ -28,4 +16,4 @@ You can create a bounty for an issue on [bountysource.com](https://www.bountysou ## For Hardware Vendors -We welcome the hardware donation, including the latest CPU and boards. \ No newline at end of file +We welcome the hardware donation, including the latest CPU and boards. diff --git a/docs/download.md b/docs/download.md deleted file mode 100644 index cb0f325261..0000000000 --- a/docs/download.md +++ /dev/null @@ -1,23 +0,0 @@ -
-
- [Home] - [Document] - [FAQ] - [Publications] - [Download] -[Mailing List] -[Donation] -
-
- -## Binary Packages - -We provide binary packages for the following platform. - -* Windows x86/x86_64 -* ARM - -You can download them from [file hosting on sourceforge.net](http://sourceforge.net/projects/openblas/files/). - -## Source -Download the latest [stable version](https://github.com/xianyi/OpenBLAS/releases) from release page. \ No newline at end of file diff --git a/docs/ecosystem.md b/docs/ecosystem.md index d667f6c405..ae2ae543b5 100644 --- a/docs/ecosystem.md +++ b/docs/ecosystem.md @@ -1,3 +1,5 @@ +# Users of OpenBLAS + ## Softwares * Julia - a high-level, high-performance dynamic programming language for technical computing
diff --git a/docs/faq.md b/docs/faq.md index 3c30de0f67..bbcbb36e4f 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -1,16 +1,6 @@ -
-
- [Home] - [Document] - [FAQ] - [Publications] - [Download] -[Mailing List] -[Donation] -
-
+# FAQ -### General questions +## General questions + **[What is BLAS? Why is it important?](#whatblas)** + **[What functions are there and how can I call them from my C code?](#whatsinblas)** + **[What is OpenBLAS? Why did you create this project?](#what)** @@ -23,7 +13,7 @@ + **[What support is there for recent PC hardware ? What about GPU ?](#recent_hardware)** + **[How about the level 3 BLAS performance on Intel Sandy Bridge?](#sandybridge_perf)** -### OS and Compiler +## OS and Compiler + **[How can I call an OpenBLAS function in Microsoft Visual Studio?](#MSVC)** + **[How can I use CBLAS and LAPACKE without C99 complex number support (e.g. in Visual Studio)?](#C99_complex_number)** + **[I get a SEGFAULT with multi-threading on Linux. What's wrong?](#Linux_SEGFAULT)** @@ -34,7 +24,6 @@ + **[How to solve undefined reference errors when statically linking against libopenblas.a](#static_link)** + **[Building OpenBLAS for Haswell or Dynamic Arch on RHEL-6, CentOS-6, Rocks-6.1](#binutils)** + **[Building OpenBLAS in QEMU/KVM/XEN](#qemu)** -+ **[Building OpenBLAS for MIPS](#mips)** + **[Building OpenBLAS on POWER fails with the IBM XL compiler](#ppcxl)** + **[Replacing system BLAS in Ubuntu/Debian](#debianlts)** + **[I built OpenBLAS for use with some other software, but that software cannot find it](#findblas)** @@ -43,7 +32,7 @@ + **[Build fails with lots of errors about undefined ?GEMM_UNROLL_M](#newcpu)** + **[CMAKE/OSX: Build fails with 'argument list too long'](#cmakeosx)** + **[Likely problems with AVX2 support in Docker Desktop for OSX](#xhyve)** -### Usage +## Usage + **[Program is Terminated. Because you tried to allocate too many memory regions](#allocmorebuffers)** + **[How to choose TARGET manually at runtime when compiled with DYNAMIC_ARCH](#choose_target_dynamic)** + **[After updating the installed OpenBLAS, a program complains about "undefined symbol gotoblas"](#missgoto)** @@ -57,13 +46,13 @@ + **[Using OpenBLAS with OpenMP](#OpenMP)**
-### General questions +## General questions -#### What is BLAS? Why is it important? +### What is BLAS? Why is it important? [BLAS](https://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms) stands for Basic Linear Algebra Subprograms. BLAS provides standard interfaces for [linear algebra](https://en.wikipedia.org/wiki/Linear_algebra), including BLAS1 (vector-vector operations), BLAS2 (matrix-vector operations), and BLAS3 (matrix-matrix operations). In general, BLAS is the computational kernel ("the bottom of the food chain") in linear algebra or scientific applications. Thus, if BLAS implementation is highly optimized, the whole application can get substantial benefit. -#### What functions are there and how can I call them from my C code? +### What functions are there and how can I call them from my C code? As BLAS is a standardized interface, you can refer to the documentation of its reference implementation at [netlib.org](http://netlib.org/blas/index.html#_blas_routines). Calls from C go through its CBLAS interface, so your code will need to include the provided cblas.h in addition to linking with -lopenblas. @@ -77,11 +66,11 @@ where M,N,K are the dimensions of your data - see https://petewarden.files.wordp (This image is part of an article on GEMM in the context of deep learning that is well worth reading in full - https://petewarden.com/2015/04/20/why-gemm-is-at-the-heart-of-deep-learning/) -#### What is OpenBLAS? Why did you create this project? +### What is OpenBLAS? Why did you create this project? OpenBLAS is an open source BLAS library forked from the GotoBLAS2-1.13 BSD version. Since Mr. Kazushige Goto left TACC, GotoBLAS is no longer being maintained. Thus, we created this project to continue developing OpenBLAS/GotoBLAS. -#### What's the difference between OpenBLAS and GotoBLAS? +### What's the difference between OpenBLAS and GotoBLAS? In OpenBLAS 0.2.0, we optimized level 3 BLAS on the Intel Sandy Bridge 64-bit OS. We obtained a performance comparable with that Intel MKL. @@ -93,24 +82,24 @@ We also added some minor features, e.g. supporting "make install", compiling wit You can find the full list of modifications in Changelog.txt. -#### Where do parameters GEMM_P, GEMM_Q, GEMM_R come from? +### Where do parameters GEMM_P, GEMM_Q, GEMM_R come from? The detailed explanation is probably in the original publication authored by Kazushige Goto - Goto, Kazushige; van de Geijn, Robert A; Anatomy of high-performance matrix multiplication. ACM Transactions on Mathematical Software (TOMS). Volume 34 Issue 3, May 2008 While this article is paywalled and too old for preprints to be available on arxiv.org, more recent publications like https://arxiv.org/pdf/1609.00076 contain at least a brief description of the algorithm. In practice, the values are derived by experimentation to yield the block sizes that give the highest performance. A general rule of thumb for selecting a starting point seems to be that PxQ is about half the size of L2 cache. -#### How can I report a bug? +### How can I report a bug? Please file an issue at this [issue page](https://github.com/xianyi/OpenBLAS/issues) or send mail to the [OpenBLAS mailing list](https://groups.google.com/forum/#!forum/openblas-users). Please provide the following information: CPU, OS, compiler, and OpenBLAS compiling flags (Makefile.rule). In addition, please describe how to reproduce this bug. -#### How to reference OpenBLAS. +### How to reference OpenBLAS. -You can reference our papers in [this page](Publications). Alternatively, you can cite the OpenBLAS homepage http://www.openblas.net. +You can reference our papers in [this page](publications.md). Alternatively, you can cite the OpenBLAS homepage http://www.openblas.net. -#### How can I use OpenBLAS in multi-threaded applications? +### How can I use OpenBLAS in multi-threaded applications? If your application is already multi-threaded, it will conflict with OpenBLAS multi-threading. Thus, you must set OpenBLAS to use single thread as following. @@ -125,12 +114,12 @@ If the application is parallelized by OpenMP, please build OpenBLAS with USE_OPE With the increased availability of fast multicore hardware it has unfortunately become clear that the thread management provided by OpenMP is not sufficient to prevent race conditions when OpenBLAS was built single-threaded by USE_THREAD=0 and there are concurrent calls from multiple threads to OpenBLAS functions. In this case, it is vital to also specify USE_LOCKING=1 (introduced with OpenBLAS 0.3.7). -#### Does OpenBLAS support sparse matrices and/or vectors ? +### Does OpenBLAS support sparse matrices and/or vectors ? OpenBLAS implements only the standard (dense) BLAS and LAPACK functions with a select few extensions popularized by Intel's MKL. Some cases can probably be made to work using e.g. GEMV or AXPBY, in general using a dedicated package like SuiteSparse (which can make use of OpenBLAS or equivalent for standard operations) is recommended. -#### What support is there for recent PC hardware ? What about GPU ? +### What support is there for recent PC hardware ? What about GPU ? As OpenBLAS is a volunteer project, it can take some time for the combination of a capable developer, free time, and particular hardware to come along, even for relatively common processors. Starting from 0.3.1, support @@ -138,7 +127,7 @@ is being added for AVX 512 (TARGET=SKYLAKEX), requiring a compiler that is capab While AMD Zen processors should be autodetected by the build system, as of 0.3.2 they are still handled exactly like Intel Haswell. There once was an effort to build an OpenCL implementation that one can still find at https://github.com/xianyi/clOpenBLAS , but work on this stopped in 2015. -#### How about the level 3 BLAS performance on Intel Sandy Bridge? +### How about the level 3 BLAS performance on Intel Sandy Bridge? We obtained a performance comparable with Intel MKL that actually outperformed Intel MKL in some cases. Here is the result of the DGEMM subroutine's performance on Intel Core i5-2500K Windows 7 SP1 64-bit: @@ -146,19 +135,19 @@ Here is the result of the DGEMM subroutine's performance on Intel Core i5-2500K
-### OS and Compiler +## OS and Compiler -#### How can I call an OpenBLAS function in Microsoft Visual Studio? +### How can I call an OpenBLAS function in Microsoft Visual Studio? -Please read [this page](How-to-use-OpenBLAS-in-Microsoft-Visual-Studio). +Please read [this page](install.md#visual-studio). -#### How can I use CBLAS and LAPACKE without C99 complex number support (e.g. in Visual Studio)? +### How can I use CBLAS and LAPACKE without C99 complex number support (e.g. in Visual Studio)? Zaheer has fixed this bug. You can now use the structure instead of C99 complex numbers. Please read [this issue page](http://github.com/xianyi/OpenBLAS/issues/95) for details. [This issue](https://github.com/xianyi/OpenBLAS/issues/305) is for using LAPACKE in Visual Studio. -#### I get a SEGFAULT with multi-threading on Linux. What's wrong? +### I get a SEGFAULT with multi-threading on Linux. What's wrong? This may be related to a bug in the Linux kernel 2.6.32 (?). Try applying the patch segaults.patch to disable mbind using @@ -166,7 +155,7 @@ This may be related to a bug in the Linux kernel 2.6.32 (?). Try applying the pa and see if the crashes persist. Note that this patch will lead to many compiler warnings. -#### When I make the library, there is no such instruction: `xgetbv' error. What's wrong? +### When I make the library, there is no such instruction: `xgetbv' error. What's wrong? Please use GCC 4.4 and later version. This version supports xgetbv instruction. If you use the library for Sandy Bridge with AVX instructions, you should use GCC 4.6 and later version. @@ -174,18 +163,18 @@ On Mac OS X, please use Clang 3.1 and later version. For example, make CC=clang For the compatibility with old compilers (GCC < 4.4), you can enable NO_AVX flag. For example, make NO_AVX=1 -#### My build fails due to the linker error "multiple definition of `dlamc3_'". What is the problem? +### My build fails due to the linker error "multiple definition of `dlamc3_'". What is the problem? This linker error occurs if GNU patch is missing or if our patch for LAPACK fails to apply. Background: OpenBLAS implements optimized versions of some LAPACK functions, so we need to disable the reference versions. If this process fails we end with duplicated implementations of the same function. -#### My build worked fine and passed all tests, but running `make lapack-test` ends with segfaults +### My build worked fine and passed all tests, but running `make lapack-test` ends with segfaults Some of the LAPACK tests, notably in xeigtstz, try to allocate around 10MB on the stack. You may need to use `ulimit -s` to change the default limits on your system to allow this. -#### How could I disable OpenBLAS threading affinity on runtime? +### How could I disable OpenBLAS threading affinity on runtime? You can define the OPENBLAS_MAIN_FREE or GOTOBLAS_MAIN_FREE environment variable to disable threading affinity on runtime. For example, before the running, ``` @@ -194,7 +183,7 @@ export OPENBLAS_MAIN_FREE=1 Alternatively, you can disable affinity feature with enabling NO_AFFINITY=1 in Makefile.rule. -#### How to solve undefined reference errors when statically linking against libopenblas.a +### How to solve undefined reference errors when statically linking against libopenblas.a On Linux, if OpenBLAS was compiled with threading support (`USE_THREAD=1` by default), custom programs statically linked against `libopenblas.a` should also link to the pthread library e.g.: @@ -231,7 +220,7 @@ blas_server.c:(.text+0x50f): undefined reference to `pthread_mutex_unlock' The `-lpthread` is not required when linking dynamically against `libopenblas.so.0`. -#### Building OpenBLAS for Haswell or Dynamic Arch on RHEL-6, CentOS-6, Rocks-6.1,Scientific Linux 6 +### Building OpenBLAS for Haswell or Dynamic Arch on RHEL-6, CentOS-6, Rocks-6.1,Scientific Linux 6 Minimum requirement to actually run AVX2-enabled software like OpenBLAS is kernel-2.6.32-358, shipped with EL6U4 in 2013 @@ -254,56 +243,20 @@ $ scl enable devtoolset-3 -- make DYNAMIC_ARCH=1 ``` AVX-512 (SKYLAKEX) support requires devtoolset-8-gcc-gfortran (which exceeds formal requirement for AVX-512 because of packaging issues in earlier packages) which dependency-installs respective binutils and gcc or later and kernel 2.6.32-696 aka 6U9 or 3.10.0-327 aka 7U2 or later to run. In absence of abovementioned toolset OpenBLAS will fall back to AVX2 instructions in place of AVX512 sacrificing some performance on SKYLAKE-X platform. -#### Building OpenBLAS in QEMU/KVM/XEN +### Building OpenBLAS in QEMU/KVM/XEN By default, QEMU reports the CPU as "QEMU Virtual CPU version 2.2.0", which shares CPUID with existing 32bit CPU even in 64bit virtual machine, and OpenBLAS recognizes it as PENTIUM2. Depending on the exact combination of CPU features the hypervisor choses to expose, this may not correspond to any CPU that exists, and OpenBLAS will error when trying to build. To fix this, pass `-cpu host` or `-cpu passthough` to QEMU, or another CPU model. Similarly, the XEN hypervisor may not pass through all features of the host cpu while reporting the cpu type itself correctly, which can lead to compiler error messages about an "ABI change" when compiling AVX512 code. Again changing the Xen configuration by running e.g. "xen-cmdline --set-xen cpuid=avx512" should get around this (as would building OpenBLAS for an older cpu lacking that particular feature, e.g. TARGET=HASWELL) -#### Building OpenBLAS for MIPS - -For mips targets you will need latest toolchains -P5600 - MTI GNU/Linux Toolchain -I6400, P6600 - IMG GNU/Linux Toolchain - -The download link is below -(http://codescape-mips-sdk.imgtec.com/components/toolchain/2016.05-03/downloads.html) - -You can use following commandlines for builds - - - IMG_TOOLCHAIN_DIR={full IMG GNU/Linux Toolchain path including "bin" directory -- for example, /opt/linux_toolchain/bin} - IMG_GCC_PREFIX=mips-img-linux-gnu - IMG_TOOLCHAIN=${IMG_TOOLCHAIN_DIR}/${IMG_GCC_PREFIX} - - I6400 Build (n32): - make BINARY=32 BINARY32=1 CC=$IMG_TOOLCHAIN-gcc AR=$IMG_TOOLCHAIN-ar FC="$IMG_TOOLCHAIN-gfortran -EL -mabi=n32" RANLIB=$IMG_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS=$CFLAGS LDFLAGS=$CFLAGS TARGET=I6400 - - I6400 Build (n64): - make BINARY=64 BINARY64=1 CC=$IMG_TOOLCHAIN-gcc AR=$IMG_TOOLCHAIN-ar FC="$IMG_TOOLCHAIN-gfortran -EL" RANLIB=$IMG_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS=$CFLAGS LDFLAGS=$CFLAGS TARGET=I6400 - - P6600 Build (n32): - make BINARY=32 BINARY32=1 CC=$IMG_TOOLCHAIN-gcc AR=$IMG_TOOLCHAIN-ar FC="$IMG_TOOLCHAIN-gfortran -EL -mabi=n32" RANLIB=$IMG_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS=$CFLAGS LDFLAGS=$CFLAGS TARGET=P6600 - - P6600 Build (n64): - make BINARY=64 BINARY64=1 CC=$IMG_TOOLCHAIN-gcc AR=$IMG_TOOLCHAIN-ar FC="$IMG_TOOLCHAIN-gfortran -EL" RANLIB=$IMG_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS="$CFLAGS" LDFLAGS="$CFLAGS" TARGET=P6600 - - MTI_TOOLCHAIN_DIR={full MTI GNU/Linux Toolchain path including "bin" directory -- for example, /opt/linux_toolchain/bin} - MTI_GCC_PREFIX=mips-mti-linux-gnu - MTI_TOOLCHAIN=${IMG_TOOLCHAIN_DIR}/${IMG_GCC_PREFIX} - - P5600 Build: - - make BINARY=32 BINARY32=1 CC=$MTI_TOOLCHAIN-gcc AR=$MTI_TOOLCHAIN-ar FC="$MTI_TOOLCHAIN-gfortran -EL" RANLIB=$MTI_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS=$CFLAGS LDFLAGS=$CFLAGS TARGET=P5600 - -#### Building OpenBLAS on POWER fails with IBM XL +### Building OpenBLAS on POWER fails with IBM XL Trying to compile OpenBLAS with IBM XL ends with error messages about unknown register names like "vs32". Working around these by using known alternate names for the vector registers only leads to another assembler error about unsupported constraints. This is a known deficiency in the IBM compiler at least up to and including 16.1.0 (and in the POWER version of clang, from which it is derived) - use gcc instead. (See issues #1078 and #1699 for related discussions) -#### Replacing system BLAS/updating APT OpenBLAS in Mint/Ubuntu/Debian +### Replacing system BLAS/updating APT OpenBLAS in Mint/Ubuntu/Debian Debian and Ubuntu LTS versions provide OpenBLAS package which is not updated after initial release, and under circumstances one might want to use more recent version of OpenBLAS e.g. to get support for newer CPUs @@ -333,41 +286,41 @@ $ sudo update-alternatives --remove libblas.so.3 /opt/OpenBLAS/lib/libopenblas.s In recent versions of the distributions, the installation path for the libraries has been changed to include the name of the host architecture, like /usr/lib/x86_64-linux-gnu/blas/libblas.so.3 or libblas.so.3.x86_64-linux-gnu. Use ```$ update-alternatives --display libblas.so.3``` to find out what layout your system has. -#### I built OpenBLAS for use with some other software, but that software cannot find it +### I built OpenBLAS for use with some other software, but that software cannot find it Openblas installs as a single library named libopenblas.so, while some programs may be searching for a separate libblas.so and liblapack.so so you may need to create appropriate symbolic links (`ln -s libopenblas.so libblas.so; ln -s libopenblas.so liblapack.so`) or copies. Also make sure that the installation location (usually /opt/OpenBLAS/lib or /usr/local/lib) is among the library search paths of your system. -#### I included cblas.h in my program, but the compiler complains about a missing common.h or functions from it +### I included cblas.h in my program, but the compiler complains about a missing common.h or functions from it You probably tried to include a cblas.h that you simply copied from the OpenBLAS source, instead you need to run `make install` after building OpenBLAS and then use the modified cblas.h that this step builds in the installation path (usually either /usr/local/include, /opt/OpenBLAS/include or whatever you specified as PREFIX= on the `make install`) -#### Compiling OpenBLAS with gcc's -fbounds-check actually triggers aborts in programs +### Compiling OpenBLAS with gcc's -fbounds-check actually triggers aborts in programs This is due to different interpretations of the (informal) standard for passing characters as arguments between C and FORTRAN functions. As the method for storing text differs in the two languages, when C calls Fortran the text length is passed as an "invisible" additional parameter. Historically, this has not been required when the text is just a single character, so older code like the Reference-LAPACK bundled with OpenBLAS does not do it. Recently gcc's checking has changed to require it, but there is no consensus yet if and how the existing LAPACK (and many other codebases) should adapt. (And for actual compilation, gcc has mostly backtracked and provided compatibility options - hence the default build settings in the OpenBLAS Makefiles add -fno-optimize-sibling-calls to the gfortran options to prevent miscompilation with "affected" versions. See ticket 2154 in the issue tracker for more details and links)
-#### Build fails with lots of errors about undefined ?GEMM_UNROLL_M +### Build fails with lots of errors about undefined ?GEMM_UNROLL_M Your cpu is apparently too new to be recognized by the build scripts, so they failed to assign appropriate parameters for the block algorithm. Do a `make clean` and try again with TARGET set to one of the cpu models listed in `TargetList.txt` - for x86_64 this will usually be HASWELL. -#### CMAKE/OSX: Build fails with 'argument list too long' +### CMAKE/OSX: Build fails with 'argument list too long' This is a limitation in the maximum length of a command on OSX, coupled with how CMAKE works. You should be able to work around this by adding the option `-DCMAKE_Fortran_USE_RESPONSE_FILE_FOR_OBJECTS=1` to your CMAKE arguments. -#### Likely problems with AVX2 support in Docker Desktop for OSX +### Likely problems with AVX2 support in Docker Desktop for OSX There have been a few reports of wrong calculation results and build-time test failures when building in a container environment managed by the OSX version of Docker Desktop, which uses the xhyve virtualizer underneath. Judging from these reports, AVX2 support in xhyve appears to be subtly broken but a corresponding ticket in the xhyve issue tracker has not drawn any reaction or comment since 2019. Therefore it is strongly recommended to build OpenBLAS with the NO_AVX2=1 option when inside a container under (or for later use with) the Docker Desktop environment on Intel-based Apple hardware. -### Usage +## Usage -#### Program is Terminated. Because you tried to allocate too many memory regions +### Program is Terminated. Because you tried to allocate too many memory regions In OpenBLAS, we mange a pool of memory buffers and allocate the number of buffers as the following. ``` @@ -378,16 +331,16 @@ This error indicates that the program exceeded the number of buffers. Please build OpenBLAS with larger `NUM_THREADS`. For example, `make NUM_THREADS=32` or `make NUM_THREADS=64`. In `Makefile.system`, we will set `MAX_CPU_NUMBER=NUM_THREADS`. -#### How to choose TARGET manually at runtime when compiled with DYNAMIC_ARCH +### How to choose TARGET manually at runtime when compiled with DYNAMIC_ARCH The environment variable which control the kernel selection is `OPENBLAS_CORETYPE` (see `driver/others/dynamic.c`) e.g. `export OPENBLAS_CORETYPE=Haswell`. And the function `char* openblas_get_corename()` returns the used target. -#### After updating the installed OpenBLAS, a program complains about "undefined symbol gotoblas" +### After updating the installed OpenBLAS, a program complains about "undefined symbol gotoblas" This symbol gets defined only when OpenBLAS is built with "make DYNAMIC_ARCH=1" (which is what distributors will choose to ensure support for more than just one CPU type). -#### How can I find out at runtime what options the library was built with ? +### How can I find out at runtime what options the library was built with ? OpenBLAS has two utility functions that may come in here: @@ -395,31 +348,31 @@ openblas_get_parallel() will return 0 for a single-threaded library, 1 if multit openblas_get_config() will return a string containing settings such as USE64BITINT or DYNAMIC_ARCH that were active at build time, as well as the target cpu (or in case of a dynamic_arch build, the currently detected one). -#### After making OpenBLAS, I find that the static library is multithreaded, but the dynamic one is not ? +### After making OpenBLAS, I find that the static library is multithreaded, but the dynamic one is not ? The shared OpenBLAS library you built is probably working fine as well, but your program may be picking up a different (probably single-threaded) version from one of the standard system paths like /usr/lib on startup. Running `ldd /path/to/your/program` will tell you which library the linkage loader will actually use. Specifying the "correct" library location with the `-L` flag (like `-L /opt/OpenBLAS/lib`) when linking your program only defines which library will be used to see if all symbols _can_ be resolved, you will need to add an rpath entry to the binary (using `-Wl,rpath=/opt/OpenBLAS/lib`) to make it request searching that location. Alternatively, remove the "wrong old" library (if you can), or set LD_LIBRARY_PATH to the desired location before running your program. -#### I want to use OpenBLAS with CUDA in the HPL 2.3 benchmark code but it keeps looking for Intel MKL +### I want to use OpenBLAS with CUDA in the HPL 2.3 benchmark code but it keeps looking for Intel MKL You need to edit file src/cuda/cuda_dgemm.c in the NVIDIA version of HPL, change the "handle2" and "handle" dlopen calls to use libopenblas.so instead of libmkl_intel_lp64.so, and add an trailing underscore in the dlsym lines for dgemm_mkl and dtrsm_mkl (like `dgemm_mkl = (void(*)())dlsym(handle, “dgemm_”);`) -#### Multithreaded OpenBLAS runs no faster or is even slower than singlethreaded on my ARMV7 board +### Multithreaded OpenBLAS runs no faster or is even slower than singlethreaded on my ARMV7 board The power saving mechanisms of your board may have shut down some cores, making them invisible to OpenBLAS in its startup phase. Try bringing them online before starting your calculation. -#### Speed varies wildly between individual runs on a typical ARMV8 smartphone processor +### Speed varies wildly between individual runs on a typical ARMV8 smartphone processor Check the technical specifications, it could be that the SoC combines fast and slow cpus and threads can end up on either. In that case, binding the process to specific cores e.g. by setting `OMP_PLACES=cores` may help. (You may need to experiment with OpenMP options, it has been reported that using `OMP_NUM_THREADS=2 OMP_PLACES=cores` caused a huge drop in performance on a 4+4 core chip while `OMP_NUM_THREADS=2 OMP_PLACES=cores(2)` worked as intended - as did OMP_PLACES=cores with 4 threads) -#### I cannot get OpenBLAS to use more than a small subset of available cores on a big system +### I cannot get OpenBLAS to use more than a small subset of available cores on a big system Multithreading support in OpenBLAS requires the use of internal buffers for sharing partial results, the number and size of which is defined at compile time. Unless you specify NUM_THREADS in your make or cmake command, the build scripts try to autodetect the number of cores available in your build host to size the library to match. This unfortunately means that if you move the resulting binary from a small "front-end node" to a larger "compute node" later, it will still be limited to the hardware capabilities of the original system. The solution is to set NUM_THREADS to a number big enough to encompass the biggest systems you expect to run the binary on - at runtime, it will scale down the maximum number of threads it uses to match the number of cores physically available. -#### Getting "ELF load command address/offset not properly aligned" when loading libopenblas.so +### Getting "ELF load command address/offset not properly aligned" when loading libopenblas.so If you get a message "error while loading shared libraries: libopenblas.so.0: ELF load command address/offset not properly aligned" when starting a program that is (dynamically) linked to OpenBLAS, this is very likely due to a bug in the GNU linker (ld) that is part of the GNU binutils package. This error was specifically observed on older versions of Ubuntu Linux updated with the (at the time) most recent binutils version 2.38, but an internet search turned up sporadic reports involving various other libraries dating back several years. A bugfix was created by the binutils developers and should be available in later versions of binutils.(See issue 3708 for details) diff --git a/docs/import_mingw.md b/docs/import_mingw.md deleted file mode 100644 index a0038c1de9..0000000000 --- a/docs/import_mingw.md +++ /dev/null @@ -1,36 +0,0 @@ -Microsoft Windows has this thing called "import libraries". You don't need it in MinGW because the `ld` linker from GNU Binutils is smart, but you may still want it for whatever reason. - -## Make the `.def` - -Import libraries are compiled from a list of what symbols to use, `.def`. This should be already in your `exports` directory: `cd OPENBLAS_TOP_DIR/exports`. - -## Making a MinGW import library - -MinGW import libraries have the suffix `.a`, same as static libraries. (It's actually more common to do `.dll.a`...) - -You need to first prepend `libopenblas.def` with a line `LIBRARY libopenblas.dll`: - - cat <(echo "LIBRARY libopenblas.dll") libopenblas.def > libopenblas.def.1 - mv libopenblas.def.1 libopenblas.def - -Now it probably looks like: - - LIBRARY libopenblas.dll - EXPORTS - caxpy=caxpy_ @1 - caxpy_=caxpy_ @2 - ... - -Then, generate the import library: `dlltool -d libopenblas.def -l libopenblas.a` - -Again, there is basically **no point** in making an import library for use in MinGW. It actually slows down linking. - -## Making a MSVC import library - -Unlike MinGW, MSVC absolutely requires an import library. Now the C ABI of MSVC and MinGW are actually identical, so linking is actually okay. (Any incompatibility in the C ABI would be a bug.) - -The import libraries of MSVC have the suffix `.lib`. They are generated from a `.def` file using MSVC's `lib.exe`. See [the MSVC instructions](https://github.com/xianyi/OpenBLAS/wiki/How-to-use-OpenBLAS-in-Microsoft-Visual-Studio#generate-import-library-before-0210-version). - -## Notes -* Always remember that MinGW is **not the same** as MSYS2 or Cygwin. MSYS2 and Cygwin are full POSIX environments with a lot of magic such as `fork()` and its own `malloc()`. MinGW, which builds on the normal Microsoft C Runtime, has none of that. Be clear about which one you are building for. - diff --git a/docs/index.md b/docs/index.md index 06b6f108db..279cfe57de 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,23 +1,8 @@ -# OpenBLAS Wiki - -
-
- [Home] - [Document] - [FAQ] - [Publications] - [Download] -[Mailing List] -[Donation] -
-
- -[![Build Status](https://travis-ci.org/xianyi/OpenBLAS.svg?branch=develop)](https://travis-ci.org/xianyi/OpenBLAS) - -### Introduction +## Introduction + OpenBLAS is an optimized Basic Linear Algebra Subprograms (BLAS) library based on [GotoBLAS2](https://www.tacc.utexas.edu/research-development/tacc-software/gotoblas2) 1.13 BSD version. -### License +## License OpenBLAS is licensed under the 3-clause BSD license. Full license text follows: @@ -49,8 +34,8 @@ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -### Acknowledgements +## Acknowledgements This work is partially supported by * Research and Development of Compiler System and Toolchain for Domestic CPU, National S&T Major Projects: Core Electronic Devices, High-end General Chips and Fundamental Software (No.2009ZX01036-001-002) -* National High-tech R&D Program of China (Grant No.2012AA010903) \ No newline at end of file +* National High-tech R&D Program of China (Grant No.2012AA010903) diff --git a/docs/install.md b/docs/install.md index 84c8e67998..e3a19db5b4 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1,21 +1,88 @@ -## Quick Installation -[Precompiled packages](https://github.com/xianyi/OpenBLAS/wiki/Precompiled-installation-packages) have recently become available for a number of platforms through their normal installation procedures, so for users of desktop devices at least, the instructions below are mostly relevant when you want to try the most recent development snapshot from git. +# Install OpenBLAS + +!!! note + Lists of precompiled packages are not comprehensive, is not meant to validate nor endorse a particular third-party build over others, and may not always lead to the newest version + + +## Quick install + +Precompiled packages have recently become available for a number of platforms through their normal installation procedures, so for users of desktop devices at least, the instructions below are mostly relevant when you want to try the most recent development snapshot from git. See your platform's relevant "Precompiled packages" section. + +The [Conda-Forge](https://github.com/conda-forge) project maintains packages for the conda package manager at . + +## Source +Download the latest [stable version](https://github.com/xianyi/OpenBLAS/releases) from release page. + +## Platforms ### Linux Just type `make` to compile the library. -Notes +Notes: + * OpenBLAS doesn't support g77. Please use gfortran or other Fortran compilers. e.g. `make FC=gfortran`. * When building in an emulator (KVM,QEMU etc.) make sure that the combination of CPU features exposed to the virtual environment matches that of an existing CPU to allow detection of the cpu model to succeed. (With qemu, this can be done by passing `-cpu host` or a supported model name at invocation) -### Windows -See [[How-to-use-OpenBLAS-in-Microsoft-Visual-Studio]]. +#### Precompiled packages + +##### Debian/Ubuntu/Mint/Kali + OpenBLAS package is available in default repositories and can act as default BLAS in system + +Example installation commands: +```bash +$ sudo apt update +$ apt search openblas +$ sudo apt install libopenblas-dev +$ sudo update-alternatives --config libblas.so.3 +``` + Alternatively, if distributor's package proves unsatisfactory, you may try latest version of OpenBLAS, [Following guide in OpenBLAS FAQ](faq.md#debianlts) + +##### openSuSE/SLE + Recent OpenSUSE versions include OpenBLAS in default repositories and also permit OpenBLAS to act as replacement of system-wide BLAS. + + Example installation commands: +```bash +$ sudo zypper ref +$ zypper se openblas +$ sudo zypper in openblas-devel +$ sudo update-alternatives --config libblas.so.3 +``` +Should you be using older OpenSUSE or SLE that provides no OpenBLAS, you can attach optional or experimental openSUSE repository as a new package source to acquire recent build of OpenBLAS following [instructions on openSUSE software site](https://software.opensuse.org/package/openblas) + +##### Fedora/CentOS/RHEL +Fedora provides OpenBLAS in default installation repositories. + +To install it try following: +```bash +$ dnf search openblas +$ dnf install openblas-devel +``` +For CentOS/RHEL/Scientific Linux packages are provided via [Fedora EPEL repository](https://fedoraproject.org/wiki/EPEL) + +After adding repository and repository keys installation is pretty straightforward: +```bash +$ yum search openblas +$ yum install openblas-devel +``` +No alternatives mechanism is provided for BLAS, and packages in system repositories are linked against NetLib BLAS or ATLAS BLAS libraries. You may wish to re-package RPMs to use OpenBLAS instead [as described here](https://fedoraproject.org/wiki/How_to_create_an_RPM_package) + +##### Mageia +Mageia offers ATLAS and NetLIB LAPACK in base repositories. +You can build your own OpenBLAS replacement, and once installed in /opt +TODO: populate /usr/lib64 /usr/include accurately to replicate netlib with update-alternatives + +##### Arch/Manjaro/Antergos +```bash +$ sudo pacman -S openblas +``` + +### Windows -The precompiled binaries available with each release (in https://github.com/xianyi/OpenBLAS/releases) are +The precompiled binaries available with each release (in ) are created with MinGW (as described in Section 2 of the wiki page mentioned above) using an option list of "NUM_THREADS=64 TARGET=GENERIC DYNAMIC_ARCH=1 DYNAMIC_OLDER=1 CONSISTENT_FPCSR=1" - they should work on any x86 or x86_64 computer. The zip archive contains the include files, static and dll libraries as well @@ -32,21 +99,546 @@ documentation of whatever program you are using with OpenBLAS mentions 64bit int addressing huge matrix sizes, you will need to build OpenBLAS from source (or open an issue ticket to make the demand for such a precompiled build known). +#### Precompiled packages + +* +* + +#### Visual Studio + +As of OpenBLAS v0.2.15, we support MinGW and Visual Studio (using CMake to generate visual studio solution files – note that you will need at least version 3.11 of CMake for linking to work correctly) to build OpenBLAS on Windows. + +Note that you need a Fortran compiler if you plan to build and use the LAPACK functions included with OpenBLAS. The sections below describe using either `flang` as an add-on to clang/LLVM or `gfortran` as part of MinGW for this purpose. If you want to use the Intel Fortran compiler `ifort` for this, be sure to also use the Intel C compiler `icc` for building the C parts, as the ABI imposed by `ifort` is incompatible with `msvc`. + +##### 1. Native (MSVC) ABI + +A fully-optimized OpenBLAS that can be statically or dynamically linked to your application can currently be built for the 64-bit architecture with the LLVM compiler infrastructure. We're going to use Miniconda3 to grab all of the tools we need, since some of them are in an experimental status. Before you begin, you'll need to have Microsoft Visual Studio 2015 or newer installed. + +1. Install Miniconda3 for 64 bits using `winget install --id Anaconda.Miniconda3` or easily download from [conda.io](https://docs.conda.io/en/latest/miniconda.html). +2. Open the "Anaconda Command Prompt," now available in the Start Menu, or at `%USERPROFILE%\miniconda3\shell\condabin\conda-hook.ps1`. +3. In that command prompt window, use `cd` to change to the directory where you want to build OpenBLAS +4. Now install all of the tools we need: + + ``` + conda update -n base conda + conda config --add channels conda-forge + conda install -y cmake flang clangdev perl libflang ninja + ``` + +5. Still in the Anaconda Command Prompt window, activate the MSVC environment for 64 bits with `vcvarsall x64`. On Windows 11 with Visual Studio 2022, this would be done by invoking: + + ```shell + "c:\Program Files\Microsoft Visual Studio\2022\Preview\vc\Auxiliary\Build\vcvars64.bat" + ``` + + With VS2019, the command should be the same – except for the year number, obviously. For other/older versions of MSVC, + the VS documentation or a quick search on the web should turn up the exact wording you need. + + Confirm that the environment is active by typing `link` – this should return a long list of possible options for the `link` command. If it just + returns "command not found" or similar, review and retype the call to vcvars64.bat. + **NOTE:** if you are working from a Visual Studio Command prompt window instead (so that you do not have to do the vcvars call), you need to invoke + `conda activate` so that CONDA_PREFIX etc. get set up correctly before proceeding to step 6. Failing to do so will lead to link errors like + libflangmain.lib not getting found later in the build. + +6. Now configure the project with CMake. Starting in the project directory, execute the following: + + ``` + set "LIB=%CONDA_PREFIX%\Library\lib;%LIB%" + set "CPATH=%CONDA_PREFIX%\Library\include;%CPATH%" + mkdir build + cd build + cmake .. -G "Ninja" -DCMAKE_CXX_COMPILER=clang-cl -DCMAKE_C_COMPILER=clang-cl -DCMAKE_Fortran_COMPILER=flang -DCMAKE_MT=mt -DBUILD_WITHOUT_LAPACK=no -DNOFORTRAN=0 -DDYNAMIC_ARCH=ON -DCMAKE_BUILD_TYPE=Release + ``` + + You may want to add further options in the `cmake` command here – for instance, the default only produces a static .lib version of the library. If you would rather have a DLL, add -DBUILD_SHARED_LIBS=ON above. Note that this step only creates some command files and directories, the actual build happens next. + + +7. Build the project: + + ``` + cmake --build . --config Release + ``` + This step will create the OpenBLAS library in the "lib" directory, and various build-time tests in the `test`, `ctest` and `openblas_utest` directories. However it will not separate the header files you might need for building your own programs from those used internally. To put all relevant files in a more convenient arrangement, run the next step. + +8. Install all relevant files created by the build + + ``` + cmake --install . --prefix c:\opt -v + ``` + This will copy all files that are needed for building and running your own programs with OpenBLAS to the given location, creating appropriate subdirectories for the individual kinds of files. In the case of "C:\opt" as given above, this would be C:\opt\include\openblas for the header files, + C:\opt\bin for the libopenblas.dll and C:\opt\lib for the static library. C:\opt\share holds various support files that enable other cmake-based build scripts to find OpenBLAS automatically. + +###### Visual studio 2017+ (C++2017 standard) + +In newer visual studio versions, Microsoft has changed [how it handles complex types](https://docs.microsoft.com/en-us/cpp/c-runtime-library/complex-math-support?view=msvc-170#types-used-in-complex-math). Even when using a precompiled version of OpenBLAS, you might need to define `LAPACK_COMPLEX_CUSTOM` in order to define complex types properly for MSVC. For example, some variant of the following might help: + +``` +#if defined(_MSC_VER) + #include + #define LAPACK_COMPLEX_CUSTOM + #define lapack_complex_float _Fcomplex + #define lapack_complex_double _Dcomplex +#endif +``` + +For reference, see https://github.com/xianyi/OpenBLAS/issues/3661, https://github.com/Reference-LAPACK/lapack/issues/683, and https://stackoverflow.com/questions/47520244/using-openblas-lapacke-in-visual-studio. + +###### CMake and Visual Studio + +To build OpenBLAS for the 32-bit architecture, you'll need to use the builtin Visual Studio compilers. + +!!! note + This method may produce binaries which demonstrate significantly lower performance than those built with the other methods. (The Visual Studio compiler does not support the dialect of assembly used in the cpu-specific optimized files, so only the "generic" TARGET which is + written in pure C will get built. For the same reason it is not possible (and not necessary) to use -DDYNAMIC_ARCH=ON in a Visual Studio build) You may consider building for the 32-bit architecture using the GNU (MinGW) ABI. + +####### 1. Install CMake at Windows + +####### 2. Use CMake to generate Visual Studio solution files + +``` +# Do this from Powershell so cmake can find visual studio +cmake -G "Visual Studio 14 Win64" -DCMAKE_BUILD_TYPE=Release . +``` + +###### Build the solution at Visual Studio + +Note that this step depends on perl, so you'll need to install perl for windows, and put perl on your path so VS can start perl (http://stackoverflow.com/questions/3051049/active-perl-installation-on-windows-operating-system). + +Step 2 will build the OpenBLAS solution, open it in VS, and build the projects. Note that the dependencies do not seem to be automatically configured: if you try to build libopenblas directly, it will fail with a message saying that some .obj files aren't found, but if you build the projects libopenblas depends on before building libopenblas, the build will succeed. + +###### Build OpenBLAS for Universal Windows Platform + +OpenBLAS can be built for use on the [Universal Windows Platform](https://en.wikipedia.org/wiki/Universal_Windows_Platform) using a two step process since commit [c66b842](https://github.com/xianyi/OpenBLAS/commit/c66b842d66c5516e52804bf5a0544d18b1da1b44). + +####### 1. Follow steps 1 and 2 above to build the Visual Studio solution files for Windows. This builds the helper executables which are required when building the OpenBLAS Visual Studio solution files for UWP in step 2. + +####### 2. Remove the generated CMakeCache.txt and CMakeFiles directory from the OpenBLAS source directory and re-run CMake with the following options: + +``` +# do this to build UWP compatible solution files +cmake -G "Visual Studio 14 Win64" -DCMAKE_SYSTEM_NAME=WindowsStore -DCMAKE_SYSTEM_VERSION="10.0" -DCMAKE_SYSTEM_PROCESSOR=AMD64 -DVS_WINRT_COMPONENT=TRUE -DCMAKE_BUILD_TYPE=Release . +``` + +####### Build the solution with Visual Studio + +This will build the OpenBLAS binaries with the required settings for use with UWP. + +##### 2. GNU (MinGW) ABI + +The resulting library can be used in Visual Studio, but it can only be linked dynamically. This configuration has not been thoroughly tested and should be considered experimental. + +###### Incompatible x86 calling conventions + +Due to incompatibilities between the calling conventions of MinGW and Visual Studio you will need to make the following modifications ( **32-bit only** ): + +1. Use the newer GCC 4.7.0. The older GCC (<4.7.0) has an ABI incompatibility for returning aggregate structures larger than 8 bytes with MSVC. + + +###### Build OpenBLAS on Windows OS +1. Install the MinGW (GCC) compiler suite, either 32-bit (http://www.mingw.org/) or 64-bit (http://mingw-w64.sourceforge.net/). Be sure to install its gfortran package as well (unless you really want to build the BLAS part of OpenBLAS only) and check that gcc and gfortran are the same version – mixing compilers from different sources or release versions can lead to strange error messages in the linking stage. In addition, please install MSYS with MinGW. +1. Build OpenBLAS in the MSYS shell. Usually, you can just type "make". OpenBLAS will detect the compiler and CPU automatically. +1. After the build is complete, OpenBLAS will generate the static library "libopenblas.a" and the shared dll library "libopenblas.dll" in the folder. You can type "make PREFIX=/your/installation/path install" to install the library to a certain location. + +!!! note + We suggest using official MinGW or MinGW-w64 compilers. A user reported that s/he met `Unhandled exception` by other compiler suite. https://groups.google.com/forum/#!topic/openblas-users/me2S4LkE55w + +Note also that older versions of the alternative builds of mingw-w64 available through http://www.msys2.org may contain a defect that leads to a compilation failure accompanied by the error message +``` +:0:4: error: expected identifier or '(' before numeric constant +``` +If you encounter this, please upgrade your msys2 setup or see https://github.com/xianyi/OpenBLAS/issues/1503 for a workaround. + +###### Generate import library (before 0.2.10 version) + +1. First, you will need to have the `lib.exe` tool in the Visual Studio command prompt. +1. Open the command prompt and type `cd OPENBLAS_TOP_DIR/exports`, where OPENBLAS_TOP_DIR is the main folder of your OpenBLAS installation. +1. For a 32-bit library, type `lib /machine:i386 /def:libopenblas.def`. For 64-bit, type `lib /machine:X64 /def:libopenblas.def`. +1. This will generate the import library "libopenblas.lib" and the export library "libopenblas.exp" in OPENBLAS_TOP_DIR/exports. Although these two files have the same name, they are totally different. + +###### Generate import library (0.2.10 and after version) +1. OpenBLAS already generated the import library "libopenblas.dll.a" for "libopenblas.dll". + +###### generate windows native PDB files from gcc/gfortran build +Tool to do so is available at https://github.com/rainers/cv2pdb + +###### Use OpenBLAS .dll library in Visual Studio +1. Copy the import library (before 0.2.10: "OPENBLAS_TOP_DIR/exports/libopenblas.lib", 0.2.10 and after: "OPENBLAS_TOP_DIR/libopenblas.dll.a") and .dll library "libopenblas.dll" into the same folder(The folder of your project that is going to use the BLAS library. You may need to add the libopenblas.dll.a to the linker input list: properties->Linker->Input). +1. Please follow the documentation about using third-party .dll libraries in MS Visual Studio 2008 or 2010. Make sure to link against a library for the correct architecture. For example, you may receive an error such as "The application was unable to start correctly (0xc000007b)" which typically indicates a mismatch between 32/64-bit libraries. + +!!! note + If you need CBLAS, you should include cblas.h in /your/installation/path/include in Visual Studio. Please read [this page](http://github.com/xianyi/OpenBLAS/issues/95). + +###### Limitations +* Both static and dynamic linking are supported with MinGW. With Visual Studio, however, only dynamic linking is supported and so you should use the import library. +* Debugging from Visual Studio does not work because MinGW and Visual Studio have incompatible formats for debug information (PDB vs. DWARF/STABS). You should either debug with GDB on the command-line or with a visual frontend, for instance [Eclipse](http://www.eclipse.org/cdt/) or [Qt Creator](http://qt.nokia.com/products/developer-tools/). + + +#### Windows on Arm + +##### Prerequisites + +Following tools needs to be installed + +###### 1. Download and install clang for windows on arm + +Find the latest LLVM build for WoA from [LLVM release page](https://releases.llvm.org/) + +E.g: LLVM 12 build for WoA64 can be found [here](https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.0/LLVM-12.0.0-woa64.exe) + +Run the LLVM installer and ensure that LLVM is added to environment PATH. + +###### 2. Download and install classic flang for windows on arm + +Classic flang is the only available FORTRAN compiler for windows on arm for now and a pre-release build can be found [here](https://github.com/kaadam/flang/releases/tag/v0.1) + +There is no installer for classic flang and the zip package can be extracted and the path needs to be added to environment PATH. + +E.g: on PowerShell + +``` +$env:Path += ";C:\flang_woa\bin" +``` + +##### Build + +The following steps describe how to build the static library for OpenBLAS with and without LAPACK + +###### 1. Build OpenBLAS static library with BLAS and LAPACK routines with Make + +Following command can be used to build OpenBLAS static library with BLAS and LAPACK routines + +```bash +$ make CC="clang-cl" HOSTCC="clang-cl" AR="llvm-ar" BUILD_WITHOUT_LAPACK=0 NOFORTRAN=0 DYNAMIC_ARCH=0 TARGET=ARMV8 ARCH=arm64 BINARY=64 USE_OPENMP=0 PARALLEL=1 RANLIB="llvm-ranlib" MAKE=make F_COMPILER=FLANG FC=FLANG FFLAGS_NOOPT="-march=armv8-a -cpp" FFLAGS="-march=armv8-a -cpp" NEED_PIC=0 HOSTARCH=arm64 libs netlib +``` + +###### 2. Build static library with BLAS routines using CMake + +Classic flang has compatibility issues with cmake hence only BLAS routines can be compiled with CMake + +```bash +$ mkdir build +$ cd build +$ cmake .. -G Ninja -DCMAKE_C_COMPILER=clang -DBUILD_WITHOUT_LAPACK=1 -DNOFORTRAN=1 -DDYNAMIC_ARCH=0 -DTARGET=ARMV8 -DARCH=arm64 -DBINARY=64 -DUSE_OPENMP=0 -DCMAKE_SYSTEM_PROCESSOR=ARM64 -DCMAKE_CROSSCOMPILING=1 -DCMAKE_SYSTEM_NAME=Windows +$ cmake --build . --config Release +``` + +###### `getarch.exe` execution error + +If you notice that platform-specific headers by `getarch.exe` are not generated correctly, It could be due to a known debug runtime DLL issue for arm64 platforms. Please check out [link](https://linaro.atlassian.net/wiki/spaces/WOAR/pages/28677636097/Debug+run-time+DLL+issue#Workaround) for the workaround. + +#### MinGW import library + +Microsoft Windows has this thing called "import libraries". You don't need it in MinGW because the `ld` linker from GNU Binutils is smart, but you may still want it for whatever reason. + +##### Make the `.def` + +Import libraries are compiled from a list of what symbols to use, `.def`. This should be already in your `exports` directory: `cd OPENBLAS_TOP_DIR/exports`. + +##### Making a MinGW import library + +MinGW import libraries have the suffix `.a`, same as static libraries. (It's actually more common to do `.dll.a`...) + +You need to first prepend `libopenblas.def` with a line `LIBRARY libopenblas.dll`: + + cat <(echo "LIBRARY libopenblas.dll") libopenblas.def > libopenblas.def.1 + mv libopenblas.def.1 libopenblas.def + +Now it probably looks like: + + LIBRARY libopenblas.dll + EXPORTS + caxpy=caxpy_ @1 + caxpy_=caxpy_ @2 + ... + +Then, generate the import library: `dlltool -d libopenblas.def -l libopenblas.a` + +Again, there is basically **no point** in making an import library for use in MinGW. It actually slows down linking. + +##### Making a MSVC import library + +Unlike MinGW, MSVC absolutely requires an import library. Now the C ABI of MSVC and MinGW are actually identical, so linking is actually okay. (Any incompatibility in the C ABI would be a bug.) + +The import libraries of MSVC have the suffix `.lib`. They are generated from a `.def` file using MSVC's `lib.exe`. See [the MSVC instructions](use_visual_studio.md#generate-import-library-before-0210-version). + +##### Notes + +* Always remember that MinGW is **not the same** as MSYS2 or Cygwin. MSYS2 and Cygwin are full POSIX environments with a lot of magic such as `fork()` and its own `malloc()`. MinGW, which builds on the normal Microsoft C Runtime, has none of that. Be clear about which one you are building for. + ### Mac OSX If your CPU is Sandy Bridge, please use Clang version 3.1 and above. The Clang 3.0 will generate the wrong AVX binary code of OpenBLAS. +#### Precompiled packages + + + +`brew install openblas` + +or using the conda package manager from + +(which also has packages for the new M1 cpu) + + `conda install openblas` + #### Build on Apple M1 +On newer versions of Xcode and on arm64, you might need to compile with a newer macOS target (11.0) than the default (10.8) with `MACOSX_DEPLOYMENT_TARGET=11.0`, or switch your command-line tools to use an older SDK (e.g., [13.1](https://developer.apple.com/download/all/?q=Xcode%2013)). + * without Fortran compiler （cannot build LAPACK) + ```bash + $ make CC=cc NOFORTRAN=1 + ``` +* with Fortran compiler (you could `brew install gfortran`) https://github.com/xianyi/OpenBLAS/issues/3032 + ```bash + $ export MACOSX_DEPLOYMENT_TARGET=11.0 + $ make CC=cc FC=gfortran + ``` + +### Android + +#### Build + +##### Prerequisites + +In addition to the Android NDK, you will need both Perl and a C compiler on the build host as these are currently +required by the OpenBLAS build environment. + + +##### Building with android NDK using clang compiler +Around version 11 Android NDKs stopped supporting gcc, so you would need to use clang to compile OpenBLAS. clang is supported from OpenBLAS 0.2.20 version onwards. See below sections on how to build with clang for ARMV7 and ARMV8 targets. The same basic principles as described below for ARMV8 should also apply to building an x86 or x86_64 version (substitute something like NEHALEM for the target instead of ARMV8 and replace all the aarch64 in the toolchain paths obviously) +"Historic" notes: +Since version 19 the default toolchain is provided as a standalone toolchain, so building one yourself following [building a standalone toolchain](http://developer.android.com/ndk/guides/standalone_toolchain.html) should no longer be necessary. +If you want to use static linking with an old NDK version older than about r17, you need to choose an API level below 23 currently due to NDK bug 272 (https://github.com/android-ndk/ndk/issues/272 , the libc.a lacks a definition of stderr) that will probably be fixed in r17 of the NDK. + +##### Build ARMV7 with clang ``` - $ make CC=cc NOFORTRAN=1 +## Set path to ndk-bundle +export NDK_BUNDLE_DIR=/path/to/ndk-bundle + +## Set the PATH to contain paths to clang and arm-linux-androideabi-* utilities +export PATH=${NDK_BUNDLE_DIR}/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/bin:${NDK_BUNDLE_DIR}/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH + +## Set LDFLAGS so that the linker finds the appropriate libgcc +export LDFLAGS="-L${NDK_BUNDLE_DIR}/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/lib/gcc/arm-linux-androideabi/4.9.x" + +## Set the clang cross compile flags +export CLANG_FLAGS="-target arm-linux-androideabi -marm -mfpu=vfp -mfloat-abi=softfp --sysroot ${NDK_BUNDLE_DIR}/platforms/android-23/arch-arm -gcc-toolchain ${NDK_BUNDLE_DIR}/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/" + +#OpenBLAS Compile +make TARGET=ARMV7 ONLY_CBLAS=1 AR=ar CC="clang ${CLANG_FLAGS}" HOSTCC=gcc ARM_SOFTFP_ABI=1 -j4 ``` -* with Fortran compiler (you could `brew install gfortran`) https://github.com/xianyi/OpenBLAS/issues/3032 +On a Mac, it may also be necessary to give the complete path to the `ar` utility in the make command above, like so: +``` +AR=${NDK_BUNDLE_DIR}/toolchains/arm-linux-androideabi-4.9/prebuilt/darwin-x86_64/bin/arm-linux-androideabi-gcc-ar +``` +otherwise you may get a linker error complaining about a "malformed archive header name at 8" when the native OSX ar command was invoked instead. + +##### Build ARMV8 with clang +``` +## Set path to ndk-bundle +export NDK_BUNDLE_DIR=/path/to/ndk-bundle/ + +## Export PATH to contain directories of clang and aarch64-linux-android-* utilities +export PATH=${NDK_BUNDLE_DIR}/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/bin/:${NDK_BUNDLE_DIR}/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH + +## Setup LDFLAGS so that loader can find libgcc and pass -lm for sqrt +export LDFLAGS="-L${NDK_BUNDLE_DIR}/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/lib/gcc/aarch64-linux-android/4.9.x -lm" + +## Setup the clang cross compile options +export CLANG_FLAGS="-target aarch64-linux-android --sysroot ${NDK_BUNDLE_DIR}/platforms/android-23/arch-arm64 -gcc-toolchain ${NDK_BUNDLE_DIR}/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/" + +## Compile +make TARGET=ARMV8 ONLY_CBLAS=1 AR=ar CC="clang ${CLANG_FLAGS}" HOSTCC=gcc -j4 +``` +Note: Using TARGET=CORTEXA57 in place of ARMV8 will pick up better optimized routines. Implementations for CORTEXA57 target is compatible with all other armv8 targets. + +Note: For NDK 23b, something as simple as +``` +export PATH=/opt/android-ndk-r23b/toolchains/llvm/prebuilt/linux-x86_64/bin/:$PATH +make HOSTCC=gcc CC=/opt/android-ndk-r23b/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android31-clang ONLY_CBLAS=1 TARGET=ARMV8 +``` +appears to be sufficient on Linux. + +##### Alternative script which was tested on OSX with NDK(21.3.6528147) +This script will build openblas for 3 architecture (ARMV7,ARMV8,X86) and put them with `sudo make install` to `/opt/OpenBLAS/lib` +``` +export NDK=YOUR_PATH_TO_SDK/Android/sdk/ndk/21.3.6528147 +export TOOLCHAIN=$NDK/toolchains/llvm/prebuilt/darwin-x86_64 + +make clean +make \ + TARGET=ARMV7 \ + ONLY_CBLAS=1 \ + CC="$TOOLCHAIN"/bin/armv7a-linux-androideabi21-clang \ + AR="$TOOLCHAIN"/bin/arm-linux-androideabi-ar \ + HOSTCC=gcc \ + ARM_SOFTFP_ABI=1 \ + -j4 +sudo make install + +make clean +make \ + TARGET=CORTEXA57 \ + ONLY_CBLAS=1 \ + CC=$TOOLCHAIN/bin/aarch64-linux-android21-clang \ + AR=$TOOLCHAIN/bin/aarch64-linux-android-ar \ + HOSTCC=gcc \ + -j4 +sudo make install + +make clean +make \ + TARGET=ATOM \ + ONLY_CBLAS=1 \ + CC="$TOOLCHAIN"/bin/i686-linux-android21-clang \ + AR="$TOOLCHAIN"/bin/i686-linux-android-ar \ + HOSTCC=gcc \ + ARM_SOFTFP_ABI=1 \ + -j4 +sudo make install + +## This will build for x86_64 +make clean +make \ + TARGET=ATOM BINARY=64\ + ONLY_CBLAS=1 \ + CC="$TOOLCHAIN"/bin/x86_64-linux-android21-clang \ + AR="$TOOLCHAIN"/bin/x86_64-linux-android-ar \ + HOSTCC=gcc \ + ARM_SOFTFP_ABI=1 \ + -j4 +sudo make install +``` +Also you can find full list of target architectures in [TargetsList.txt](https://github.com/xianyi/OpenBLAS/blob/develop/TargetList.txt) + +*** +anything below this line should be irrelevant nowadays unless you need to perform software archeology +*** +##### Building OpenBLAS with very old gcc-based versions of the NDK, without Fortran + +The prebuilt Android NDK toolchains do not include Fortran, hence parts like LAPACK cannot be built. You can still build OpenBLAS without it. For instructions on how to build OpenBLAS with Fortran, see the [next section](#building-openblas-with-fortran). + +To use easily the prebuilt toolchains, follow [building a standalone toolchain](http://developer.android.com/ndk/guides/standalone_toolchain.html) for your desired architecture. +This would be `arm-linux-androideabi-gcc-4.9` for ARMV7 and `aarch64-linux-android-gcc-4.9` for ARMV8. + +You can build OpenBLAS (0.2.19 and earlier) with: +``` +## Add the toolchain to your path +export PATH=/path/to/standalone-toolchain/bin:$PATH + +## Build without Fortran for ARMV7 +make TARGET=ARMV7 HOSTCC=gcc CC=arm-linux-androideabi-gcc NOFORTRAN=1 libs +## Build without Fortran for ARMV8 +make TARGET=ARMV8 BINARY=64 HOSTCC=gcc CC=aarch64-linux-android-gcc NOFORTRAN=1 libs +``` + +Since we are cross-compiling, we make the `libs` recipe, not `all`. Otherwise you will get errors when trying to link/run tests as versions up to and including 0.2.19 cannot build a shared library for Android. + +From 0.2.20 on, you should leave off the "libs" to get a full build, and you may want to use the softfp ABI instead of the deprecated hardfp one on ARMV7 so you would use ``` - $ export MACOSX_DEPLOYMENT_TARGET=11.0 - $ make CC=cc FC=gfortran +## Add the toolchain to your path +export PATH=/path/to/standalone-toolchain/bin:$PATH + +## Build without Fortran for ARMV7 +make TARGET=ARMV7 ARM_SOFTFP_ABI=1 HOSTCC=gcc CC=arm-linux-androideabi-gcc NOFORTRAN=1 +## Build without Fortran for ARMV8 +make TARGET=ARMV8 BINARY=64 HOSTCC=gcc CC=aarch64-linux-android-gcc NOFORTRAN=1 +``` + +If you get an error about stdio.h not being found, you need to specify your sysroot in the CFLAGS argument to `make` like +```CFLAGS=--sysroot=$NDK/platforms/android-16/arch-arm``` +When you are done, install OpenBLAS into the desired directory. Be sure to also use all command line options +here that you specified for building, otherwise errors may occur as it tries to install things you did not build: +``` +make PREFIX=/path/to/install-dir TARGET=... install +``` + +##### Building OpenBLAS with Fortran + +Instructions on how to build the GNU toolchains with Fortran can be found [here](https://github.com/buffer51/android-gfortran). The [Releases section](https://github.com/buffer51/android-gfortran/releases) provides prebuilt versions, use the standalone one. + +You can build OpenBLAS with: +``` +## Add the toolchain to your path +export PATH=/path/to/standalone-toolchain-with-fortran/bin:$PATH + +## Build with Fortran for ARMV7 +make TARGET=ARMV7 HOSTCC=gcc CC=arm-linux-androideabi-gcc FC=arm-linux-androideabi-gfortran libs +## Build with LAPACK for ARMV8 +make TARGET=ARMV8 BINARY=64 HOSTCC=gcc CC=aarch64-linux-android-gcc FC=aarch64-linux-android-gfortran libs +``` + +As mentioned above you can leave off the `libs` argument here when building 0.2.20 and later, and you may want to add ARM_SOFTFP_ABI=1 when building for ARMV7. + +##### Linking OpenBLAS (0.2.19 and earlier) for ARMV7 + +If you are using `ndk-build`, you need to set the ABI to hard floating points in your Application.mk: +``` +APP_ABI := armeabi-v7a-hard ``` + +This will set the appropriate flags for you. If you are not using `ndk-build`, you will want to add the following flags: +``` +TARGET_CFLAGS += -mhard-float -D_NDK_MATH_NO_SOFTFP=1 +TARGET_LDFLAGS += -Wl,--no-warn-mismatch -lm_hard +``` + +From 0.2.20 on, it is also possible to build for the softfp ABI by specifying ARM_SOFTFP_ABI=1 during the build. +In that case, also make sure that all your dependencies are compiled with -mfloat-abi=softfp as well, as mixing +"hard" and "soft" floating point ABIs in a program will make it crash. + +### iPhone/iOS + +As none of the current developers uses iOS, the following instructions are what was found to work in our Azure CI setup, but as far as we know this builds a fully working OpenBLAS for this platform. + +Go to the directory where you unpacked OpenBLAS,and enter the following commands: +``` + CC=/Applications/Xcode_12.4.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang + +CFLAGS= -O2 -Wno-macro-redefined -isysroot /Applications/Xcode_12.4.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS14.4.sdk -arch arm64 -miphoneos-version-min=10.0 + +make TARGET=ARMV8 DYNAMIC_ARCH=1 NUM_THREADS=32 HOSTCC=clang NOFORTRAN=1 +``` +Adjust MIN_IOS_VERSION as necessary for your installation, e.g. change the version number +to the minimum iOS version you want to target and execute this file to build the library. + +### MIPS + +For mips targets you will need latest toolchains +P5600 - MTI GNU/Linux Toolchain +I6400, P6600 - IMG GNU/Linux Toolchain + +The download link is below +(http://codescape-mips-sdk.imgtec.com/components/toolchain/2016.05-03/downloads.html) + +You can use following commandlines for builds + + + IMG_TOOLCHAIN_DIR={full IMG GNU/Linux Toolchain path including "bin" directory -- for example, /opt/linux_toolchain/bin} + IMG_GCC_PREFIX=mips-img-linux-gnu + IMG_TOOLCHAIN=${IMG_TOOLCHAIN_DIR}/${IMG_GCC_PREFIX} + + I6400 Build (n32): + make BINARY=32 BINARY32=1 CC=$IMG_TOOLCHAIN-gcc AR=$IMG_TOOLCHAIN-ar FC="$IMG_TOOLCHAIN-gfortran -EL -mabi=n32" RANLIB=$IMG_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS=$CFLAGS LDFLAGS=$CFLAGS TARGET=I6400 + + I6400 Build (n64): + make BINARY=64 BINARY64=1 CC=$IMG_TOOLCHAIN-gcc AR=$IMG_TOOLCHAIN-ar FC="$IMG_TOOLCHAIN-gfortran -EL" RANLIB=$IMG_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS=$CFLAGS LDFLAGS=$CFLAGS TARGET=I6400 + + P6600 Build (n32): + make BINARY=32 BINARY32=1 CC=$IMG_TOOLCHAIN-gcc AR=$IMG_TOOLCHAIN-ar FC="$IMG_TOOLCHAIN-gfortran -EL -mabi=n32" RANLIB=$IMG_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS=$CFLAGS LDFLAGS=$CFLAGS TARGET=P6600 + + P6600 Build (n64): + make BINARY=64 BINARY64=1 CC=$IMG_TOOLCHAIN-gcc AR=$IMG_TOOLCHAIN-ar FC="$IMG_TOOLCHAIN-gfortran -EL" RANLIB=$IMG_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS="$CFLAGS" LDFLAGS="$CFLAGS" TARGET=P6600 + + MTI_TOOLCHAIN_DIR={full MTI GNU/Linux Toolchain path including "bin" directory -- for example, /opt/linux_toolchain/bin} + MTI_GCC_PREFIX=mips-mti-linux-gnu + MTI_TOOLCHAIN=${IMG_TOOLCHAIN_DIR}/${IMG_GCC_PREFIX} + + P5600 Build: + + make BINARY=32 BINARY32=1 CC=$MTI_TOOLCHAIN-gcc AR=$MTI_TOOLCHAIN-ar FC="$MTI_TOOLCHAIN-gfortran -EL" RANLIB=$MTI_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS=$CFLAGS LDFLAGS=$CFLAGS TARGET=P5600 + ### FreeBSD You will need to install the following tools from the FreeBSD ports tree: @@ -64,8 +656,46 @@ Note that you need to build with GNU make and manually specify the compiler, oth [1]: [Removal of Fortran from the FreeBSD base system](http://www.bsdunix.ch/serendipity/index.php?/archives/345-Removal-of-Fortran-from-the-FreeBSD-base-system.html) -### Android -See [this page](https://github.com/xianyi/OpenBLAS/wiki/How-to-build-OpenBLAS-for-Android) -### MIPS -See [this page](https://github.com/xianyi/OpenBLAS/wiki/Faq#mips) \ No newline at end of file +``` +pkg install openblas +``` + +see + +### Cortex-M + +Cortex-M is a widely used microcontroller that is present in a variety of industrial and consumer electronics. +A common variant of the Cortex-M is the STM32F4xx series. Here, we will give instructions for building for +the STM32F4xx. + +First, install the embedded arm gcc compiler from the arm website. Then, create the following toolchain file and build as follows. + +```cmake +# cmake .. -G Ninja -DCMAKE_C_COMPILER=arm-none-eabi-gcc -DCMAKE_TOOLCHAIN_FILE:PATH="toolchain.cmake" -DNOFORTRAN=1 -DTARGET=ARMV5 -DEMBEDDED=1 + +set(CMAKE_SYSTEM_NAME Generic) +set(CMAKE_SYSTEM_PROCESSOR arm) + +set(CMAKE_C_COMPILER "arm-none-eabi-gcc.exe") +set(CMAKE_CXX_COMPILER "arm-none-eabi-g++.exe") + +set(CMAKE_EXE_LINKER_FLAGS "--specs=nosys.specs" CACHE INTERNAL "") + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) +``` + +In your embedded application, the following functions need to be provided for OpenBLAS to work correctly: + +```C +void free(void* ptr); +void* malloc(size_t size); +``` + +!!! note + If you are developing for an embedded platform, it is your responsibility to make sure that the device has sufficient memory for malloc calls. [Libmemory][2] provides one implementation of malloc for embedded platforms. + +[2]: https://github.com/embeddedartistry/libmemory diff --git a/docs/logo.svg b/docs/logo.svg new file mode 100644 index 0000000000..e5a5df296b --- /dev/null +++ b/docs/logo.svg @@ -0,0 +1,450 @@ + + + + + + image/svg+xml + + OpenBLAS logo + + + + OpenBLAS logo + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | + + diff --git a/docs/mailing_list.md b/docs/mailing_list.md index 8de323e639..c3637a875f 100644 --- a/docs/mailing_list.md +++ b/docs/mailing_list.md @@ -1,15 +1,3 @@ -
-
- [Home] - [Document] - [FAQ] - [Publications] - [Download] -[Mailing List] -[Donation] -
-
- OpenBLAS users: https://groups.google.com/forum/#!forum/openblas-users -OpenBLAS developers: https://groups.google.com/forum/#!forum/openblas-dev \ No newline at end of file +OpenBLAS developers: https://groups.google.com/forum/#!forum/openblas-dev diff --git a/docs/packages.md b/docs/packages.md deleted file mode 100644 index d7796db14d..0000000000 --- a/docs/packages.md +++ /dev/null @@ -1,84 +0,0 @@ -# Precompiled installation packages -_note this list is not comprehensive, is not meant to validate nor endorse a particular third-party build over others and may not always lead to the newest version_ - -## Multiple operating systems -Binaries designed to be compatible with [Julia](https://github.com/JuliaLang/julia) are regularly provided in -https://github.com/staticfloat/OpenBLASBuilder/releases . Note that their 64bit builds with INTERFACE64=1 have -`_64` appended to the function symbol names, so your code needs to refer to e.g. gemm_64_ rather than gemm_ - -the [Conda-Forge](https://github.com/conda-forge) project maintains packages for the conda package manager at -https://github.com/conda-forge/openblas-feedstock -## FreeBSD - > pkg install openblas - - see https://www.freebsd.org/ports/index.html -## Linux -### Debian/Ubuntu/Mint/Kali - OpenBLAS package is available in default repositories and can act as default BLAS in system - -Example installation commands: -``` - $ sudo apt update - $ apt search openblas - $ sudo apt install libopenblas-dev - $ sudo update-alternatives --config libblas.so.3 -``` - Alternatively, if distributor's package proves unsatisfactory, you may try latest version of OpenBLAS, [Following guide in OpenBLAS FAQ](https://github.com/xianyi/OpenBLAS/wiki/faq#debianlts) - -### openSuSE/SLE: - Recent OpenSUSE versions include OpenBLAS in default repositories and also permit OpenBLAS to act as replacement of system-wide BLAS. - - Example installation commands: -``` - $ sudo zypper ref - $ zypper se openblas - $ sudo zypper in openblas-devel - $ sudo update-alternatives --config libblas.so.3 -``` -Should you be using older OpenSUSE or SLE that provides no OpenBLAS, you can attach optional or experimental openSUSE repository as a new package source to acquire recent build of OpenBLAS following [instructions on openSUSE software site](https://software.opensuse.org/package/openblas) - -### Fedora/CentOS/RHEL -Fedora provides OpenBLAS in default installation repositories. - -To install it try following: -``` - $ dnf search openblas - $ dnf install openblas-devel -``` -For CentOS/RHEL/Scientific Linux packages are provided via [Fedora EPEL repository](https://fedoraproject.org/wiki/EPEL) - -After adding repository and repository keys installation is pretty straightforward: -``` - $ yum search openblas - $ yum install openblas-devel -``` -No alternatives mechanism is provided for BLAS, and packages in system repositories are linked against NetLib BLAS or ATLAS BLAS libraries. You may wish to re-package RPMs to use OpenBLAS instead [as described here](https://fedoraproject.org/wiki/How_to_create_an_RPM_package) - -### Mageia -Mageia offers ATLAS and NetLIB LAPACK in base repositories. -You can build your own OpenBLAS replacement, and once installed in /opt -TODO: populate /usr/lib64 /usr/include accurately to replicate netlib with update-alternatives - -### Arch/Manjaro/Antergos -``` - $ sudo pacman -S openblas -``` - -## Solaris (via pkgsrc) - - -## OSX -https://www.macports.org/ports.php?by=name&substr=openblas - -`brew install openblas` - -or using the conda package manager from -https://github.com/conda-forge/miniforge#download -(which also has packages for the new M1 cpu) - - `conda install openblas` - -## Windows -http://sourceforge.net/projects/openblas/files - -https://www.nuget.org/packages?q=openblas diff --git a/docs/publications.md b/docs/publications.md index 4161fd4443..37889eec0e 100644 --- a/docs/publications.md +++ b/docs/publications.md @@ -1,19 +1,7 @@ -
-
- [Home] - [Document] - [FAQ] - [Publications] - [Download] -[Mailing List] -[Donation] -
-
- ### 2013 * Wang Qian, Zhang Xianyi, Zhang Yunquan, Qing Yi, **AUGEM: Automatically Generate High Performance Dense Linear Algebra Kernels on x86 CPUs**, In the International Conference for High Performance Computing, Networking, Storage and Analysis (SC'13), Denver CO, November 2013. [[pdf](http://xianyi.github.io/paper/augem_SC13.pdf)] ### 2012 -* Zhang Xianyi, Wang Qian, Zhang Yunquan, **Model-driven Level 3 BLAS Performance Optimization on Loongson 3A Processor**, 2012 IEEE 18th International Conference on Parallel and Distributed Systems (ICPADS), 17-19 Dec. 2012. \ No newline at end of file +* Zhang Xianyi, Wang Qian, Zhang Yunquan, **Model-driven Level 3 BLAS Performance Optimization on Loongson 3A Processor**, 2012 IEEE 18th International Conference on Parallel and Distributed Systems (ICPADS), 17-19 Dec. 2012. diff --git a/docs/todo.md b/docs/todo.md index 006f2cfa9f..080a250a84 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -1,14 +1,16 @@ +# TODO + Werner found a lot of bugs about lapack testing. In 0.2.9 version, we used a work-around,e.g. fallback to the old kernel, replacing the optimized kernel with reference kernel. We must fix them in future release. -* ~~Nehalem dgemm kernel. Fallback to core2 kernel.~~ -* ~~Sandy bridge sgemm kernel. Fallback to Nehalem kernels.~~ -* ~~Sandy bridge cgemm kernel. Fallback to Nehalem kernels.~~ +* Nehalem dgemm kernel. Fallback to core2 kernel. +* Sandy bridge sgemm kernel. Fallback to Nehalem kernels. +* Sandy bridge cgemm kernel. Fallback to Nehalem kernels. * sbmv, zsbmv, smp bug (interface/sbmv.c zsbmv.c). Now, it is only sequential. -* zhbmv, smp bug. -* ~~scal, zscal x86/x86_64 kernels. Fallback to C implementation.~~ -* ~~potri, zpotri. Fallback to LAPACK reference implementation.~~ -* ~~lauu2, zlauu2. Fallback to LAPACK reference implementation.~~ -* ~~trtri, ztrtri. Fallback to LAPACK reference implementation.~~ -* ~~lauum, zlauum. Fallback to LAPACK reference implementation.~~ -* ~~trti2, ztrti2. Fallback to LAPACK reference implementation.~~ -* Disable SMP in ger.c and zger.c \ No newline at end of file +* zhbmv, smp bug. +* scal, zscal x86/x86_64 kernels. Fallback to C implementation. +* potri, zpotri. Fallback to LAPACK reference implementation. +* lauu2, zlauu2. Fallback to LAPACK reference implementation. +* trtri, ztrtri. Fallback to LAPACK reference implementation. +* lauum, zlauum. Fallback to LAPACK reference implementation. +* trti2, ztrti2. Fallback to LAPACK reference implementation. +* Disable SMP in ger.c and zger.c diff --git a/docs/use_cortex.md b/docs/use_cortex.md deleted file mode 100644 index a90b1324f2..0000000000 --- a/docs/use_cortex.md +++ /dev/null @@ -1,37 +0,0 @@ -Cortex-M is a widely used microcontroller that is present in a variety of industrial and consumer electronics. -A common variant of the Cortex-M is the STM32F4xx series. Here, we will give instructions for building for -the STM32F4xx. - -First, install the embedded arm gcc compiler from the arm website. Then, create the following toolchain file and build as follows. - -```cmake -# cmake .. -G Ninja -DCMAKE_C_COMPILER=arm-none-eabi-gcc -DCMAKE_TOOLCHAIN_FILE:PATH="toolchain.cmake" -DNOFORTRAN=1 -DTARGET=ARMV5 -DEMBEDDED=1 - -set(CMAKE_SYSTEM_NAME Generic) -set(CMAKE_SYSTEM_PROCESSOR arm) - -set(CMAKE_C_COMPILER "arm-none-eabi-gcc.exe") -set(CMAKE_CXX_COMPILER "arm-none-eabi-g++.exe") - -set(CMAKE_EXE_LINKER_FLAGS "--specs=nosys.specs" CACHE INTERNAL "") - -set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) -set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) -``` - -In your embedded application, the following functions need to be provided for OpenBLAS to work correctly: - -```C -void free(void* ptr); -void* malloc(size_t size); -``` - -Note: if you are developing for an embedded platform, it is your responsibility to make sure that the device -has sufficient memory for malloc calls. [Libmemory][1] provides one implementation of malloc for embedded -platforms. - - -[1]: https://github.com/embeddedartistry/libmemory - diff --git a/docs/use_visual_studio.md b/docs/use_visual_studio.md deleted file mode 100644 index fe031383f6..0000000000 --- a/docs/use_visual_studio.md +++ /dev/null @@ -1,162 +0,0 @@ -As of OpenBLAS v0.2.15, we support MinGW and Visual Studio (using CMake to generate visual studio solution files – note that you will need at least version 3.11 of CMake for linking to work correctly) to build OpenBLAS on Windows. - -Note that you need a Fortran compiler if you plan to build and use the LAPACK functions included with OpenBLAS. The sections below describe using either `flang` as an add-on to clang/LLVM or `gfortran` as part of MinGW for this purpose. If you want to use the Intel Fortran compiler `ifort` for this, be sure to also use the Intel C compiler `icc` for building the C parts, as the ABI imposed by `ifort` is incompatible with `msvc`. - -## 1. Native (MSVC) ABI - -A fully-optimized OpenBLAS that can be statically or dynamically linked to your application can currently be built for the 64-bit architecture with the LLVM compiler infrastructure. We're going to use Miniconda3 to grab all of the tools we need, since some of them are in an experimental status. Before you begin, you'll need to have Microsoft Visual Studio 2015 or newer installed. - -1. Install Miniconda3 for 64 bits using `winget install --id Anaconda.Miniconda3` or easily download from [conda.io](https://docs.conda.io/en/latest/miniconda.html). -2. Open the "Anaconda Command Prompt," now available in the Start Menu, or at `%USERPROFILE%\miniconda3\shell\condabin\conda-hook.ps1`. -3. In that command prompt window, use `cd` to change to the directory where you want to build OpenBLAS -4. Now install all of the tools we need: - - ``` - conda update -n base conda - conda config --add channels conda-forge - conda install -y cmake flang clangdev perl libflang ninja - ``` - -5. Still in the Anaconda Command Prompt window, activate the MSVC environment for 64 bits with `vcvarsall x64`. On Windows 11 with Visual Studio 2022, this would be done by invoking: - - ```shell - "c:\Program Files\Microsoft Visual Studio\2022\Preview\vc\Auxiliary\Build\vcvars64.bat" - ``` - - With VS2019, the command should be the same – except for the year number, obviously. For other/older versions of MSVC, - the VS documentation or a quick search on the web should turn up the exact wording you need. - - Confirm that the environment is active by typing `link` – this should return a long list of possible options for the `link` command. If it just - returns "command not found" or similar, review and retype the call to vcvars64.bat. - **NOTE:** if you are working from a Visual Studio Command prompt window instead (so that you do not have to do the vcvars call), you need to invoke - `conda activate` so that CONDA_PREFIX etc. get set up correctly before proceeding to step 6. Failing to do so will lead to link errors like - libflangmain.lib not getting found later in the build. - -6. Now configure the project with CMake. Starting in the project directory, execute the following: - - ``` - set "LIB=%CONDA_PREFIX%\Library\lib;%LIB%" - set "CPATH=%CONDA_PREFIX%\Library\include;%CPATH%" - mkdir build - cd build - cmake .. -G "Ninja" -DCMAKE_CXX_COMPILER=clang-cl -DCMAKE_C_COMPILER=clang-cl -DCMAKE_Fortran_COMPILER=flang -DCMAKE_MT=mt -DBUILD_WITHOUT_LAPACK=no -DNOFORTRAN=0 -DDYNAMIC_ARCH=ON -DCMAKE_BUILD_TYPE=Release - ``` - - You may want to add further options in the `cmake` command here – for instance, the default only produces a static .lib version of the library. If you would rather have a DLL, add -DBUILD_SHARED_LIBS=ON above. Note that this step only creates some command files and directories, the actual build happens next. - - -7. Build the project: - - ``` - cmake --build . --config Release - ``` - This step will create the OpenBLAS library in the "lib" directory, and various build-time tests in the `test`, `ctest` and `openblas_utest` directories. However it will not separate the header files you might need for building your own programs from those used internally. To put all relevant files in a more convenient arrangement, run the next step. - -8. Install all relevant files created by the build - - ``` - cmake --install . --prefix c:\opt -v - ``` - This will copy all files that are needed for building and running your own programs with OpenBLAS to the given location, creating appropriate subdirectories for the individual kinds of files. In the case of "C:\opt" as given above, this would be C:\opt\include\openblas for the header files, - C:\opt\bin for the libopenblas.dll and C:\opt\lib for the static library. C:\opt\share holds various support files that enable other cmake-based build scripts to find OpenBLAS automatically. - -### Visual studio 2017+ (C++2017 standard) - -In newer visual studio versions, Microsoft has changed [how it handles complex types](https://docs.microsoft.com/en-us/cpp/c-runtime-library/complex-math-support?view=msvc-170#types-used-in-complex-math). Even when using a precompiled version of OpenBLAS, you might need to define `LAPACK_COMPLEX_CUSTOM` in order to define complex types properly for MSVC. For example, some variant of the following might help: - -``` -#if defined(_MSC_VER) - #include - #define LAPACK_COMPLEX_CUSTOM - #define lapack_complex_float _Fcomplex - #define lapack_complex_double _Dcomplex -#endif -``` - -For reference, see https://github.com/xianyi/OpenBLAS/issues/3661, https://github.com/Reference-LAPACK/lapack/issues/683, and https://stackoverflow.com/questions/47520244/using-openblas-lapacke-in-visual-studio. - -### CMake and Visual Studio - -To build OpenBLAS for the 32-bit architecture, you'll need to use the builtin Visual Studio compilers. - -**[Notice]** This method may produce binaries which demonstrate significantly lower performance than those built with the other methods. (The Visual Studio compiler does not support the dialect of assembly used in the cpu-specific optimized files, so only the "generic" TARGET which is -written in pure C will get built. For the same reason it is not possible (and not necessary) to use -DDYNAMIC_ARCH=ON in a Visual Studio build) You may consider building for the 32-bit architecture using the GNU (MinGW) ABI. - -#### 1. Install CMake at Windows - -#### 2. Use CMake to generate Visual Studio solution files - -``` -# Do this from Powershell so cmake can find visual studio -cmake -G "Visual Studio 14 Win64" -DCMAKE_BUILD_TYPE=Release . -``` - -### Build the solution at Visual Studio - -Note that this step depends on perl, so you'll need to install perl for windows, and put perl on your path so VS can start perl (http://stackoverflow.com/questions/3051049/active-perl-installation-on-windows-operating-system). - -Step 2 will build the OpenBLAS solution, open it in VS, and build the projects. Note that the dependencies do not seem to be automatically configured: if you try to build libopenblas directly, it will fail with a message saying that some .obj files aren't found, but if you build the projects libopenblas depends on before building libopenblas, the build will succeed. - -### Build OpenBLAS for Universal Windows Platform - -OpenBLAS can be built for use on the [Universal Windows Platform](https://en.wikipedia.org/wiki/Universal_Windows_Platform) using a two step process since commit [c66b842](https://github.com/xianyi/OpenBLAS/commit/c66b842d66c5516e52804bf5a0544d18b1da1b44). - -#### 1. Follow steps 1 and 2 above to build the Visual Studio solution files for Windows. This builds the helper executables which are required when building the OpenBLAS Visual Studio solution files for UWP in step 2. - -#### 2. Remove the generated CMakeCache.txt and CMakeFiles directory from the OpenBLAS source directory and re-run CMake with the following options: - -``` -# do this to build UWP compatible solution files -cmake -G "Visual Studio 14 Win64" -DCMAKE_SYSTEM_NAME=WindowsStore -DCMAKE_SYSTEM_VERSION="10.0" -DCMAKE_SYSTEM_PROCESSOR=AMD64 -DVS_WINRT_COMPONENT=TRUE -DCMAKE_BUILD_TYPE=Release . -``` - -#### Build the solution with Visual Studio - -This will build the OpenBLAS binaries with the required settings for use with UWP. - -## 2. GNU (MinGW) ABI - -The resulting library can be used in Visual Studio, but it can only be linked dynamically. This configuration has not been thoroughly tested and should be considered experimental. - -### Incompatible x86 calling conventions - -Due to incompatibilities between the calling conventions of MinGW and Visual Studio you will need to make the following modifications ( **32-bit only** ): - -1. Use the newer GCC 4.7.0. The older GCC (<4.7.0) has an ABI incompatibility for returning aggregate structures larger than 8 bytes with MSVC. - - -### Build OpenBLAS on Windows OS -1. Install the MinGW (GCC) compiler suite, either 32-bit (http://www.mingw.org/) or 64-bit (http://mingw-w64.sourceforge.net/). Be sure to install its gfortran package as well (unless you really want to build the BLAS part of OpenBLAS only) and check that gcc and gfortran are the same version – mixing compilers from different sources or release versions can lead to strange error messages in the linking stage. In addition, please install MSYS with MinGW. -1. Build OpenBLAS in the MSYS shell. Usually, you can just type "make". OpenBLAS will detect the compiler and CPU automatically. -1. After the build is complete, OpenBLAS will generate the static library "libopenblas.a" and the shared dll library "libopenblas.dll" in the folder. You can type "make PREFIX=/your/installation/path install" to install the library to a certain location. - -**[Notice]** We suggest using official MinGW or MinGW-w64 compilers. A user reported that s/he met `Unhandled exception` by other compiler suite. https://groups.google.com/forum/#!topic/openblas-users/me2S4LkE55w - -Note also that older versions of the alternative builds of mingw-w64 available through http://www.msys2.org may contain a defect that leads to a compilation failure accompanied by the error message -``` -:0:4: error: expected identifier or '(' before numeric constant -``` -If you encounter this, please upgrade your msys2 setup or see https://github.com/xianyi/OpenBLAS/issues/1503 for a workaround. - -### Generate import library (before 0.2.10 version) - -1. First, you will need to have the `lib.exe` tool in the Visual Studio command prompt. -1. Open the command prompt and type `cd OPENBLAS_TOP_DIR/exports`, where OPENBLAS_TOP_DIR is the main folder of your OpenBLAS installation. -1. For a 32-bit library, type `lib /machine:i386 /def:libopenblas.def`. For 64-bit, type `lib /machine:X64 /def:libopenblas.def`. -1. This will generate the import library "libopenblas.lib" and the export library "libopenblas.exp" in OPENBLAS_TOP_DIR/exports. Although these two files have the same name, they are totally different. - -### Generate import library (0.2.10 and after version) -1. OpenBLAS already generated the import library "libopenblas.dll.a" for "libopenblas.dll". - -### generate windows native PDB files from gcc/gfortran build -Tool to do so is available at https://github.com/rainers/cv2pdb - -### Use OpenBLAS .dll library in Visual Studio -1. Copy the import library (before 0.2.10: "OPENBLAS_TOP_DIR/exports/libopenblas.lib", 0.2.10 and after: "OPENBLAS_TOP_DIR/libopenblas.dll.a") and .dll library "libopenblas.dll" into the same folder(The folder of your project that is going to use the BLAS library. You may need to add the libopenblas.dll.a to the linker input list: properties->Linker->Input). -1. Please follow the documentation about using third-party .dll libraries in MS Visual Studio 2008 or 2010. Make sure to link against a library for the correct architecture. For example, you may receive an error such as "The application was unable to start correctly (0xc000007b)" which typically indicates a mismatch between 32/64-bit libraries. - -**[Notice]** If you need CBLAS, you should include cblas.h in /your/installation/path/include in Visual Studio. Please read [this page](http://github.com/xianyi/OpenBLAS/issues/95). - -### Limitations -* Both static and dynamic linking are supported with MinGW. With Visual Studio, however, only dynamic linking is supported and so you should use the import library. -* Debugging from Visual Studio does not work because MinGW and Visual Studio have incompatible formats for debug information (PDB vs. DWARF/STABS). You should either debug with GDB on the command-line or with a visual frontend, for instance [Eclipse](http://www.eclipse.org/cdt/) or [Qt Creator](http://qt.nokia.com/products/developer-tools/). diff --git a/docs/user_manual.md b/docs/user_manual.md index 2efb0f16d8..b5cd632f15 100644 --- a/docs/user_manual.md +++ b/docs/user_manual.md @@ -1,11 +1,3 @@ -##### Table of Contents -[Compile the library](#compile-the-library) -[Link the library](#link-the-library) -[Code examples](#code-examples) -[Troubleshooting](#troubleshooting) -[BLAS reference manual](#blas-reference-manual) - - ## Compile the library ### Normal compile * type `make` to detect the CPU automatically. @@ -47,7 +39,7 @@ Example: The default directory is /opt/OpenBLAS. Note that any flags passed to `make` during build should also be passed to `make install` to circumvent any install errors, i.e. some headers not being copied over correctly. -For more information, please read [Installation Guide](Installation-Guide). +For more information, please read [Installation Guide](install.md). ## Link the library @@ -173,7 +165,7 @@ gcc -o time_dgemm time_dgemm.c /your/path/libopenblas.a ## Troubleshooting -* Please read [Faq](https://github.com/xianyi/OpenBLAS/wiki/Faq) at first. +* Please read [Faq](faq.md) at first. * Please use gcc version 4.6 and above to compile Sandy Bridge AVX kernels on Linux/MingW/BSD. * Please use Clang version 3.1 and above to compile the library on Sandy Bridge microarchitecture. The Clang 3.0 will generate the wrong AVX binary code. * The number of CPUs/Cores should less than or equal to 256. On Linux x86_64(amd64), there is experimental support for up to 1024 CPUs/Cores and 128 numa nodes if you build the library with BIGNUMA=1. @@ -184,4 +176,4 @@ gcc -o time_dgemm time_dgemm.c /your/path/libopenblas.a If you want to understand every BLAS function and definition, please read [Intel MKL reference manual](https://software.intel.com/en-us/intel-mkl/documentation) or [netlib.org](http://netlib.org/blas/) -Here are [OpenBLAS extension functions](OpenBLAS-Extensions) \ No newline at end of file +Here are [OpenBLAS extension functions](extensions.md) diff --git a/mkdocs.yml b/mkdocs.yml index fa486d208f..8c20365ba6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,3 +1,29 @@ site_name: OpenBLAS theme: name: material + logo: logo.svg + favicon: logo.svg + palette: + primary: grey +markdown_extensions: + - admonition + - pymdownx.details + - pymdownx.superfences + - toc: + toc_depth: 4 +nav: + - index.md + - install.md + - user_manual.md + - extensions.md + - developers.md + - build_system.md + - distributing.md + - ci.md + - machine_list.md + - todo.md + - ecosystem.md + - publications.md + - mailing_list.md + - donate.md + - faq.md From 27841b8b447ae9ebd8dd39ec13cd0b868c8f2cce Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Tue, 12 Mar 2024 14:58:32 +0000 Subject: [PATCH 007/267] More doc improvements * Fold "Users of OpenBLAS", "Publications", "Mailing list", "Donation" pages into "About" * Remove full license * Remove outdated TODO and machine list pages * Shorter titles for CI and distributing pages * Link to GotoBLAS readme in repo * Brief description of OpenBLAS in homepage * Move acknowledgements into About page * Remove manual TOC in FAQ docs --- docs/about.md | 49 +++++++++++++++++++++++++++++++++++++++++ docs/ci.md | 2 +- docs/distributing.md | 2 +- docs/donate.md | 19 ---------------- docs/ecosystem.md | 15 ------------- docs/faq.md | 52 +++++--------------------------------------- docs/index.md | 38 ++++---------------------------- docs/install.md | 20 ++++++++--------- docs/machine_list.md | 9 -------- docs/mailing_list.md | 3 --- docs/publications.md | 7 ------ docs/todo.md | 16 -------------- mkdocs.yml | 7 +----- 13 files changed, 71 insertions(+), 168 deletions(-) create mode 100644 docs/about.md delete mode 100644 docs/donate.md delete mode 100644 docs/ecosystem.md delete mode 100644 docs/machine_list.md delete mode 100644 docs/mailing_list.md delete mode 100644 docs/publications.md delete mode 100644 docs/todo.md diff --git a/docs/about.md b/docs/about.md new file mode 100644 index 0000000000..f8bdb9e647 --- /dev/null +++ b/docs/about.md @@ -0,0 +1,49 @@ +## Mailing list + +OpenBLAS users: + +OpenBLAS developers: + +## Donations + +Thank you for the support. + +You can read OpenBLAS statement of receipts and disbursement and cash balance on [google doc](https://docs.google.com/spreadsheet/ccc?key=0AghkTjXe2lDndE1UZml0dGpaUzJmZGhvenBZd1F2R1E&usp=sharing). + +### Fundraiser + +* [2013.8] [Testbed for OpenBLAS project](https://www.bountysource.com/fundraisers/443-testbed-for-openblas-project) + +Here is [Backer list](https://github.com/xianyi/OpenBLAS/blob/develop/BACKERS.md). + +### For Personal Donation + +You can create a bounty for an issue on [bountysource.com](https://www.bountysource.com/trackers/69691-openblas). + +### For Hardware Vendors + +We welcome the hardware donation, including the latest CPU and boards. + +## Acknowledgements + +This work is partially supported by +* Research and Development of Compiler System and Toolchain for Domestic CPU, National S&T Major Projects: Core Electronic Devices, High-end General Chips and Fundamental Software (No.2009ZX01036-001-002) +* National High-tech R&D Program of China (Grant No.2012AA010903) + +## Users of OpenBLAS + +* Julia - a high-level, high-performance dynamic programming language for technical computing
+* Ceemple v1.0.3 (C++ technical computing environment), including OpenBLAS, Qt, Boost, OpenCV and others. The only solution with immediate-recompilation of C++ code. Available from Ceemple C++ Technical Computing. +* [netlib-java](https://github.com/fommil/netlib-java) and various upstream libraries, allowing OpenBLAS to be used from languages on the Java Virtual Machine. + + + +## Publications + +### 2013 + +* Wang Qian, Zhang Xianyi, Zhang Yunquan, Qing Yi, **AUGEM: Automatically Generate High Performance Dense Linear Algebra Kernels on x86 CPUs**, In the International Conference for High Performance Computing, Networking, Storage and Analysis (SC'13), Denver CO, November 2013. [[pdf](http://xianyi.github.io/paper/augem_SC13.pdf)] + +### 2012 + +* Zhang Xianyi, Wang Qian, Zhang Yunquan, **Model-driven Level 3 BLAS Performance Optimization on Loongson 3A Processor**, 2012 IEEE 18th International Conference on Parallel and Distributed Systems (ICPADS), 17-19 Dec. 2012. diff --git a/docs/ci.md b/docs/ci.md index e36fb633bc..6d6112cb26 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -1,4 +1,4 @@ -# CI jobs overview +# CI jobs | Arch|Target CPU|OS|Build system|XComp to|C Compiler|Fortran Compiler|threading|DYN_ARCH|INT64|Libraries| CI Provider| CPU count| | ------------|---|---|-----------|-------------|----------|----------------|------|------------|----------|-----------|----------|-------| diff --git a/docs/distributing.md b/docs/distributing.md index 2f3828d08a..ff481b048d 100644 --- a/docs/distributing.md +++ b/docs/distributing.md @@ -1,4 +1,4 @@ -# Guidance for redistributing OpenBLAS +# Redistributing OpenBLAS !!! note This document contains recommendations only - packagers and other diff --git a/docs/donate.md b/docs/donate.md deleted file mode 100644 index 94d73bab53..0000000000 --- a/docs/donate.md +++ /dev/null @@ -1,19 +0,0 @@ -# Donations - -Thank you for the support. - -You can read OpenBLAS statement of receipts and disbursement and cash balance on [google doc](https://docs.google.com/spreadsheet/ccc?key=0AghkTjXe2lDndE1UZml0dGpaUzJmZGhvenBZd1F2R1E&usp=sharing). - -## Fundraiser - -* [2013.8] [Testbed for OpenBLAS project](https://www.bountysource.com/fundraisers/443-testbed-for-openblas-project) - -Here is [Backer list](https://github.com/xianyi/OpenBLAS/blob/develop/BACKERS.md). - -## For Personal Donation - -You can create a bounty for an issue on [bountysource.com](https://www.bountysource.com/trackers/69691-openblas). - -## For Hardware Vendors - -We welcome the hardware donation, including the latest CPU and boards. diff --git a/docs/ecosystem.md b/docs/ecosystem.md deleted file mode 100644 index ae2ae543b5..0000000000 --- a/docs/ecosystem.md +++ /dev/null @@ -1,15 +0,0 @@ -# Users of OpenBLAS - -## Softwares - -* Julia - a high-level, high-performance dynamic programming language for technical computing
-* Ceemple v1.0.3 (C++ technical computing environment), including OpenBLAS, Qt, Boost, OpenCV and others. The only solution with immediate-recompilation of C++ code. Available from Ceemple C++ Technical Computing. -* [netlib-java](https://github.com/fommil/netlib-java) and various upstream libraries, allowing OpenBLAS to be used from languages on the Java Virtual Machine. - -## Academia Users - -## Industry Users - -## HPC Centers Deployed OpenBLAS - -## Others diff --git a/docs/faq.md b/docs/faq.md index bbcbb36e4f..699042d512 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -1,50 +1,10 @@ -# FAQ +--- +title: FAQ +--- -## General questions -+ **[What is BLAS? Why is it important?](#whatblas)** -+ **[What functions are there and how can I call them from my C code?](#whatsinblas)** -+ **[What is OpenBLAS? Why did you create this project?](#what)** -+ **[What's the difference between OpenBLAS and GotoBLAS?](#gotoblas)** -+ **[Where do parameters GEMM_P, GEMM_Q, GEMM_R come from?](#gemmpqr)** -+ **[How can I report a bug?](#reportbug)** -+ **[How to reference OpenBLAS.](#publication)** -+ **[How can I use OpenBLAS in multi-threaded applications?](#multi-threaded)** -+ **[Does OpenBLAS support sparse matrices and/or vectors ?](#sparse)** -+ **[What support is there for recent PC hardware ? What about GPU ?](#recent_hardware)** -+ **[How about the level 3 BLAS performance on Intel Sandy Bridge?](#sandybridge_perf)** + -## OS and Compiler -+ **[How can I call an OpenBLAS function in Microsoft Visual Studio?](#MSVC)** -+ **[How can I use CBLAS and LAPACKE without C99 complex number support (e.g. in Visual Studio)?](#C99_complex_number)** -+ **[I get a SEGFAULT with multi-threading on Linux. What's wrong?](#Linux_SEGFAULT)** -+ **[When I make the library, there is no such instruction: `xgetbv' error. What's wrong?](#xgetbv)** -+ **[My build fails due to the linker error "multiple definition of `dlamc3_'". What is the problem?](#patch_missing)** -+ **[My build worked fine and passed all tests, but running `make lapack-test` ends with segfaults](#xeigtst)** -+ **[How could I disable OpenBLAS threading affinity on runtime?](#no_affinity)** -+ **[How to solve undefined reference errors when statically linking against libopenblas.a](#static_link)** -+ **[Building OpenBLAS for Haswell or Dynamic Arch on RHEL-6, CentOS-6, Rocks-6.1](#binutils)** -+ **[Building OpenBLAS in QEMU/KVM/XEN](#qemu)** -+ **[Building OpenBLAS on POWER fails with the IBM XL compiler](#ppcxl)** -+ **[Replacing system BLAS in Ubuntu/Debian](#debianlts)** -+ **[I built OpenBLAS for use with some other software, but that software cannot find it](#findblas)** -+ **[I included cblas.h in my program, but the compiler complains about a missing common.h or functions from it](#installincludes)** -+ **[Compiling OpenBLAS with gcc's -fbounds-check actually triggers aborts in programs](#boundscheck)** -+ **[Build fails with lots of errors about undefined ?GEMM_UNROLL_M](#newcpu)** -+ **[CMAKE/OSX: Build fails with 'argument list too long'](#cmakeosx)** -+ **[Likely problems with AVX2 support in Docker Desktop for OSX](#xhyve)** -## Usage -+ **[Program is Terminated. Because you tried to allocate too many memory regions](#allocmorebuffers)** -+ **[How to choose TARGET manually at runtime when compiled with DYNAMIC_ARCH](#choose_target_dynamic)** -+ **[After updating the installed OpenBLAS, a program complains about "undefined symbol gotoblas"](#missgoto)** -+ **[How can I find out at runtime what options the library was built wih ?](#buildoptions)** -+ **[After making OpenBLAS, I find that the static library is multithreaded, but the dynamic one is not ?](#wronglibrary)** -+ **[I want to use OpenBLAS with CUDA in the HPL 2.3 benchmark code but it keeps looking for Intel MKL](#cudahpl)** -+ **[Multithreaded OpenBLAS runs no faster or is even slower than singlethreaded on my ARMV7 board](#cpusoffline)** -+ **[Speed varies wildly between individual runs on a typical ARMV8 smartphone processor](#biglittle)** -+ **[I cannot get OpenBLAS to use more than a small subset of available cores on a big system](#numthreads)** -+ **[Getting "ELF load command address/offset not properly aligned" when loading libopenblas.so](#ELFoffset)** -+ **[Using OpenBLAS with OpenMP](#OpenMP)** -
+[TOC] ## General questions @@ -97,7 +57,7 @@ Please provide the following information: CPU, OS, compiler, and OpenBLAS compil ### How to reference OpenBLAS. -You can reference our papers in [this page](publications.md). Alternatively, you can cite the OpenBLAS homepage http://www.openblas.net. +You can reference our papers in [this page](about.md#publications). Alternatively, you can cite the OpenBLAS homepage http://www.openblas.net. ### How can I use OpenBLAS in multi-threaded applications? diff --git a/docs/index.md b/docs/index.md index 279cfe57de..a8866e946f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,40 +2,10 @@ OpenBLAS is an optimized Basic Linear Algebra Subprograms (BLAS) library based on [GotoBLAS2](https://www.tacc.utexas.edu/research-development/tacc-software/gotoblas2) 1.13 BSD version. -## License - -OpenBLAS is licensed under the 3-clause BSD license. Full license text follows: - -Copyright (c) 2011-2015, The OpenBLAS Project -All rights reserved. +OpenBLAS implements low-level routines for performing linear algebra operations such as vector addition, scalar multiplication, dot products, linear combinations, and matrix multiplicationon. OpenBLAS makes these routines available on multiple popular platforms, covering desktop (Windows, Linux, Mac) and mobile (Android, iOS) operating systems, as well as different architectures (namely x86 and ARM). -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: +The old GotoBLAS documentation can be found [on GitHub](https://github.com/OpenMathLib/OpenBLAS/blob/develop/GotoBLAS_01Readme.txt). - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - 3. Neither the name of the OpenBLAS project nor the names of - its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -## Acknowledgements +## License -This work is partially supported by -* Research and Development of Compiler System and Toolchain for Domestic CPU, National S&T Major Projects: Core Electronic Devices, High-end General Chips and Fundamental Software (No.2009ZX01036-001-002) -* National High-tech R&D Program of China (Grant No.2012AA010903) +OpenBLAS is licensed under the 3-clause BSD license. The full license can be found [on GitHub](https://github.com/OpenMathLib/OpenBLAS/blob/develop/LICENSE). diff --git a/docs/install.md b/docs/install.md index e3a19db5b4..dff18e1507 100644 --- a/docs/install.md +++ b/docs/install.md @@ -83,7 +83,7 @@ $ sudo pacman -S openblas ### Windows The precompiled binaries available with each release (in ) are -created with MinGW (as described in Section 2 of the wiki page mentioned above) using an option list of +created with MinGW using an option list of "NUM_THREADS=64 TARGET=GENERIC DYNAMIC_ARCH=1 DYNAMIC_OLDER=1 CONSISTENT_FPCSR=1" - they should work on any x86 or x86_64 computer. The zip archive contains the include files, static and dll libraries as well as configuration files for getting them found via CMAKE or pkgconfig - just create a suitable folder for @@ -397,21 +397,19 @@ On newer versions of Xcode and on arm64, you might need to compile with a newer ### Android -#### Build - -##### Prerequisites +#### Prerequisites In addition to the Android NDK, you will need both Perl and a C compiler on the build host as these are currently required by the OpenBLAS build environment. -##### Building with android NDK using clang compiler +#### Building with android NDK using clang compiler Around version 11 Android NDKs stopped supporting gcc, so you would need to use clang to compile OpenBLAS. clang is supported from OpenBLAS 0.2.20 version onwards. See below sections on how to build with clang for ARMV7 and ARMV8 targets. The same basic principles as described below for ARMV8 should also apply to building an x86 or x86_64 version (substitute something like NEHALEM for the target instead of ARMV8 and replace all the aarch64 in the toolchain paths obviously) "Historic" notes: Since version 19 the default toolchain is provided as a standalone toolchain, so building one yourself following [building a standalone toolchain](http://developer.android.com/ndk/guides/standalone_toolchain.html) should no longer be necessary. If you want to use static linking with an old NDK version older than about r17, you need to choose an API level below 23 currently due to NDK bug 272 (https://github.com/android-ndk/ndk/issues/272 , the libc.a lacks a definition of stderr) that will probably be fixed in r17 of the NDK. -##### Build ARMV7 with clang +#### Build ARMV7 with clang ``` ## Set path to ndk-bundle export NDK_BUNDLE_DIR=/path/to/ndk-bundle @@ -434,7 +432,7 @@ AR=${NDK_BUNDLE_DIR}/toolchains/arm-linux-androideabi-4.9/prebuilt/darwin-x86_64 ``` otherwise you may get a linker error complaining about a "malformed archive header name at 8" when the native OSX ar command was invoked instead. -##### Build ARMV8 with clang +#### Build ARMV8 with clang ``` ## Set path to ndk-bundle export NDK_BUNDLE_DIR=/path/to/ndk-bundle/ @@ -460,7 +458,7 @@ make HOSTCC=gcc CC=/opt/android-ndk-r23b/toolchains/llvm/prebuilt/linux-x86_64/b ``` appears to be sufficient on Linux. -##### Alternative script which was tested on OSX with NDK(21.3.6528147) +#### Alternative script which was tested on OSX with NDK(21.3.6528147) This script will build openblas for 3 architecture (ARMV7,ARMV8,X86) and put them with `sudo make install` to `/opt/OpenBLAS/lib` ``` export NDK=YOUR_PATH_TO_SDK/Android/sdk/ndk/21.3.6528147 @@ -515,7 +513,7 @@ Also you can find full list of target architectures in [TargetsList.txt](https:/ *** anything below this line should be irrelevant nowadays unless you need to perform software archeology *** -##### Building OpenBLAS with very old gcc-based versions of the NDK, without Fortran +#### Building OpenBLAS with very old gcc-based versions of the NDK, without Fortran The prebuilt Android NDK toolchains do not include Fortran, hence parts like LAPACK cannot be built. You can still build OpenBLAS without it. For instructions on how to build OpenBLAS with Fortran, see the [next section](#building-openblas-with-fortran). @@ -554,7 +552,7 @@ here that you specified for building, otherwise errors may occur as it tries to make PREFIX=/path/to/install-dir TARGET=... install ``` -##### Building OpenBLAS with Fortran +#### Building OpenBLAS with Fortran Instructions on how to build the GNU toolchains with Fortran can be found [here](https://github.com/buffer51/android-gfortran). The [Releases section](https://github.com/buffer51/android-gfortran/releases) provides prebuilt versions, use the standalone one. @@ -571,7 +569,7 @@ make TARGET=ARMV8 BINARY=64 HOSTCC=gcc CC=aarch64-linux-android-gcc FC=aarch64-l As mentioned above you can leave off the `libs` argument here when building 0.2.20 and later, and you may want to add ARM_SOFTFP_ABI=1 when building for ARMV7. -##### Linking OpenBLAS (0.2.19 and earlier) for ARMV7 +#### Linking OpenBLAS (0.2.19 and earlier) for ARMV7 If you are using `ndk-build`, you need to set the ABI to hard floating points in your Application.mk: ``` diff --git a/docs/machine_list.md b/docs/machine_list.md deleted file mode 100644 index 2039b3a8fa..0000000000 --- a/docs/machine_list.md +++ /dev/null @@ -1,9 +0,0 @@ -We deployed the following machine to support developing OpenBLAS. The interested developer can apply the remote access by sending the mail to [OpenBLAS-dev mailing list](https://groups.google.com/forum/#!forum/openblas-dev). - -* 2-way Intel Xeon E5-2680 (2.7GHz, Intel Sandy Bridge), 32GB memory, 1TB hard disk, Ubuntu 14.04 64-bit. -* 1-way Intel Core i7-4770 (3.4GHz, Intel Haswell), 8GB memory, 512GB hard disk, Ubuntu 14.04 64-bit. -* 1-way AMD FX-8320 (3.5GHz, AMD Piledriver), 8GB memory, 512GB hard disk, Ubuntu 14.04 64-bit. -* [Odroid-U2 board](http://www.hardkernel.com/main/products/prdt_info.php?g_code=G135341370451). 1-way Exynos4412 Prime 1.7Ghz ARM Cortex-A9 Quad Cores, 2GB memory, 16GB SD card, Ubuntu 14.04 32-bit. -* AMD A10-7850K APU (Kaveri, Steamroller+GCN GPU), 8GB memory, 1TB hard disk, Ubuntu 14.04.1 64-bit -* NVIDIA TK1 (ARM Cortex-A15), ARM 32-bit, Ubuntu -* NVIDIA TX1 (ARM Cortex-A57), AArch64, Ubuntu \ No newline at end of file diff --git a/docs/mailing_list.md b/docs/mailing_list.md deleted file mode 100644 index c3637a875f..0000000000 --- a/docs/mailing_list.md +++ /dev/null @@ -1,3 +0,0 @@ -OpenBLAS users: https://groups.google.com/forum/#!forum/openblas-users - -OpenBLAS developers: https://groups.google.com/forum/#!forum/openblas-dev diff --git a/docs/publications.md b/docs/publications.md deleted file mode 100644 index 37889eec0e..0000000000 --- a/docs/publications.md +++ /dev/null @@ -1,7 +0,0 @@ -### 2013 - -* Wang Qian, Zhang Xianyi, Zhang Yunquan, Qing Yi, **AUGEM: Automatically Generate High Performance Dense Linear Algebra Kernels on x86 CPUs**, In the International Conference for High Performance Computing, Networking, Storage and Analysis (SC'13), Denver CO, November 2013. [[pdf](http://xianyi.github.io/paper/augem_SC13.pdf)] - -### 2012 - -* Zhang Xianyi, Wang Qian, Zhang Yunquan, **Model-driven Level 3 BLAS Performance Optimization on Loongson 3A Processor**, 2012 IEEE 18th International Conference on Parallel and Distributed Systems (ICPADS), 17-19 Dec. 2012. diff --git a/docs/todo.md b/docs/todo.md deleted file mode 100644 index 080a250a84..0000000000 --- a/docs/todo.md +++ /dev/null @@ -1,16 +0,0 @@ -# TODO - -Werner found a lot of bugs about lapack testing. In 0.2.9 version, we used a work-around,e.g. fallback to the old kernel, replacing the optimized kernel with reference kernel. We must fix them in future release. - -* Nehalem dgemm kernel. Fallback to core2 kernel. -* Sandy bridge sgemm kernel. Fallback to Nehalem kernels. -* Sandy bridge cgemm kernel. Fallback to Nehalem kernels. -* sbmv, zsbmv, smp bug (interface/sbmv.c zsbmv.c). Now, it is only sequential. -* zhbmv, smp bug. -* scal, zscal x86/x86_64 kernels. Fallback to C implementation. -* potri, zpotri. Fallback to LAPACK reference implementation. -* lauu2, zlauu2. Fallback to LAPACK reference implementation. -* trtri, ztrtri. Fallback to LAPACK reference implementation. -* lauum, zlauum. Fallback to LAPACK reference implementation. -* trti2, ztrti2. Fallback to LAPACK reference implementation. -* Disable SMP in ger.c and zger.c diff --git a/mkdocs.yml b/mkdocs.yml index 8c20365ba6..cd23bd8924 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -20,10 +20,5 @@ nav: - build_system.md - distributing.md - ci.md - - machine_list.md - - todo.md - - ecosystem.md - - publications.md - - mailing_list.md - - donate.md + - about.md - faq.md From 7a8b1d40a28aefc4db88359e7b84cea498e904a4 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Wed, 13 Mar 2024 12:58:50 +0000 Subject: [PATCH 008/267] Address review comments * Update backers link to OpenBLAS repo * Remove reference to Bountysource --- docs/about.md | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/docs/about.md b/docs/about.md index f8bdb9e647..686d8cf617 100644 --- a/docs/about.md +++ b/docs/about.md @@ -6,21 +6,7 @@ OpenBLAS developers: ## Donations -Thank you for the support. - -You can read OpenBLAS statement of receipts and disbursement and cash balance on [google doc](https://docs.google.com/spreadsheet/ccc?key=0AghkTjXe2lDndE1UZml0dGpaUzJmZGhvenBZd1F2R1E&usp=sharing). - -### Fundraiser - -* [2013.8] [Testbed for OpenBLAS project](https://www.bountysource.com/fundraisers/443-testbed-for-openblas-project) - -Here is [Backer list](https://github.com/xianyi/OpenBLAS/blob/develop/BACKERS.md). - -### For Personal Donation - -You can create a bounty for an issue on [bountysource.com](https://www.bountysource.com/trackers/69691-openblas). - -### For Hardware Vendors +You can read OpenBLAS statement of receipts and disbursement and cash balance on [google doc](https://docs.google.com/spreadsheet/ccc?key=0AghkTjXe2lDndE1UZml0dGpaUzJmZGhvenBZd1F2R1E&usp=sharing). A backer list is available [on GitHub](https://github.com/OpenMathLib/OpenBLAS/blob/develop/BACKERS.md). We welcome the hardware donation, including the latest CPU and boards. From 51a465a162a3d039b290803ca8770ef772072cf9 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Wed, 13 Mar 2024 14:10:27 +0000 Subject: [PATCH 009/267] Mailing list doc updates --- docs/about.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/about.md b/docs/about.md index 686d8cf617..95acdccc8c 100644 --- a/docs/about.md +++ b/docs/about.md @@ -1,8 +1,6 @@ ## Mailing list -OpenBLAS users: - -OpenBLAS developers: +We have a [GitHub discussions](https://github.com/OpenMathLib/OpenBLAS/discussions/) forum to discuss usage and development of OpenBLAS. We also have a [Google group for *users*](https://groups.google.com/forum/#!forum/openblas-users) and a [Google group for *development of*](https://groups.google.com/forum/#!forum/openblas-dev) OpenBLAS. ## Donations From 45cbabcf58d5a2f44fc5dbc2205941af44c152ea Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Wed, 20 Mar 2024 11:19:51 +0000 Subject: [PATCH 010/267] Improve `index.md` * Fix typo * Improve platforms sentence --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index a8866e946f..3358491d59 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,7 +2,7 @@ OpenBLAS is an optimized Basic Linear Algebra Subprograms (BLAS) library based on [GotoBLAS2](https://www.tacc.utexas.edu/research-development/tacc-software/gotoblas2) 1.13 BSD version. -OpenBLAS implements low-level routines for performing linear algebra operations such as vector addition, scalar multiplication, dot products, linear combinations, and matrix multiplicationon. OpenBLAS makes these routines available on multiple popular platforms, covering desktop (Windows, Linux, Mac) and mobile (Android, iOS) operating systems, as well as different architectures (namely x86 and ARM). +OpenBLAS implements low-level routines for performing linear algebra operations such as vector addition, scalar multiplication, dot products, linear combinations, and matrix multiplication. OpenBLAS makes these routines available on multiple platforms, covering server, desktop and mobile operating systems, as well as different architectures including x86, ARM, MIPS, PPC, RISC-V, and zarch. The old GotoBLAS documentation can be found [on GitHub](https://github.com/OpenMathLib/OpenBLAS/blob/develop/GotoBLAS_01Readme.txt). From 32c31ec82aebfbec6cf790b5db5597a24ba026c4 Mon Sep 17 00:00:00 2001 From: Matthew Barber Date: Wed, 20 Mar 2024 12:40:57 +0000 Subject: [PATCH 011/267] Build docs to `docs/` in GitHub pages --- .github/workflows/docs.yml | 10 +++++++++- mkdocs.yml | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1091824e04..359281d4e8 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -13,4 +13,12 @@ jobs: with: python-version: "3.10" - run: pip install mkdocs mkdocs-material - - run: mkdocs gh-deploy --force --clean --verbose + # mkdocs gh-deploy command only builds to the top-level, hence building then deploying ourselves + - run: mkdocs build + - name: Deploy docs + uses: peaceiris/actions-gh-pages@v3 + if: ${{ github.ref == 'refs/heads/develop' }} + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./site + destination_dir: docs/ diff --git a/mkdocs.yml b/mkdocs.yml index cd23bd8924..74a18a5bb8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,4 +1,5 @@ site_name: OpenBLAS +site_url: https://openblas.net/docs/ theme: name: material logo: logo.svg From 7102367fde4e8281491cdd88b091e373e114bd50 Mon Sep 17 00:00:00 2001 From: shivammonaka Date: Thu, 11 Jan 2024 18:37:11 +0530 Subject: [PATCH 012/267] Introduced callback to Pthread, Win32 and OpenMP backend --- cblas.h | 5 + common_interface.h | 5 + driver/others/CMakeLists.txt | 1 + driver/others/Makefile | 5 +- driver/others/blas_server.c | 261 ++++++++++++++++----------- driver/others/blas_server_callback.c | 12 ++ driver/others/blas_server_omp.c | 30 +-- driver/others/blas_server_win32.c | 222 ++++++++++++++--------- 8 files changed, 337 insertions(+), 204 deletions(-) create mode 100644 driver/others/blas_server_callback.c diff --git a/cblas.h b/cblas.h index beaa32cc2b..ab8b7dcde4 100644 --- a/cblas.h +++ b/cblas.h @@ -26,6 +26,11 @@ char* openblas_get_config(void); /*Get the CPU corename on runtime.*/ char* openblas_get_corename(void); +/*Set the threading backend to a custom callback.*/ +typedef void (*openblas_dojob_callback)(int thread_num, void *jobdata, int dojob_data); +typedef void (*openblas_threads_callback)(int sync, openblas_dojob_callback dojob, int numjobs, size_t jobdata_elsize, void *jobdata, int dojob_data); +void openblas_set_threads_callback_function(openblas_threads_callback callback); + #ifdef OPENBLAS_OS_LINUX /* Sets thread affinity for OpenBLAS threads. `thread_idx` is in [0, openblas_get_num_threads()-1]. */ int openblas_setaffinity(int thread_idx, size_t cpusetsize, cpu_set_t* cpu_set); diff --git a/common_interface.h b/common_interface.h index 5a2e1654c9..efd3c6649d 100644 --- a/common_interface.h +++ b/common_interface.h @@ -47,6 +47,11 @@ int BLASFUNC(xerbla)(char *, blasint *info, blasint); void openblas_set_num_threads_(int *); +/*Set the threading backend to a custom callback.*/ +typedef void (*openblas_dojob_callback)(int thread_num, void *jobdata, int dojob_data); +typedef void (*openblas_threads_callback)(int sync, openblas_dojob_callback dojob, int numjobs, size_t jobdata_elsize, void *jobdata, int dojob_data); +extern openblas_threads_callback openblas_threads_callback_; + FLOATRET BLASFUNC(sdot) (blasint *, float *, blasint *, float *, blasint *); FLOATRET BLASFUNC(sdsdot)(blasint *, float *, float *, blasint *, float *, blasint *); diff --git a/driver/others/CMakeLists.txt b/driver/others/CMakeLists.txt index 1a38740a32..192bf2d8fc 100644 --- a/driver/others/CMakeLists.txt +++ b/driver/others/CMakeLists.txt @@ -25,6 +25,7 @@ if (USE_THREAD) ${BLAS_SERVER} divtable.c # TODO: Makefile has -UDOUBLE blas_l1_thread.c + blas_server_callback.c ) if (NOT NO_AFFINITY) diff --git a/driver/others/Makefile b/driver/others/Makefile index e4e9ee108f..ff7e3e96db 100644 --- a/driver/others/Makefile +++ b/driver/others/Makefile @@ -6,7 +6,7 @@ COMMONOBJS = memory.$(SUFFIX) xerbla.$(SUFFIX) c_abs.$(SUFFIX) z_abs.$(SUFFIX) #COMMONOBJS += slamch.$(SUFFIX) slamc3.$(SUFFIX) dlamch.$(SUFFIX) dlamc3.$(SUFFIX) ifdef SMP -COMMONOBJS += blas_server.$(SUFFIX) divtable.$(SUFFIX) blasL1thread.$(SUFFIX) +COMMONOBJS += blas_server.$(SUFFIX) divtable.$(SUFFIX) blasL1thread.$(SUFFIX) blas_server_callback.$(SUFFIX) ifneq ($(NO_AFFINITY), 1) COMMONOBJS += init.$(SUFFIX) endif @@ -140,6 +140,9 @@ memory.$(SUFFIX) : $(MEMORY) ../../common.h ../../param.h blas_server.$(SUFFIX) : $(BLAS_SERVER) ../../common.h ../../common_thread.h ../../param.h $(CC) $(CFLAGS) -c $< -o $(@F) +blas_server_callback.$(SUFFIX) : blas_server_callback.c ../../common.h + $(CC) $(CFLAGS) -c $< -o $(@F) + openblas_set_num_threads.$(SUFFIX) : openblas_set_num_threads.c $(CC) $(CFLAGS) -c $< -o $(@F) diff --git a/driver/others/blas_server.c b/driver/others/blas_server.c index 2531c57e9a..c25d04b7b8 100644 --- a/driver/others/blas_server.c +++ b/driver/others/blas_server.c @@ -115,6 +115,8 @@ int blas_server_avail __attribute__((aligned(ATTRIBUTE_SIZE))) = 0; int blas_omp_threads_local = 1; +static void * blas_thread_buffer[MAX_CPU_NUMBER]; + /* Local Variables */ #if defined(USE_PTHREAD_LOCK) static pthread_mutex_t server_lock = PTHREAD_MUTEX_INITIALIZER; @@ -190,6 +192,10 @@ static int main_status[MAX_CPU_NUMBER]; BLASLONG exit_time[MAX_CPU_NUMBER]; #endif +//Prototypes +static void exec_threads(int , blas_queue_t *, int); +static void adjust_thread_buffers(); + static void legacy_exec(void *func, int mode, blas_arg_t *args, void *sb){ if (!(mode & BLAS_COMPLEX)){ @@ -375,7 +381,6 @@ static void* blas_thread_server(void *arg){ /* Thread identifier */ BLASLONG cpu = (BLASLONG)arg; unsigned int last_tick; - void *buffer, *sa, *sb; blas_queue_t *queue; blas_queue_t *tscq; @@ -395,8 +400,6 @@ blas_queue_t *tscq; main_status[cpu] = MAIN_ENTER; #endif - buffer = blas_memory_alloc(2); - #ifdef SMP_DEBUG fprintf(STDERR, "Server[%2ld] Thread has just been spawned!\n", cpu); #endif @@ -456,109 +459,10 @@ blas_queue_t *tscq; start = rpcc(); #endif - if (queue) { - int (*routine)(blas_arg_t *, void *, void *, void *, void *, BLASLONG) = (int (*)(blas_arg_t *, void *, void *, void *, void *, BLASLONG))queue -> routine; - - atomic_store_queue(&thread_status[cpu].queue, (blas_queue_t *)1); - - sa = queue -> sa; - sb = queue -> sb; - -#ifdef SMP_DEBUG - if (queue -> args) { - fprintf(STDERR, "Server[%2ld] Calculation started. Mode = 0x%03x M = %3ld N=%3ld K=%3ld\n", - cpu, queue->mode, queue-> args ->m, queue->args->n, queue->args->k); - } -#endif - -#ifdef CONSISTENT_FPCSR -#ifdef __aarch64__ - __asm__ __volatile__ ("msr fpcr, %0" : : "r" (queue -> sse_mode)); -#else - __asm__ __volatile__ ("ldmxcsr %0" : : "m" (queue -> sse_mode)); - __asm__ __volatile__ ("fldcw %0" : : "m" (queue -> x87_mode)); -#endif -#endif + if(queue) { -#ifdef MONITOR - main_status[cpu] = MAIN_RUNNING1; -#endif - - if (sa == NULL) sa = (void *)((BLASLONG)buffer + GEMM_OFFSET_A); - - if (sb == NULL) { - if (!(queue -> mode & BLAS_COMPLEX)){ -#ifdef EXPRECISION - if ((queue -> mode & BLAS_PREC) == BLAS_XDOUBLE){ - sb = (void *)(((BLASLONG)sa + ((QGEMM_P * QGEMM_Q * sizeof(xdouble) - + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); - } else -#endif - if ((queue -> mode & BLAS_PREC) == BLAS_DOUBLE) { -#ifdef BUILD_DOUBLE - sb = (void *)(((BLASLONG)sa + ((DGEMM_P * DGEMM_Q * sizeof(double) - + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); -#endif - } else if ((queue -> mode & BLAS_PREC) == BLAS_SINGLE) { -#ifdef BUILD_SINGLE - sb = (void *)(((BLASLONG)sa + ((SGEMM_P * SGEMM_Q * sizeof(float) - + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); -#endif - } else { - /* Other types in future */ - } - } else { -#ifdef EXPRECISION - if ((queue -> mode & BLAS_PREC) == BLAS_XDOUBLE){ - sb = (void *)(((BLASLONG)sa + ((XGEMM_P * XGEMM_Q * 2 * sizeof(xdouble) - + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); - } else -#endif - if ((queue -> mode & BLAS_PREC) == BLAS_DOUBLE){ -#ifdef BUILD_COMPLEX16 - sb = (void *)(((BLASLONG)sa + ((ZGEMM_P * ZGEMM_Q * 2 * sizeof(double) - + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); -#endif - } else if ((queue -> mode & BLAS_PREC) == BLAS_SINGLE) { -#ifdef BUILD_COMPLEX - sb = (void *)(((BLASLONG)sa + ((CGEMM_P * CGEMM_Q * 2 * sizeof(float) - + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); -#endif - } else { - /* Other types in future */ - } - } - queue->sb=sb; - } - -#ifdef MONITOR - main_status[cpu] = MAIN_RUNNING2; -#endif - - if (queue -> mode & BLAS_LEGACY) { - legacy_exec(routine, queue -> mode, queue -> args, sb); - } else - if (queue -> mode & BLAS_PTHREAD) { - void (*pthreadcompat)(void *) = (void(*)(void*))queue -> routine; - (pthreadcompat)(queue -> args); - } else - (routine)(queue -> args, queue -> range_m, queue -> range_n, sa, sb, queue -> position); - -#ifdef SMP_DEBUG - fprintf(STDERR, "Server[%2ld] Calculation finished!\n", cpu); -#endif - -#ifdef MONITOR - main_status[cpu] = MAIN_FINISH; -#endif - - // arm: make sure all results are written out _before_ - // thread is marked as done and other threads use them - MB; - atomic_store_queue(&thread_status[cpu].queue, (blas_queue_t *)0); - - - } + exec_threads(cpu, queue, 0); + } #ifdef MONITOR main_status[cpu] = MAIN_DONE; @@ -580,8 +484,6 @@ blas_queue_t *tscq; fprintf(STDERR, "Server[%2ld] Shutdown!\n", cpu); #endif - blas_memory_free(buffer); - //pthread_exit(NULL); return NULL; @@ -663,6 +565,9 @@ int blas_thread_init(void){ LOCK_COMMAND(&server_lock); + // Adjust thread buffers + adjust_thread_buffers(); + if (!blas_server_avail){ thread_timeout_env=openblas_thread_timeout(); @@ -893,6 +798,18 @@ int exec_blas(BLASLONG num, blas_queue_t *queue){ fprintf(STDERR, "Exec_blas is called. Number of executing threads : %ld\n", num); #endif +//Redirect to caller's callback routine +if (openblas_threads_callback_) { + int buf_index = 0, i = 0; +#ifndef USE_SIMPLE_THREADED_LEVEL3 + for (i = 0; i < num; i ++) + queue[i].position = i; +#endif + openblas_threads_callback_(1, (openblas_dojob_callback) exec_threads, num, sizeof(blas_queue_t), (void*) queue, buf_index); + return 0; + } + + #ifdef __ELF__ if (omp_in_parallel && (num > 1)) { if (omp_in_parallel() > 0) { @@ -1066,6 +983,14 @@ int BLASFUNC(blas_thread_shutdown)(void){ LOCK_COMMAND(&server_lock); + //Free buffers allocated for threads + for(i=0; i routine; + + atomic_store_queue(&thread_status[cpu].queue, (blas_queue_t *)1); + + void *buffer = blas_thread_buffer[cpu]; + void *sa = queue -> sa; + void *sb = queue -> sb; + +#ifdef SMP_DEBUG + if (queue -> args) { +fprintf(STDERR, "Server[%2ld] Calculation started. Mode = 0x%03x M = %3ld N=%3ld K=%3ld\n", + cpu, queue->mode, queue-> args ->m, queue->args->n, queue->args->k); + } #endif +#ifdef CONSISTENT_FPCSR +#ifdef __aarch64__ + __asm__ __volatile__ ("msr fpcr, %0" : : "r" (queue -> sse_mode)); +#else + __asm__ __volatile__ ("ldmxcsr %0" : : "m" (queue -> sse_mode)); + __asm__ __volatile__ ("fldcw %0" : : "m" (queue -> x87_mode)); +#endif +#endif + +#ifdef MONITOR + main_status[cpu] = MAIN_RUNNING1; +#endif + + if (sa == NULL) sa = (void *)((BLASLONG)buffer + GEMM_OFFSET_A); + + if (sb == NULL) { +if (!(queue -> mode & BLAS_COMPLEX)){ +#ifdef EXPRECISION + if ((queue -> mode & BLAS_PREC) == BLAS_XDOUBLE){ + sb = (void *)(((BLASLONG)sa + ((QGEMM_P * QGEMM_Q * sizeof(xdouble) + + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); + } else +#endif + if ((queue -> mode & BLAS_PREC) == BLAS_DOUBLE) { +#ifdef BUILD_DOUBLE + sb = (void *)(((BLASLONG)sa + ((DGEMM_P * DGEMM_Q * sizeof(double) + + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); +#endif + } else if ((queue -> mode & BLAS_PREC) == BLAS_SINGLE) { +#ifdef BUILD_SINGLE + sb = (void *)(((BLASLONG)sa + ((SGEMM_P * SGEMM_Q * sizeof(float) + + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); +#endif + } else { + /* Other types in future */ + } +} else { +#ifdef EXPRECISION + if ((queue -> mode & BLAS_PREC) == BLAS_XDOUBLE){ + sb = (void *)(((BLASLONG)sa + ((XGEMM_P * XGEMM_Q * 2 * sizeof(xdouble) + + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); + } else +#endif + if ((queue -> mode & BLAS_PREC) == BLAS_DOUBLE){ +#ifdef BUILD_COMPLEX16 + sb = (void *)(((BLASLONG)sa + ((ZGEMM_P * ZGEMM_Q * 2 * sizeof(double) + + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); +#endif + } else if ((queue -> mode & BLAS_PREC) == BLAS_SINGLE) { +#ifdef BUILD_COMPLEX + sb = (void *)(((BLASLONG)sa + ((CGEMM_P * CGEMM_Q * 2 * sizeof(float) + + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); +#endif + } else { + /* Other types in future */ + } +} +queue->sb=sb; + } + +#ifdef MONITOR +main_status[cpu] = MAIN_RUNNING2; +#endif + + if (queue -> mode & BLAS_LEGACY) { +legacy_exec(routine, queue -> mode, queue -> args, sb); + } else +if (queue -> mode & BLAS_PTHREAD) { + void (*pthreadcompat)(void *) = (void(*)(void*))queue -> routine; + (pthreadcompat)(queue -> args); +} else + (routine)(queue -> args, queue -> range_m, queue -> range_n, sa, sb, queue -> position); + +#ifdef SMP_DEBUG + fprintf(STDERR, "Server[%2ld] Calculation finished!\n", cpu); +#endif + +#ifdef MONITOR + main_status[cpu] = MAIN_FINISH; +#endif + + // arm: make sure all results are written out _before_ + // thread is marked as done and other threads use them + MB; + atomic_store_queue(&thread_status[cpu].queue, (blas_queue_t *)0); + +} + +#endif \ No newline at end of file diff --git a/driver/others/blas_server_callback.c b/driver/others/blas_server_callback.c new file mode 100644 index 0000000000..48cf3541a8 --- /dev/null +++ b/driver/others/blas_server_callback.c @@ -0,0 +1,12 @@ +#include "common.h" + +/* global variable to change threading backend from openblas-managed to caller-managed */ +openblas_threads_callback openblas_threads_callback_ = 0; + +/* non-threadsafe function should be called before any other + openblas function to change how threads are managed */ + +void openblas_set_threads_callback_function(openblas_threads_callback callback) +{ + openblas_threads_callback_ = callback; +} \ No newline at end of file diff --git a/driver/others/blas_server_omp.c b/driver/others/blas_server_omp.c index 6f2ea8623d..2d06c1c705 100644 --- a/driver/others/blas_server_omp.c +++ b/driver/others/blas_server_omp.c @@ -285,7 +285,7 @@ static void legacy_exec(void *func, int mode, blas_arg_t *args, void *sb){ } } -static void exec_threads(blas_queue_t *queue, int buf_index){ +static void exec_threads(int thread_num, blas_queue_t *queue, int buf_index){ void *buffer, *sa, *sb; int pos=0, release_flag=0; @@ -305,7 +305,7 @@ static void exec_threads(blas_queue_t *queue, int buf_index){ if ((sa == NULL) && (sb == NULL) && ((queue -> mode & BLAS_PTHREAD) == 0)) { - pos = omp_get_thread_num(); + pos= thread_num; buffer = blas_thread_buffer[buf_index][pos]; //fallback @@ -420,18 +420,25 @@ while (true) { break; } } - if (i != MAX_PARALLEL_NUMBER) - break; -} -if (openblas_omp_adaptive_env() != 0) { -#pragma omp parallel for num_threads(num) schedule(OMP_SCHED) - for (i = 0; i < num; i ++) { + if(i != MAX_PARALLEL_NUMBER) + break; + } + /*For caller-managed threading, if caller has registered the callback, pass exec_thread as callback function*/ + if (openblas_threads_callback_) { +#ifndef USE_SIMPLE_THREADED_LEVEL3 + for (i = 0; i < num; i ++) + queue[i].position = i; +#endif + openblas_threads_callback_(1, (openblas_dojob_callback) exec_threads, num, sizeof(blas_queue_t), (void*) queue, buf_index); + } else { + if (openblas_omp_adaptive_env() != 0) { + #pragma omp parallel for num_threads(num) schedule(OMP_SCHED) + for (i = 0; i < num; i ++) { #ifndef USE_SIMPLE_THREADED_LEVEL3 queue[i].position = i; #endif - - exec_threads(&queue[i], buf_index); + exec_threads(omp_get_thread_num(), &queue[i], buf_index); } } else { #pragma omp parallel for schedule(OMP_SCHED) @@ -441,9 +448,10 @@ if (openblas_omp_adaptive_env() != 0) { queue[i].position = i; #endif - exec_threads(&queue[i], buf_index); + exec_threads(omp_get_thread_num(), &queue[i], buf_index); } } +} #ifdef HAVE_C11 atomic_store(&blas_buffer_inuse[buf_index], false); diff --git a/driver/others/blas_server_win32.c b/driver/others/blas_server_win32.c index 2ad8b8c5fd..56512ec333 100644 --- a/driver/others/blas_server_win32.c +++ b/driver/others/blas_server_win32.c @@ -1,3 +1,4 @@ + /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ /* All rights reserved. */ @@ -67,6 +68,8 @@ int blas_server_avail = 0; int blas_omp_threads_local = 1; +static void * blas_thread_buffer[MAX_CPU_NUMBER]; + /* Local Variables */ static BLASULONG server_lock = 0; @@ -74,6 +77,10 @@ static HANDLE blas_threads [MAX_CPU_NUMBER]; static DWORD blas_threads_id[MAX_CPU_NUMBER]; static volatile int thread_target; // target num of live threads, volatile for cross-thread reads +//Prototypes +static void exec_threads(int , blas_queue_t *, int); +static void adjust_thread_buffers(); + // // Legacy code path // @@ -207,13 +214,9 @@ static DWORD WINAPI blas_thread_server(void *arg) { /* Thread identifier */ BLASLONG cpu = (BLASLONG)arg; - - void *buffer, *sa, *sb; + blas_queue_t *queue; - /* Each server needs each buffer */ - buffer = blas_memory_alloc(2); - MT_TRACE("Server[%2ld] Thread is started!\n", cpu); while (1) { @@ -240,87 +243,14 @@ static DWORD WINAPI blas_thread_server(void *arg) { LeaveCriticalSection(&queue_lock); - if (queue) { - int (*routine)(blas_arg_t *, void *, void *, void *, void *, BLASLONG) = queue -> routine; - - sa = queue -> sa; - sb = queue -> sb; - - #ifdef CONSISTENT_FPCSR - __asm__ __volatile__ ("ldmxcsr %0" : : "m" (queue -> sse_mode)); - __asm__ __volatile__ ("fldcw %0" : : "m" (queue -> x87_mode)); - #endif - - MT_TRACE("Server[%2ld] Started. Mode = 0x%03x M = %3ld N=%3ld K=%3ld\n", - cpu, queue->mode, queue-> args ->m, queue->args->n, queue->args->k); - - // fprintf(stderr, "queue start[%ld]!!!\n", cpu); - - #ifdef MONITOR - main_status[cpu] = MAIN_RUNNING1; - #endif - - if (sa == NULL) - sa = (void *)((BLASLONG)buffer + GEMM_OFFSET_A); - - if (sb == NULL) { - if (!(queue -> mode & BLAS_COMPLEX)) { -#ifdef EXPRECISION - if ((queue -> mode & BLAS_PREC) == BLAS_XDOUBLE) { - sb = (void *)(((BLASLONG)sa + ((XGEMM_P * XGEMM_Q * sizeof(xdouble) - + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); - } else -#endif - if ((queue -> mode & BLAS_PREC) == BLAS_DOUBLE) { -#ifdef BUILD_DOUBLE - sb = (void *)(((BLASLONG)sa + ((DGEMM_P * DGEMM_Q * sizeof(double) - + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); -#endif - } else if ((queue -> mode & BLAS_PREC) == BLAS_SINGLE) { -#ifdef BUILD_SINGLE - sb = (void *)(((BLASLONG)sa + ((SGEMM_P * SGEMM_Q * sizeof(float) - + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); -#endif - } else { - /* Other types in future */ - } - } else { -#ifdef EXPRECISION - if ((queue -> mode & BLAS_PREC) == BLAS_XDOUBLE){ - sb = (void *)(((BLASLONG)sa + ((XGEMM_P * XGEMM_Q * 2 * sizeof(xdouble) - + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); - } else -#endif - if ((queue -> mode & BLAS_PREC) == BLAS_DOUBLE){ -#ifdef BUILD_COMPLEX16 - sb = (void *)(((BLASLONG)sa + ((ZGEMM_P * ZGEMM_Q * 2 * sizeof(double) - + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); -#endif - } else if ((queue -> mode & BLAS_PREC) == BLAS_SINGLE) { -#ifdef BUILD_COMPLEX - sb = (void *)(((BLASLONG)sa + ((CGEMM_P * CGEMM_Q * 2 * sizeof(float) - + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); -#endif - } else { - /* Other types in future */ - } - } - queue->sb=sb; - } - - #ifdef MONITOR - main_status[cpu] = MAIN_RUNNING2; - #endif + if(queue) { - if (!(queue -> mode & BLAS_LEGACY)) { - (routine)(queue -> args, queue -> range_m, queue -> range_n, sa, sb, queue -> position); - } else { - legacy_exec(routine, queue -> mode, queue -> args, sb); - } + exec_threads(cpu, queue, 0); } else { - continue; //if queue == NULL - } + continue; //if queue == NULL + } + MT_TRACE("Server[%2ld] Finished!\n", cpu); queue->finished = 1; @@ -330,8 +260,6 @@ static DWORD WINAPI blas_thread_server(void *arg) { MT_TRACE("Server[%2ld] Shutdown!\n", cpu); - blas_memory_free(buffer); - return 0; } @@ -345,6 +273,8 @@ int blas_thread_init(void) { LOCK_COMMAND(&server_lock); + adjust_thread_buffers(); + MT_TRACE("Initializing Thread(Num. threads = %d)\n", blas_cpu_number); if (!blas_server_avail) { @@ -473,6 +403,17 @@ int exec_blas(BLASLONG num, blas_queue_t *queue) { if ((num <= 0) || (queue == NULL)) return 0; + //Redirect to caller's callback routine + if (openblas_threads_callback_) { + int buf_index = 0, i = 0; +#ifndef USE_SIMPLE_THREADED_LEVEL3 + for (i = 0; i < num; i ++) + queue[i].position = i; +#endif + openblas_threads_callback_(1, (openblas_dojob_callback) exec_threads, num, sizeof(blas_queue_t), (void*) queue, buf_index); + return 0; + } + if ((num > 1) && queue -> next) exec_blas_async(1, queue -> next); @@ -507,6 +448,14 @@ int BLASFUNC(blas_thread_shutdown)(void) { LOCK_COMMAND(&server_lock); + //Free buffers allocated for threads + for(i=0; i sa; + sb = queue -> sb; + + int (*routine)(blas_arg_t *, void *, void *, void *, void *, BLASLONG) = queue -> routine; + + #ifdef CONSISTENT_FPCSR + __asm__ __volatile__ ("ldmxcsr %0" : : "m" (queue -> sse_mode)); + __asm__ __volatile__ ("fldcw %0" : : "m" (queue -> x87_mode)); + #endif + + MT_TRACE("Server[%2ld] Started. Mode = 0x%03x M = %3ld N=%3ld K=%3ld\n", + cpu, queue->mode, queue-> args ->m, queue->args->n, queue->args->k); + + // fprintf(stderr, "queue start[%ld]!!!\n", cpu); + + #ifdef MONITOR + main_status[cpu] = MAIN_RUNNING1; + #endif + + if (sa == NULL) + sa = (void *)((BLASLONG)buffer + GEMM_OFFSET_A); + + if (sb == NULL) { + if (!(queue -> mode & BLAS_COMPLEX)) { +#ifdef EXPRECISION +if ((queue -> mode & BLAS_PREC) == BLAS_XDOUBLE) { + sb = (void *)(((BLASLONG)sa + ((XGEMM_P * XGEMM_Q * sizeof(xdouble) + + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); +} else +#endif + if ((queue -> mode & BLAS_PREC) == BLAS_DOUBLE) { +#ifdef BUILD_DOUBLE + sb = (void *)(((BLASLONG)sa + ((DGEMM_P * DGEMM_Q * sizeof(double) + + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); +#endif + } else if ((queue -> mode & BLAS_PREC) == BLAS_SINGLE) { +#ifdef BUILD_SINGLE + sb = (void *)(((BLASLONG)sa + ((SGEMM_P * SGEMM_Q * sizeof(float) + + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); +#endif + } else { + /* Other types in future */ + } +} else { +#ifdef EXPRECISION +if ((queue -> mode & BLAS_PREC) == BLAS_XDOUBLE){ + sb = (void *)(((BLASLONG)sa + ((XGEMM_P * XGEMM_Q * 2 * sizeof(xdouble) + + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); +} else +#endif + if ((queue -> mode & BLAS_PREC) == BLAS_DOUBLE){ +#ifdef BUILD_COMPLEX16 + sb = (void *)(((BLASLONG)sa + ((ZGEMM_P * ZGEMM_Q * 2 * sizeof(double) + + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); +#endif + } else if ((queue -> mode & BLAS_PREC) == BLAS_SINGLE) { +#ifdef BUILD_COMPLEX + sb = (void *)(((BLASLONG)sa + ((CGEMM_P * CGEMM_Q * 2 * sizeof(float) + + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); +#endif + } else { + /* Other types in future */ + } +} + queue->sb=sb; + } + + #ifdef MONITOR + main_status[cpu] = MAIN_RUNNING2; + #endif + + if (!(queue -> mode & BLAS_LEGACY)) { + (routine)(queue -> args, queue -> range_m, queue -> range_n, sa, sb, queue -> position); + } else { + legacy_exec(routine, queue -> mode, queue -> args, sb); + } + +} \ No newline at end of file From 17ab724da9b9a28451012e02054a59ca573dcae1 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 4 Apr 2024 23:01:32 +0200 Subject: [PATCH 013/267] Update version to 0.3.27.dev --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 096ca88b30..07a3457cc2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ project(OpenBLAS C ASM) set(OpenBLAS_MAJOR_VERSION 0) set(OpenBLAS_MINOR_VERSION 3) -set(OpenBLAS_PATCH_VERSION 27) +set(OpenBLAS_PATCH_VERSION 27.dev) set(OpenBLAS_VERSION "${OpenBLAS_MAJOR_VERSION}.${OpenBLAS_MINOR_VERSION}.${OpenBLAS_PATCH_VERSION}") From 1c7d27c750ebd8580434592e6fc454db92d2fcdd Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 4 Apr 2024 23:02:29 +0200 Subject: [PATCH 014/267] Update version to 0.3.27.dev --- Makefile.rule | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.rule b/Makefile.rule index e920ea4870..2e777cdc67 100644 --- a/Makefile.rule +++ b/Makefile.rule @@ -3,7 +3,7 @@ # # This library's version -VERSION = 0.3.27 +VERSION = 0.3.27.dev # If you set this prefix, the library name will be lib$(LIBNAMESUFFIX)openblas.a # and lib$(LIBNAMESUFFIX)openblas.so, with a matching soname in the shared library From 8e39c05efde5fbe2be412193e1c97743417dfa4b Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Fri, 5 Apr 2024 11:39:01 +0800 Subject: [PATCH 015/267] Get the l2 cache size via environment variable on confidential VM The CPUID(leaf:2 or leaf:0x80000006) is not supported on some confidential VMs. As a result the get_l2_size() returns the default 512M which brings performance issues. Introduce the environment variable OPENBLAS_L2_SIZE provided by the user to get the l2 cache size. Suggested-by: "Keshavamurthy, Anil S" Signed-off-by: Chen Yu --- kernel/setparam-ref.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/setparam-ref.c b/kernel/setparam-ref.c index 4c361f1559..180268e2c0 100644 --- a/kernel/setparam-ref.c +++ b/kernel/setparam-ref.c @@ -1248,6 +1248,10 @@ static __inline__ int get_l2_size(void){ int eax, ebx, ecx, edx, l2; + l2 = readenv_atoi("OPENBLAS_L2_SIZE"); + if (l2 != 0) + return l2; + cpuid(0x80000006, &eax, &ebx, &ecx, &edx); l2 = BITMASK(ecx, 16, 0xffff); From ec8e9451f0b3af6bc397aa677f77ac57f3fa6344 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 5 Apr 2024 06:45:59 +0200 Subject: [PATCH 016/267] make independent of CBLAS --- utest/test_zscal.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/utest/test_zscal.c b/utest/test_zscal.c index ffc851e8b5..e40e6e0dc5 100644 --- a/utest/test_zscal.c +++ b/utest/test_zscal.c @@ -13,7 +13,7 @@ CTEST(zscal, i_nan) { double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double nan[] = {NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0}; - cblas_zscal(9, i, &nan, 1); + BLASFUNC(zscal)(9, i, &nan, 1); ASSERT_TRUE(isnan(nan[0])); ASSERT_TRUE(isnan(nan[1])); ASSERT_TRUE(isnan(nan[16])); @@ -25,7 +25,7 @@ CTEST(zscal, i_nan_inc_2) double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double nan[] = {NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0}; - cblas_zscal(9, i, &nan, 2); + BLASFUNC(zscal)(9, i, &nan, 2); ASSERT_TRUE(isnan(nan[0])); ASSERT_TRUE(isnan(nan[1])); ASSERT_TRUE(isnan(nan[16])); @@ -36,7 +36,7 @@ CTEST(zscal, nan_i) { double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double nan[] = {NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0}; - cblas_zscal(9, &nan, &i, 1); + BLASFUNC(zscal)(9, &nan, &i, 1); ASSERT_TRUE(isnan(i[0])); ASSERT_TRUE(isnan(i[1])); ASSERT_TRUE(isnan(i[16])); @@ -48,7 +48,7 @@ CTEST(zscal, nan_i_inc_2) double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double nan[] = {NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0}; - cblas_zscal(9, &nan, &i, 2); + BLASFUNC(zscal)(9, &nan, &i, 2); ASSERT_TRUE(isnan(i[0])); ASSERT_TRUE(isnan(i[1])); ASSERT_TRUE(isnan(i[16])); @@ -59,7 +59,7 @@ CTEST(zscal, i_inf) { double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double inf[] = {INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0}; - cblas_zscal(9, i, &inf, 1); + BLASFUNC(zscal)(9, i, &inf, 1); ASSERT_TRUE(isnan(inf[0])); ASSERT_TRUE(isinf(inf[1])); ASSERT_TRUE(isnan(inf[16])); @@ -71,7 +71,7 @@ CTEST(zscal, i_inf_inc_2) double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double inf[] = {INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0}; - cblas_zscal(9, i, &inf, 2); + BLASFUNC(zscal)(9, i, &inf, 2); ASSERT_TRUE(isnan(inf[0])); ASSERT_TRUE(isinf(inf[1])); ASSERT_TRUE(isnan(inf[16])); @@ -82,7 +82,7 @@ CTEST(zscal, inf_i) { double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double inf[] = {INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0}; - cblas_zscal(9, &inf, &i, 1); + BLASFUNC(zscal)(9, &inf, &i, 1); ASSERT_TRUE(isnan(i[0])); ASSERT_TRUE(isinf(i[1])); ASSERT_TRUE(isnan(i[16])); @@ -94,7 +94,7 @@ CTEST(zscal, inf_i_inc_2) double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double inf[] = {INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0}; - cblas_zscal(9, &inf, &i, 2); + BLASFUNC(zscal)(9, &inf, &i, 2); ASSERT_TRUE(isnan(i[0])); ASSERT_TRUE(isinf(i[1])); ASSERT_TRUE(isnan(i[16])); From 1f080b93286b85bea630f3219b307d9767f15d69 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 5 Apr 2024 07:09:34 +0200 Subject: [PATCH 017/267] Update test_zscal.c --- utest/test_zscal.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/utest/test_zscal.c b/utest/test_zscal.c index e40e6e0dc5..2ee5547b73 100644 --- a/utest/test_zscal.c +++ b/utest/test_zscal.c @@ -13,7 +13,7 @@ CTEST(zscal, i_nan) { double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double nan[] = {NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0}; - BLASFUNC(zscal)(9, i, &nan, 1); + BLASFUNC(zscal)(9, i, nan, 1); ASSERT_TRUE(isnan(nan[0])); ASSERT_TRUE(isnan(nan[1])); ASSERT_TRUE(isnan(nan[16])); @@ -25,7 +25,7 @@ CTEST(zscal, i_nan_inc_2) double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double nan[] = {NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0}; - BLASFUNC(zscal)(9, i, &nan, 2); + BLASFUNC(zscal)(9, i, nan, 2); ASSERT_TRUE(isnan(nan[0])); ASSERT_TRUE(isnan(nan[1])); ASSERT_TRUE(isnan(nan[16])); @@ -36,7 +36,7 @@ CTEST(zscal, nan_i) { double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double nan[] = {NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0}; - BLASFUNC(zscal)(9, &nan, &i, 1); + BLASFUNC(zscal)(9, nan, i, 1); ASSERT_TRUE(isnan(i[0])); ASSERT_TRUE(isnan(i[1])); ASSERT_TRUE(isnan(i[16])); @@ -48,7 +48,7 @@ CTEST(zscal, nan_i_inc_2) double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double nan[] = {NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0}; - BLASFUNC(zscal)(9, &nan, &i, 2); + BLASFUNC(zscal)(9, nan, i, 2); ASSERT_TRUE(isnan(i[0])); ASSERT_TRUE(isnan(i[1])); ASSERT_TRUE(isnan(i[16])); @@ -59,7 +59,7 @@ CTEST(zscal, i_inf) { double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double inf[] = {INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0}; - BLASFUNC(zscal)(9, i, &inf, 1); + BLASFUNC(zscal)(9, i, inf, 1); ASSERT_TRUE(isnan(inf[0])); ASSERT_TRUE(isinf(inf[1])); ASSERT_TRUE(isnan(inf[16])); @@ -71,7 +71,7 @@ CTEST(zscal, i_inf_inc_2) double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double inf[] = {INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0}; - BLASFUNC(zscal)(9, i, &inf, 2); + BLASFUNC(zscal)(9, i, inf, 2); ASSERT_TRUE(isnan(inf[0])); ASSERT_TRUE(isinf(inf[1])); ASSERT_TRUE(isnan(inf[16])); @@ -82,7 +82,7 @@ CTEST(zscal, inf_i) { double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double inf[] = {INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0}; - BLASFUNC(zscal)(9, &inf, &i, 1); + BLASFUNC(zscal)(9, inf, i, 1); ASSERT_TRUE(isnan(i[0])); ASSERT_TRUE(isinf(i[1])); ASSERT_TRUE(isnan(i[16])); @@ -94,7 +94,7 @@ CTEST(zscal, inf_i_inc_2) double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double inf[] = {INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0}; - BLASFUNC(zscal)(9, &inf, &i, 2); + BLASFUNC(zscal)(9, inf, i, 2); ASSERT_TRUE(isnan(i[0])); ASSERT_TRUE(isinf(i[1])); ASSERT_TRUE(isnan(i[16])); From d3f93c6015a8e8f879bcf719cd4c32e4a4466d96 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 5 Apr 2024 07:23:41 +0200 Subject: [PATCH 018/267] fix arguments of zscal --- utest/test_zscal.c | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/utest/test_zscal.c b/utest/test_zscal.c index 2ee5547b73..1d4f7bb6b8 100644 --- a/utest/test_zscal.c +++ b/utest/test_zscal.c @@ -11,9 +11,11 @@ CTEST(zscal, i_nan) { + int N=9; + int incX=1; double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double nan[] = {NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0}; - BLASFUNC(zscal)(9, i, nan, 1); + BLASFUNC(zscal)(&N, i, nan, &incX); ASSERT_TRUE(isnan(nan[0])); ASSERT_TRUE(isnan(nan[1])); ASSERT_TRUE(isnan(nan[16])); @@ -22,10 +24,12 @@ CTEST(zscal, i_nan) CTEST(zscal, i_nan_inc_2) { + int N=9; + int incX=1; double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double nan[] = {NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0}; - BLASFUNC(zscal)(9, i, nan, 2); + BLASFUNC(zscal)(&N, i, nan, &incX); ASSERT_TRUE(isnan(nan[0])); ASSERT_TRUE(isnan(nan[1])); ASSERT_TRUE(isnan(nan[16])); @@ -34,9 +38,11 @@ CTEST(zscal, i_nan_inc_2) CTEST(zscal, nan_i) { + int N=9; + int incX=1; double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double nan[] = {NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0}; - BLASFUNC(zscal)(9, nan, i, 1); + BLASFUNC(zscal)(&N, nan, i, &incX); ASSERT_TRUE(isnan(i[0])); ASSERT_TRUE(isnan(i[1])); ASSERT_TRUE(isnan(i[16])); @@ -45,10 +51,12 @@ CTEST(zscal, nan_i) CTEST(zscal, nan_i_inc_2) { + int N=9; + int incX=1; double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double nan[] = {NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0}; - BLASFUNC(zscal)(9, nan, i, 2); + BLASFUNC(zscal)(&N, nan, i, &incX); ASSERT_TRUE(isnan(i[0])); ASSERT_TRUE(isnan(i[1])); ASSERT_TRUE(isnan(i[16])); @@ -57,9 +65,11 @@ CTEST(zscal, nan_i_inc_2) CTEST(zscal, i_inf) { + int N=9; + int incX=1; double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double inf[] = {INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0}; - BLASFUNC(zscal)(9, i, inf, 1); + BLASFUNC(zscal)(&N, i, inf, &incX); ASSERT_TRUE(isnan(inf[0])); ASSERT_TRUE(isinf(inf[1])); ASSERT_TRUE(isnan(inf[16])); @@ -68,10 +78,12 @@ CTEST(zscal, i_inf) CTEST(zscal, i_inf_inc_2) { + int N=9; + int incX=2; double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double inf[] = {INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0}; - BLASFUNC(zscal)(9, i, inf, 2); + BLASFUNC(zscal)(&N, i, inf, &incX); ASSERT_TRUE(isnan(inf[0])); ASSERT_TRUE(isinf(inf[1])); ASSERT_TRUE(isnan(inf[16])); @@ -80,9 +92,11 @@ CTEST(zscal, i_inf_inc_2) CTEST(zscal, inf_i) { + int N=9; + int incX=1; double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double inf[] = {INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0}; - BLASFUNC(zscal)(9, inf, i, 1); + BLASFUNC(zscal)(&N, inf, i, &incX); ASSERT_TRUE(isnan(i[0])); ASSERT_TRUE(isinf(i[1])); ASSERT_TRUE(isnan(i[16])); @@ -91,10 +105,12 @@ CTEST(zscal, inf_i) CTEST(zscal, inf_i_inc_2) { + int N=9; + int incX=2; double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double inf[] = {INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0}; - BLASFUNC(zscal)(9, inf, i, 2); + BLASFUNC(zscal)(&N, inf, i, &incX); ASSERT_TRUE(isnan(i[0])); ASSERT_TRUE(isinf(i[1])); ASSERT_TRUE(isnan(i[16])); From 9c86838279505e95d1ff180a578ac66f850932b4 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 5 Apr 2024 22:48:08 +0200 Subject: [PATCH 019/267] use blasint for INTERFACE64 compatibility --- utest/test_zscal.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/utest/test_zscal.c b/utest/test_zscal.c index 1d4f7bb6b8..195e4945f3 100644 --- a/utest/test_zscal.c +++ b/utest/test_zscal.c @@ -11,8 +11,8 @@ CTEST(zscal, i_nan) { - int N=9; - int incX=1; + blasint N=9; + blasint incX=1; double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double nan[] = {NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0}; BLASFUNC(zscal)(&N, i, nan, &incX); @@ -24,8 +24,8 @@ CTEST(zscal, i_nan) CTEST(zscal, i_nan_inc_2) { - int N=9; - int incX=1; + blasint N=9; + blasint incX=1; double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double nan[] = {NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0}; @@ -38,8 +38,8 @@ CTEST(zscal, i_nan_inc_2) CTEST(zscal, nan_i) { - int N=9; - int incX=1; + blasint N=9; + blasint incX=1; double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double nan[] = {NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0}; BLASFUNC(zscal)(&N, nan, i, &incX); @@ -51,8 +51,8 @@ CTEST(zscal, nan_i) CTEST(zscal, nan_i_inc_2) { - int N=9; - int incX=1; + blasint N=9; + blasint incX=1; double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double nan[] = {NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0}; @@ -65,8 +65,8 @@ CTEST(zscal, nan_i_inc_2) CTEST(zscal, i_inf) { - int N=9; - int incX=1; + blasint N=9; + blasint incX=1; double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double inf[] = {INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0}; BLASFUNC(zscal)(&N, i, inf, &incX); @@ -78,8 +78,8 @@ CTEST(zscal, i_inf) CTEST(zscal, i_inf_inc_2) { - int N=9; - int incX=2; + blasint N=9; + blasint incX=2; double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double inf[] = {INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0}; @@ -92,8 +92,8 @@ CTEST(zscal, i_inf_inc_2) CTEST(zscal, inf_i) { - int N=9; - int incX=1; + blasint N=9; + blasint incX=1; double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double inf[] = {INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0}; BLASFUNC(zscal)(&N, inf, i, &incX); @@ -105,8 +105,8 @@ CTEST(zscal, inf_i) CTEST(zscal, inf_i_inc_2) { - int N=9; - int incX=2; + blasint N=9; + blasint incX=2; double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double inf[] = {INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0}; From 48e017de095018c60d83355804a3075658b4970c Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 5 Apr 2024 22:54:34 +0200 Subject: [PATCH 020/267] fix position of endif - gemm3m tests should not be run in cross-compiles --- ctest/Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ctest/Makefile b/ctest/Makefile index bbaf96f8ed..6c7cc1ed56 100644 --- a/ctest/Makefile +++ b/ctest/Makefile @@ -203,7 +203,6 @@ ifeq ($(BUILD_COMPLEX16),1) OPENBLAS_NUM_THREADS=2 ./xzcblat3 < zin3 endif endif -endif ifeq ($(SUPPORT_GEMM3M),1) ifeq ($(USE_OPENMP), 1) @@ -222,7 +221,7 @@ ifeq ($(BUILD_COMPLEX16),1) endif endif endif - +endif From 14e71c249df888fb56da83c786e45e41f26615cc Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 5 Apr 2024 23:01:34 +0200 Subject: [PATCH 021/267] retain the bf16 capability in fallback versions of the -march option for NeoverseN2 --- Makefile.arm64 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile.arm64 b/Makefile.arm64 index eeb7287351..4feb545239 100644 --- a/Makefile.arm64 +++ b/Makefile.arm64 @@ -145,13 +145,13 @@ ifeq (1, $(filter 1,$(GCCMINORVERSIONGTEQ4) $(GCCVERSIONGTEQ11) $(ISCLANG))) ifneq ($(OSNAME), Darwin) CCOMMON_OPT += -march=armv8.5-a+sve+sve2+bf16 -mtune=neoverse-n2 else -CCOMMON_OPT += -march=armv8.2-a+sve -mtune=cortex-a72 +CCOMMON_OPT += -march=armv8.2-a+sve+bf16 -mtune=cortex-a72 endif ifneq ($(F_COMPILER), NAG) FCOMMON_OPT += -march=armv8.5-a+sve+sve2+bf16 -mtune=neoverse-n2 endif else -CCOMMON_OPT += -march=armv8.5-a+sve +CCOMMON_OPT += -march=armv8.5-a+sve+bf16 ifneq ($(CROSS), 1) CCOMMON_OPT += -mtune=native endif @@ -163,13 +163,13 @@ endif endif endif else -CCOMMON_OPT += -march=armv8.2-a+sve -mtune=cortex-a72 +CCOMMON_OPT += -march=armv8.2-a+sve+bf16 -mtune=cortex-a72 ifneq ($(F_COMPILER), NAG) FCOMMON_OPT += -march=armv8.2-a -mtune=cortex-a72 endif endif else -CCOMMON_OPT += -march=armv8-a+sve -mtune=cortex-a72 +CCOMMON_OPT += -march=armv8-a+sve+bf16 -mtune=cortex-a72 ifneq ($(F_COMPILER), NAG) FCOMMON_OPT += -march=armv8-a -mtune=cortex-a72 endif From cda55f2fd29191b79c9ef160ef20743b2efa30e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D9=85=D9=87=D8=AF=D9=8A=20=D8=B4=D9=8A=D9=86=D9=88=D9=86?= =?UTF-8?q?=20=28Mehdi=20Chinoune=29?= Date: Sat, 6 Apr 2024 05:54:48 +0100 Subject: [PATCH 022/267] Don't pass `-exhaustive-register-search` directly to clang compiler `-exhaustive-register-search` is an LLVM code generation flag that shouldn't be passed directly to clang compiler. --- cmake/system.cmake | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmake/system.cmake b/cmake/system.cmake index c26b415c6d..7166cc07f5 100644 --- a/cmake/system.cmake +++ b/cmake/system.cmake @@ -172,9 +172,9 @@ if (DEFINED TARGET) endif() elseif (${CMAKE_C_COMPILER_ID} STREQUAL "Clang" OR ${CMAKE_C_COMPILER_ID} STREQUAL "AppleClang") if (${CMAKE_C_COMPILER_VERSION} VERSION_GREATER 8.99) - set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=cooperlake -exhaustive-register-search") + set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=cooperlake -mllvm -exhaustive-register-search") else() - set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=skylake-avx512 -exhaustive-register-search") + set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=skylake-avx512 -mllvm -exhaustive-register-search") endif() endif() endif() @@ -188,16 +188,16 @@ if (DEFINED TARGET) endif() elseif (${CMAKE_C_COMPILER_ID} STREQUAL "Clang" OR ${CMAKE_C_COMPILER_ID} STREQUAL "AppleClang") if (${CMAKE_C_COMPILER_VERSION} VERSION_GREATER 12.0) - set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=sapphirerapids -exhaustive-register-search") + set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=sapphirerapids -mllvm -exhaustive-register-search") else() - set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=skylake-avx512 -exhaustive-register-search") + set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=skylake-avx512 -mllvm -exhaustive-register-search") endif() endif() endif() if (${TARGET} STREQUAL SKYLAKEX AND NOT NO_AVX512) set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=skylake-avx512") if (${CMAKE_C_COMPILER_ID} STREQUAL "Clang" OR ${CMAKE_C_COMPILER_ID} STREQUAL "AppleClang") - set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -exhaustive-register-search") + set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -mllvm -exhaustive-register-search") endif() endif() @@ -215,7 +215,7 @@ if (DEFINED TARGET) else() set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=skylake-avx512") endif() - set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -exhaustive-register-search") + set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -mllvm -exhaustive-register-search") endif() endif() From 95315f95d2f5af3d3c0813aef175430bc9931344 Mon Sep 17 00:00:00 2001 From: cyk <57099728+cyk2018@users.noreply.github.com> Date: Sat, 6 Apr 2024 13:51:01 +0800 Subject: [PATCH 023/267] Update user_manual.md for static linker when I use static link method to compile , result is undefined with pthread_create, so we should add -lpthread --- docs/user_manual.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user_manual.md b/docs/user_manual.md index b5cd632f15..0855f81194 100644 --- a/docs/user_manual.md +++ b/docs/user_manual.md @@ -159,7 +159,7 @@ int main(int argc, char* argv[]) ``` ``` -gcc -o time_dgemm time_dgemm.c /your/path/libopenblas.a +gcc -o time_dgemm time_dgemm.c /your/path/libopenblas.a -lpthread ./time_dgemm ``` From d8c4ea8793c4340b20e28dd89f661bc50e605b06 Mon Sep 17 00:00:00 2001 From: gxw Date: Mon, 25 Mar 2024 04:56:13 -0400 Subject: [PATCH 024/267] loongarch: Optimizing the performance of the GEMM on servers --- common_loongarch64.h | 26 ++++++++ driver/others/blas_server.c | 10 ++- driver/others/blas_server_omp.c | 2 +- driver/others/memory.c | 4 +- driver/others/parameter.c | 94 ++++++++++++++++++++++++++++ interface/gemm.c | 11 ++++ kernel/setparam-ref.c | 106 +++++++++++++++++++++++++++++--- param.h | 20 +++--- 8 files changed, 252 insertions(+), 21 deletions(-) diff --git a/common_loongarch64.h b/common_loongarch64.h index b1426da792..367e5df18b 100644 --- a/common_loongarch64.h +++ b/common_loongarch64.h @@ -96,6 +96,32 @@ static inline int WhereAmI(void){ } #endif +static inline int get_cpu_model(char *model_name) { + FILE *cpuinfo_file = fopen("/proc/cpuinfo", "r"); + if (!cpuinfo_file) { + return 0; + } + char line[1024]; + while (fgets(line, sizeof(line), cpuinfo_file)) { + if (strstr(line, "model name")) { + char *token = strtok(line, ":"); + token = strtok(NULL, ":"); + while (*token == ' ') + token++; + char *end = token + strlen(token) - 1; + while (end > token && (*end == '\n' || *end == '\r')) { + *end = '\0'; + end--; + } + strcpy(model_name, token); + fclose(cpuinfo_file); + return 1; + } + } + fclose(cpuinfo_file); + return 0; +} + #ifdef DOUBLE #define GET_IMAGE(res) __asm__ __volatile__("fmov.d %0, $f2" : "=f"(res) : : "memory") #else diff --git a/driver/others/blas_server.c b/driver/others/blas_server.c index 2531c57e9a..f6c158a99d 100644 --- a/driver/others/blas_server.c +++ b/driver/others/blas_server.c @@ -484,6 +484,14 @@ blas_queue_t *tscq; main_status[cpu] = MAIN_RUNNING1; #endif +//For Loongson servers, like the 3C5000 (featuring 16 cores), applying an +//offset to the buffer is essential for minimizing cache conflicts and optimizing performance. +#if defined(LOONGSON3R5) && !defined(NO_AFFINITY) + char model_name[128]; + get_cpu_model(model_name); + if ((strstr(model_name, "3C5000") != NULL) || (strstr(model_name, "3D5000") != NULL)) + if (sa == NULL) sa = (void *)((BLASLONG)buffer + (WhereAmI() & 0xf) * GEMM_OFFSET_A); +#endif if (sa == NULL) sa = (void *)((BLASLONG)buffer + GEMM_OFFSET_A); if (sb == NULL) { @@ -1006,7 +1014,7 @@ void goto_set_num_threads(int num_threads) { blas_cpu_number = num_threads; -#if defined(ARCH_MIPS64) +#if defined(ARCH_MIPS64) || defined(ARCH_LOONGARCH64) #ifndef DYNAMIC_ARCH //set parameters for different number of threads. blas_set_parameter(); diff --git a/driver/others/blas_server_omp.c b/driver/others/blas_server_omp.c index 6f2ea8623d..53e6c98e00 100644 --- a/driver/others/blas_server_omp.c +++ b/driver/others/blas_server_omp.c @@ -113,7 +113,7 @@ void goto_set_num_threads(int num_threads) { blas_cpu_number = num_threads; adjust_thread_buffers(); -#if defined(ARCH_MIPS64) +#if defined(ARCH_MIPS64) || defined(ARCH_LOONGARCH64) //set parameters for different number of threads. blas_set_parameter(); #endif diff --git a/driver/others/memory.c b/driver/others/memory.c index 4ee8f9a2eb..c55688aced 100644 --- a/driver/others/memory.c +++ b/driver/others/memory.c @@ -1219,7 +1219,7 @@ UNLOCK_COMMAND(&alloc_lock); if (!blas_num_threads) blas_cpu_number = blas_get_cpu_number(); #endif -#if defined(ARCH_X86) || defined(ARCH_X86_64) || defined(ARCH_IA64) || defined(ARCH_MIPS64) || defined(ARCH_ARM64) +#if defined(ARCH_X86) || defined(ARCH_X86_64) || defined(ARCH_IA64) || defined(ARCH_MIPS64) || defined(ARCH_ARM64) || defined(ARCH_LOONGARCH64) #ifndef DYNAMIC_ARCH blas_set_parameter(); #endif @@ -2814,7 +2814,7 @@ void *blas_memory_alloc(int procpos){ if (!blas_num_threads) blas_cpu_number = blas_get_cpu_number(); #endif -#if defined(ARCH_X86) || defined(ARCH_X86_64) || defined(ARCH_IA64) || defined(ARCH_MIPS64) || defined(ARCH_ARM64) +#if defined(ARCH_X86) || defined(ARCH_X86_64) || defined(ARCH_IA64) || defined(ARCH_MIPS64) || defined(ARCH_ARM64) || defined(ARCH_LOONGARCH64) #ifndef DYNAMIC_ARCH blas_set_parameter(); #endif diff --git a/driver/others/parameter.c b/driver/others/parameter.c index de6bf0de46..a208a1a9d7 100644 --- a/driver/others/parameter.c +++ b/driver/others/parameter.c @@ -739,6 +739,100 @@ void blas_set_parameter(void){ } #endif +#if defined(ARCH_LOONGARCH64) +int get_L3_size() { + int ret = 0, id = 0x14; + __asm__ volatile ( + "cpucfg %[ret], %[id]" + : [ret]"=r"(ret) + : [id]"r"(id) + : "memory" + ); + return ((ret & 0xffff) + 1) * pow(2, ((ret >> 16) & 0xff)) * pow(2, ((ret >> 24) & 0x7f)) / 1024 / 1024; // MB +} + +void blas_set_parameter(void){ +#if defined(LOONGSON3R5) + int L3_size = get_L3_size(); +#ifdef SMP + if(blas_num_threads == 1){ +#endif + //single thread + if (L3_size == 32){ // 3C5000 and 3D5000 + sgemm_p = 256; + sgemm_q = 384; + sgemm_r = 8192; + + dgemm_p = 112; + dgemm_q = 289; + dgemm_r = 4096; + + cgemm_p = 128; + cgemm_q = 256; + cgemm_r = 4096; + + zgemm_p = 128; + zgemm_q = 128; + zgemm_r = 2048; + } else { // 3A5000 and 3C5000L + sgemm_p = 256; + sgemm_q = 384; + sgemm_r = 4096; + + dgemm_p = 112; + dgemm_q = 300; + dgemm_r = 3024; + + cgemm_p = 128; + cgemm_q = 256; + cgemm_r = 2048; + + zgemm_p = 128; + zgemm_q = 128; + zgemm_r = 1024; + } +#ifdef SMP + }else{ + //multi thread + if (L3_size == 32){ // 3C5000 and 3D5000 + sgemm_p = 256; + sgemm_q = 384; + sgemm_r = 1024; + + dgemm_p = 112; + dgemm_q = 289; + dgemm_r = 342; + + cgemm_p = 128; + cgemm_q = 256; + cgemm_r = 512; + + zgemm_p = 128; + zgemm_q = 128; + zgemm_r = 512; + } else { // 3A5000 and 3C5000L + sgemm_p = 256; + sgemm_q = 384; + sgemm_r = 2048; + + dgemm_p = 112; + dgemm_q = 300; + dgemm_r = 738; + + cgemm_p = 128; + cgemm_q = 256; + cgemm_r = 1024; + + zgemm_p = 128; + zgemm_q = 128; + zgemm_r = 1024; + } + } +#endif +#endif +} +#endif + #if defined(ARCH_ARM64) void blas_set_parameter(void) diff --git a/interface/gemm.c b/interface/gemm.c index 4778b641bb..f552059b84 100644 --- a/interface/gemm.c +++ b/interface/gemm.c @@ -521,7 +521,18 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANS buffer = (XFLOAT *)blas_memory_alloc(0); +//For Loongson servers, like the 3C5000 (featuring 16 cores), applying an +//offset to the buffer is essential for minimizing cache conflicts and optimizing performance. +#if defined(LOONGSON3R5) && !defined(NO_AFFINITY) + char model_name[128]; + get_cpu_model(model_name); + if ((strstr(model_name, "3C5000") != NULL) || (strstr(model_name, "3D5000") != NULL)) + sa = (XFLOAT *)((BLASLONG)buffer + (WhereAmI() & 0xf) * GEMM_OFFSET_A); + else + sa = (XFLOAT *)((BLASLONG)buffer + GEMM_OFFSET_A); +#else sa = (XFLOAT *)((BLASLONG)buffer +GEMM_OFFSET_A); +#endif sb = (XFLOAT *)(((BLASLONG)sa + ((GEMM_P * GEMM_Q * COMPSIZE * SIZE + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); #ifdef SMP diff --git a/kernel/setparam-ref.c b/kernel/setparam-ref.c index 4c361f1559..b573fc61ba 100644 --- a/kernel/setparam-ref.c +++ b/kernel/setparam-ref.c @@ -1066,31 +1066,123 @@ static void init_parameter(void) { } #else // (ARCH_MIPS64) #if (ARCH_LOONGARCH64) +static int get_L3_size() { + int ret = 0, id = 0x14; + __asm__ volatile ( + "cpucfg %[ret], %[id]" + : [ret]"=r"(ret) + : [id]"r"(id) + : "memory" + ); + return ((ret & 0xffff) + 1) * pow(2, ((ret >> 16) & 0xff)) * pow(2, ((ret >> 24) & 0x7f)) / 1024 / 1024; // MB +} static void init_parameter(void) { #ifdef BUILD_BFLOAT16 TABLE_NAME.sbgemm_p = SBGEMM_DEFAULT_P; #endif + +#ifdef BUILD_BFLOAT16 + TABLE_NAME.sbgemm_r = SBGEMM_DEFAULT_R; +#endif + +#if defined(LOONGSON3R5) + int L3_size = get_L3_size(); +#ifdef SMP + if(blas_num_threads == 1){ +#endif + //single thread + if (L3_size == 32){ // 3C5000 and 3D5000 + TABLE_NAME.sgemm_p = 256; + TABLE_NAME.sgemm_q = 384; + TABLE_NAME.sgemm_r = 8192; + + TABLE_NAME.dgemm_p = 112; + TABLE_NAME.dgemm_q = 289; + TABLE_NAME.dgemm_r = 4096; + + TABLE_NAME.cgemm_p = 128; + TABLE_NAME.cgemm_q = 256; + TABLE_NAME.cgemm_r = 4096; + + TABLE_NAME.zgemm_p = 128; + TABLE_NAME.zgemm_q = 128; + TABLE_NAME.zgemm_r = 2048; + } else { // 3A5000 and 3C5000L + TABLE_NAME.sgemm_p = 256; + TABLE_NAME.sgemm_q = 384; + TABLE_NAME.sgemm_r = 4096; + + TABLE_NAME.dgemm_p = 112; + TABLE_NAME.dgemm_q = 300; + TABLE_NAME.dgemm_r = 3024; + + TABLE_NAME.cgemm_p = 128; + TABLE_NAME.cgemm_q = 256; + TABLE_NAME.cgemm_r = 2048; + + TABLE_NAME.zgemm_p = 128; + TABLE_NAME.zgemm_q = 128; + TABLE_NAME.zgemm_r = 1024; + } +#ifdef SMP + }else{ + //multi thread + if (L3_size == 32){ // 3C5000 and 3D5000 + TABLE_NAME.sgemm_p = 256; + TABLE_NAME.sgemm_q = 384; + TABLE_NAME.sgemm_r = 1024; + + TABLE_NAME.dgemm_p = 112; + TABLE_NAME.dgemm_q = 289; + TABLE_NAME.dgemm_r = 342; + + TABLE_NAME.cgemm_p = 128; + TABLE_NAME.cgemm_q = 256; + TABLE_NAME.cgemm_r = 512; + + TABLE_NAME.zgemm_p = 128; + TABLE_NAME.zgemm_q = 128; + TABLE_NAME.zgemm_r = 512; + } else { // 3A5000 and 3C5000L + TABLE_NAME.sgemm_p = 256; + TABLE_NAME.sgemm_q = 384; + TABLE_NAME.sgemm_r = 2048; + + TABLE_NAME.dgemm_p = 112; + TABLE_NAME.dgemm_q = 300; + TABLE_NAME.dgemm_r = 738; + + TABLE_NAME.cgemm_p = 128; + TABLE_NAME.cgemm_q = 256; + TABLE_NAME.cgemm_r = 1024; + + TABLE_NAME.zgemm_p = 128; + TABLE_NAME.zgemm_q = 128; + TABLE_NAME.zgemm_r = 1024; + } + } +#endif +#else TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P; TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P; TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P; TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P; -#ifdef BUILD_BFLOAT16 - TABLE_NAME.sbgemm_r = SBGEMM_DEFAULT_R; -#endif + TABLE_NAME.sgemm_q = SGEMM_DEFAULT_Q; + TABLE_NAME.dgemm_q = DGEMM_DEFAULT_Q; + TABLE_NAME.cgemm_q = CGEMM_DEFAULT_Q; + TABLE_NAME.zgemm_q = ZGEMM_DEFAULT_Q; + TABLE_NAME.sgemm_r = SGEMM_DEFAULT_R; TABLE_NAME.dgemm_r = DGEMM_DEFAULT_R; TABLE_NAME.cgemm_r = CGEMM_DEFAULT_R; TABLE_NAME.zgemm_r = ZGEMM_DEFAULT_R; +#endif #ifdef BUILD_BFLOAT16 TABLE_NAME.sbgemm_q = SBGEMM_DEFAULT_Q; #endif - TABLE_NAME.sgemm_q = SGEMM_DEFAULT_Q; - TABLE_NAME.dgemm_q = DGEMM_DEFAULT_Q; - TABLE_NAME.cgemm_q = CGEMM_DEFAULT_Q; - TABLE_NAME.zgemm_q = ZGEMM_DEFAULT_Q; } #else // (ARCH_LOONGARCH64) #if (ARCH_POWER) diff --git a/param.h b/param.h index fef3a09914..9c85a3aeeb 100644 --- a/param.h +++ b/param.h @@ -2836,7 +2836,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define SNUMOPT 2 #define DNUMOPT 2 -#define GEMM_DEFAULT_OFFSET_A 0 +#define GEMM_DEFAULT_OFFSET_A 0x20000 #define GEMM_DEFAULT_OFFSET_B 0 #define GEMM_DEFAULT_ALIGN 0x0ffffUL @@ -2866,20 +2866,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define QGEMM_DEFAULT_UNROLL_M 2 #define XGEMM_DEFAULT_UNROLL_M 1 -#define SGEMM_DEFAULT_P 256 -#define DGEMM_DEFAULT_P 32 +#define SGEMM_DEFAULT_P sgemm_p +#define DGEMM_DEFAULT_P dgemm_p #define CGEMM_DEFAULT_P 128 -#define ZGEMM_DEFAULT_P 128 +#define ZGEMM_DEFAULT_P zgemm_p -#define SGEMM_DEFAULT_R 1024 -#define DGEMM_DEFAULT_R 858 +#define SGEMM_DEFAULT_R sgemm_r +#define DGEMM_DEFAULT_R dgemm_r #define CGEMM_DEFAULT_R 4096 -#define ZGEMM_DEFAULT_R 4096 +#define ZGEMM_DEFAULT_R zgemm_r -#define SGEMM_DEFAULT_Q 256 -#define DGEMM_DEFAULT_Q 152 +#define SGEMM_DEFAULT_Q sgemm_q +#define DGEMM_DEFAULT_Q dgemm_q #define CGEMM_DEFAULT_Q 128 -#define ZGEMM_DEFAULT_Q 128 +#define ZGEMM_DEFAULT_Q zgemm_q #define SYMV_P 16 #endif From 1c13cda3fc69bf6b3794970eded0fc1dc540e3d6 Mon Sep 17 00:00:00 2001 From: Chip Kerchner Date: Wed, 10 Apr 2024 15:16:47 -0500 Subject: [PATCH 025/267] Remove -openmp flag from XLF (since it doesn't support it). --- Makefile.system | 3 --- 1 file changed, 3 deletions(-) diff --git a/Makefile.system b/Makefile.system index f89cc8bdfa..902b536052 100644 --- a/Makefile.system +++ b/Makefile.system @@ -1195,9 +1195,6 @@ endif else FCOMMON_OPT += -q32 endif -ifeq ($(USE_OPENMP), 1) -FCOMMON_OPT += -openmp -endif endif ifeq ($(F_COMPILER), PGI) From 4d96e0ce1891f0a855160e07a7714242e30172e4 Mon Sep 17 00:00:00 2001 From: Matti Picus Date: Thu, 11 Apr 2024 10:36:30 +1000 Subject: [PATCH 026/267] remove extraneous suffix from shared object SONAME --- exports/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exports/Makefile b/exports/Makefile index 4d929c8d3c..33c47d4778 100644 --- a/exports/Makefile +++ b/exports/Makefile @@ -178,7 +178,7 @@ FEXTRALIB += -lm EXTRALIB += -lm else ifeq ($(FIXED_LIBNAME),1) -INTERNALNAME = $(LIBPREFIX)$(LIBNAMESUFFIX).so +INTERNALNAME = $(LIBPREFIX).so else INTERNALNAME = $(LIBPREFIX).so.$(MAJOR_VERSION) endif From d9e2db373514d7dfd7fb7723907597c35829d429 Mon Sep 17 00:00:00 2001 From: gxw Date: Fri, 12 Apr 2024 11:32:27 +0800 Subject: [PATCH 027/267] CI: add openblas_utest_ext for c910v, mips64 and loongarch64 --- .github/workflows/c910v.yml | 1 + .github/workflows/loongarch64.yml | 1 + .github/workflows/mips64.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/c910v.yml b/.github/workflows/c910v.yml index 68ba2ddd72..a47ca1dce6 100644 --- a/.github/workflows/c910v.yml +++ b/.github/workflows/c910v.yml @@ -84,6 +84,7 @@ jobs: run: | export PATH=$GITHUB_WORKSPACE/qemu-install/bin/:$PATH qemu-riscv64 ./utest/openblas_utest + qemu-riscv64 ./utest/openblas_utest_ext OPENBLAS_NUM_THREADS=2 qemu-riscv64 ./ctest/xscblat1 OPENBLAS_NUM_THREADS=2 qemu-riscv64 ./ctest/xdcblat1 OPENBLAS_NUM_THREADS=2 qemu-riscv64 ./ctest/xccblat1 diff --git a/.github/workflows/loongarch64.yml b/.github/workflows/loongarch64.yml index f1bf8064c6..b23c81f965 100644 --- a/.github/workflows/loongarch64.yml +++ b/.github/workflows/loongarch64.yml @@ -77,6 +77,7 @@ jobs: - name: Test run: | qemu-loongarch64-static ./utest/openblas_utest + qemu-loongarch64-static ./utest/openblas_utest_ext OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xscblat1 OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xdcblat1 OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xccblat1 diff --git a/.github/workflows/mips64.yml b/.github/workflows/mips64.yml index 4686ba713a..1491aff78b 100644 --- a/.github/workflows/mips64.yml +++ b/.github/workflows/mips64.yml @@ -80,6 +80,7 @@ jobs: run: | export PATH=$GITHUB_WORKSPACE/qemu-install/bin/:$PATH qemu-mips64el ./utest/openblas_utest + qemu-mips64el ./utest/openblas_utest_ext OPENBLAS_NUM_THREADS=2 qemu-mips64el ./ctest/xscblat1 OPENBLAS_NUM_THREADS=2 qemu-mips64el ./ctest/xdcblat1 OPENBLAS_NUM_THREADS=2 qemu-mips64el ./ctest/xccblat1 From ca673ca7742c85c965769479311e96eda61775cb Mon Sep 17 00:00:00 2001 From: "Usui, Tetsuzo" Date: Fri, 12 Apr 2024 17:21:14 +0900 Subject: [PATCH 028/267] Add GEMM_PREFERED_SIZE parameter for Neoverse V1 --- param.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/param.h b/param.h index a0c72d1992..84e241fcd2 100644 --- a/param.h +++ b/param.h @@ -3547,8 +3547,10 @@ is a big desktop or server with abundant cache rather than a phone or embedded d #if defined(XDOUBLE) || defined(DOUBLE) #define SWITCH_RATIO 8 +#define GEMM_PREFERED_SIZE 4 #else #define SWITCH_RATIO 16 +#define GEMM_PREFERED_SIZE 8 #endif #define SGEMM_DEFAULT_UNROLL_M 16 From d26caff60ceefbccd2d19953a74cd8afa4f871de Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 12 Apr 2024 14:29:27 +0200 Subject: [PATCH 029/267] Add another object signature for classic flang --- f_check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/f_check b/f_check index 81f598ffad..93c5962dee 100755 --- a/f_check +++ b/f_check @@ -86,7 +86,7 @@ else vendor=CRAY openmp='-fopenmp' ;; - *Arm\ F90*) + *Arm\ F90*|*F90\ Flang*) vendor=FLANG openmp='-fopenmp' ;; From 7cfd433d0cf8139b11fcb540a68b6209ba93c736 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 12 Apr 2024 15:34:04 +0200 Subject: [PATCH 030/267] revert the C/Z NRM2 kernels to the base NEON kernel as well --- kernel/arm64/KERNEL.NEOVERSEN1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/arm64/KERNEL.NEOVERSEN1 b/kernel/arm64/KERNEL.NEOVERSEN1 index 9fe981c585..5b31744730 100644 --- a/kernel/arm64/KERNEL.NEOVERSEN1 +++ b/kernel/arm64/KERNEL.NEOVERSEN1 @@ -93,8 +93,8 @@ IZAMAXKERNEL = izamax_thunderx2t99.c SNRM2KERNEL = nrm2.S DNRM2KERNEL = nrm2.S -CNRM2KERNEL = scnrm2_thunderx2t99.c -ZNRM2KERNEL = dznrm2_thunderx2t99.c +CNRM2KERNEL = znrm2.S +ZNRM2KERNEL = znrm2.S DDOTKERNEL = dot.c SDOTKERNEL = dot.c From 00ee5d0367cd59a3b1cdd1f36dae101d05495be3 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 12 Apr 2024 15:59:45 +0200 Subject: [PATCH 031/267] On ARM, do not assume -marm by default if OS_EMBEDDED=1 --- Makefile.system | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile.system b/Makefile.system index 902b536052..a4f41236b6 100644 --- a/Makefile.system +++ b/Makefile.system @@ -811,8 +811,10 @@ ifeq ($(ARCH), arm) NO_BINARY_MODE = 1 BINARY_DEFINED = 1 +ifneq ($(OS_EMBEDDED), 1) CCOMMON_OPT += -marm FCOMMON_OPT += -marm +endif # If softfp abi is mentioned on the command line, force it. ifeq ($(ARM_SOFTFP_ABI), 1) From b883526a34ff49e43b9d719da64e2db47fbe7f5c Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 13 Apr 2024 12:26:35 +0200 Subject: [PATCH 032/267] Fix uninitialized variables in the extensions utest --- utest/test_extensions/test_damin.c | 2 +- utest/test_extensions/test_dsum.c | 4 +++- utest/test_extensions/test_dzsum.c | 4 +++- utest/test_extensions/test_scsum.c | 4 +++- utest/test_extensions/test_ssum.c | 4 +++- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/utest/test_extensions/test_damin.c b/utest/test_extensions/test_damin.c index 736921fa3b..50bc5a928e 100644 --- a/utest/test_extensions/test_damin.c +++ b/utest/test_extensions/test_damin.c @@ -351,4 +351,4 @@ CTEST(damin, negative_step_2_N_70){ double amin = BLASFUNC(damin)(&N, x, &inc); ASSERT_DBL_NEAR_TOL(1.0, amin, DOUBLE_EPS); } -#endif \ No newline at end of file +#endif diff --git a/utest/test_extensions/test_dsum.c b/utest/test_extensions/test_dsum.c index e987c5a42a..304619d0a9 100644 --- a/utest/test_extensions/test_dsum.c +++ b/utest/test_extensions/test_dsum.c @@ -62,6 +62,7 @@ CTEST(dsum, step_zero){ blasint i; blasint N = ELEMENTS, inc = 0; double x[ELEMENTS]; + x[0]=0.; for (i = 0; i < N * inc; i ++) { x[i] = i + 1000; } @@ -243,6 +244,7 @@ CTEST(dsum, c_api_step_zero){ blasint i; blasint N = ELEMENTS, inc = 0; double x[ELEMENTS]; + x[0]=0.; for (i = 0; i < N * inc; i ++) { x[i] = i + 1000; } @@ -400,4 +402,4 @@ CTEST(dsum, c_api_step_2_N_50){ double sum = cblas_dsum(N, x, inc); ASSERT_DBL_NEAR_TOL(50.0, sum, DOUBLE_EPS); } -#endif \ No newline at end of file +#endif diff --git a/utest/test_extensions/test_dzsum.c b/utest/test_extensions/test_dzsum.c index 5139f59cbf..a184ad9475 100644 --- a/utest/test_extensions/test_dzsum.c +++ b/utest/test_extensions/test_dzsum.c @@ -62,6 +62,7 @@ CTEST(dzsum, step_zero){ blasint i; blasint N = ELEMENTS, inc = 0; double x[ELEMENTS]; + x[0] = 0.0; for (i = 0; i < N * inc * 2; i ++) { x[i] = i + 1000; } @@ -243,6 +244,7 @@ CTEST(dzsum, c_api_step_zero){ blasint i; blasint N = ELEMENTS, inc = 0; double x[ELEMENTS]; + x[0] = 0.0; for (i = 0; i < N * inc * 2; i ++) { x[i] = i + 1000; } @@ -400,4 +402,4 @@ CTEST(dzsum, c_api_step_2_N_50){ double sum = cblas_dzsum(N, x, inc); ASSERT_DBL_NEAR_TOL(0.0, sum, DOUBLE_EPS); } -#endif \ No newline at end of file +#endif diff --git a/utest/test_extensions/test_scsum.c b/utest/test_extensions/test_scsum.c index 492e1a4ca1..8e943de229 100644 --- a/utest/test_extensions/test_scsum.c +++ b/utest/test_extensions/test_scsum.c @@ -62,6 +62,7 @@ CTEST(scsum, step_zero){ blasint i; blasint N = ELEMENTS, inc = 0; float x[ELEMENTS]; + x[0] = 0.0f; for (i = 0; i < N * inc * 2; i ++) { x[i] = i + 1000; } @@ -243,6 +244,7 @@ CTEST(scsum, c_api_step_zero){ blasint i; blasint N = ELEMENTS, inc = 0; float x[ELEMENTS]; + x[0] = 0.0f; for (i = 0; i < N * inc * 2; i ++) { x[i] = i + 1000; } @@ -400,4 +402,4 @@ CTEST(scsum, c_api_step_2_N_50){ float sum = cblas_scsum(N, x, inc); ASSERT_DBL_NEAR_TOL(0.0f, sum, SINGLE_EPS); } -#endif \ No newline at end of file +#endif diff --git a/utest/test_extensions/test_ssum.c b/utest/test_extensions/test_ssum.c index 971a0d2e07..9c5e297286 100644 --- a/utest/test_extensions/test_ssum.c +++ b/utest/test_extensions/test_ssum.c @@ -62,6 +62,7 @@ CTEST(ssum, step_zero){ blasint i; blasint N = ELEMENTS, inc = 0; float x[ELEMENTS]; + x[0] = 0.0f; for (i = 0; i < N * inc; i ++) { x[i] = i + 1000; } @@ -243,6 +244,7 @@ CTEST(ssum, c_api_step_zero){ blasint i; blasint N = ELEMENTS, inc = 0; float x[ELEMENTS]; + x[0] = 0.0f; for (i = 0; i < N * inc; i ++) { x[i] = i + 1000; } @@ -400,4 +402,4 @@ CTEST(ssum, c_api_step_2_N_50){ float sum = cblas_ssum(N, x, inc); ASSERT_DBL_NEAR_TOL(50.0f, sum, SINGLE_EPS); } -#endif \ No newline at end of file +#endif From 4c03ed437f3a021dfd21ee470b4f0191f32d9885 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sun, 14 Apr 2024 15:39:11 +0200 Subject: [PATCH 033/267] Fix SICORTEX ASUM/ZASUM and SUM/ZSUM for INCX <=0 (#4640) * Exit early if INCX <= 0 --- kernel/mips64/asum.S | 1 + kernel/mips64/sum.S | 1 + kernel/mips64/zasum.S | 1 + kernel/mips64/zsum.S | 1 + 4 files changed, 4 insertions(+) diff --git a/kernel/mips64/asum.S b/kernel/mips64/asum.S index 2bf95c65d2..4e75529f52 100644 --- a/kernel/mips64/asum.S +++ b/kernel/mips64/asum.S @@ -73,6 +73,7 @@ MTC $0, s1 MTC $0, s2 + blez INCX, .L999 dsll INCX, INCX, BASE_SHIFT blez N, .L999 diff --git a/kernel/mips64/sum.S b/kernel/mips64/sum.S index 261630d49d..725346ff62 100644 --- a/kernel/mips64/sum.S +++ b/kernel/mips64/sum.S @@ -73,6 +73,7 @@ MTC $0, s1 MTC $0, s2 + blez INCX, .L999 dsll INCX, INCX, BASE_SHIFT blez N, .L999 diff --git a/kernel/mips64/zasum.S b/kernel/mips64/zasum.S index cd22f98414..a7af6918da 100644 --- a/kernel/mips64/zasum.S +++ b/kernel/mips64/zasum.S @@ -73,6 +73,7 @@ MTC $0, s1 MTC $0, s2 + blez INCX, .L999 dsll INCX, INCX, ZBASE_SHIFT blez N, .L999 diff --git a/kernel/mips64/zsum.S b/kernel/mips64/zsum.S index 129b97900e..83c5393197 100644 --- a/kernel/mips64/zsum.S +++ b/kernel/mips64/zsum.S @@ -73,6 +73,7 @@ MTC $0, s1 MTC $0, s2 + blez INCX, .L999 dsll INCX, INCX, ZBASE_SHIFT blez N, .L999 From 9c39e969f5ae23fe7ddd489a314f151b56caa502 Mon Sep 17 00:00:00 2001 From: gxw Date: Mon, 15 Apr 2024 15:17:29 +0800 Subject: [PATCH 034/267] mips64: Fixed MSA optimization bugs for zgemv and cgemv --- kernel/mips/cgemv_t_msa.c | 81 ++++++++++++++++++++------------------- kernel/mips/zgemv_t_msa.c | 43 ++++++++++----------- 2 files changed, 62 insertions(+), 62 deletions(-) diff --git a/kernel/mips/cgemv_t_msa.c b/kernel/mips/cgemv_t_msa.c index 800667b6ea..f05c42bd8c 100644 --- a/kernel/mips/cgemv_t_msa.c +++ b/kernel/mips/cgemv_t_msa.c @@ -31,27 +31,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #undef OP0 #undef OP1 #undef OP2 +#undef OP3 +#undef OP4 +#undef OP5 + +#if (!defined(CONJ) && !defined(XCONJ)) || (defined(CONJ) && defined(XCONJ)) + #define OP0 -= + #define OP1 += + #define OP2 += +#else + #define OP0 += + #define OP1 += + #define OP2 -= +#endif -#if !defined(CONJ) - #if !defined(XCONJ) - #define OP0 -= - #define OP1 += - #define OP2 += - #else - #define OP0 += - #define OP1 += - #define OP2 -= - #endif +#if !defined(XCONJ) + #define OP3 -= + #define OP4 += + #define OP5 += #else - #if !defined(XCONJ) - #define OP0 += - #define OP1 -= - #define OP2 += - #else - #define OP0 -= - #define OP1 -= - #define OP2 -= - #endif + #define OP3 += + #define OP4 -= + #define OP5 += #endif #define CGEMV_T_8x4() \ @@ -268,22 +269,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. res3i = y[3 * inc_y2 + 1]; \ \ res0r += alphar * temp0r; \ - res0r OP0 alphai * temp0i; \ + res0r OP3 alphai * temp0i; \ res1r += alphar * temp1r; \ - res1r OP0 alphai * temp1i; \ + res1r OP3 alphai * temp1i; \ res2r += alphar * temp2r; \ - res2r OP0 alphai * temp2i; \ + res2r OP3 alphai * temp2i; \ res3r += alphar * temp3r; \ - res3r OP0 alphai * temp3i; \ + res3r OP3 alphai * temp3i; \ \ - res0i OP1 alphar * temp0i; \ - res0i OP2 alphai * temp0r; \ - res1i OP1 alphar * temp1i; \ - res1i OP2 alphai * temp1r; \ - res2i OP1 alphar * temp2i; \ - res2i OP2 alphai * temp2r; \ - res3i OP1 alphar * temp3i; \ - res3i OP2 alphai * temp3r; \ + res0i OP4 alphar * temp0i; \ + res0i OP5 alphai * temp0r; \ + res1i OP4 alphar * temp1i; \ + res1i OP5 alphai * temp1r; \ + res2i OP4 alphar * temp2i; \ + res2i OP5 alphai * temp2r; \ + res3i OP4 alphar * temp3i; \ + res3i OP5 alphai * temp3r; \ \ y[0 * inc_y2] = res0r; \ y[1 * inc_y2] = res1r; \ @@ -303,14 +304,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. res1i = y[1 * inc_y2 + 1]; \ \ res0r += alphar * temp0r; \ - res0r OP0 alphai * temp0i; \ + res0r OP3 alphai * temp0i; \ res1r += alphar * temp1r; \ - res1r OP0 alphai * temp1i; \ + res1r OP3 alphai * temp1i; \ \ - res0i OP1 alphar * temp0i; \ - res0i OP2 alphai * temp0r; \ - res1i OP1 alphar * temp1i; \ - res1i OP2 alphai * temp1r; \ + res0i OP4 alphar * temp0i; \ + res0i OP5 alphai * temp0r; \ + res1i OP4 alphar * temp1i; \ + res1i OP5 alphai * temp1r; \ \ y[0 * inc_y2] = res0r; \ y[1 * inc_y2] = res1r; \ @@ -324,10 +325,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. res0i = y[0 * inc_y2 + 1]; \ \ res0r += alphar * temp0r; \ - res0r OP0 alphai * temp0i; \ + res0r OP3 alphai * temp0i; \ \ - res0i OP1 alphar * temp0i; \ - res0i OP2 alphai * temp0r; \ + res0i OP4 alphar * temp0i; \ + res0i OP5 alphai * temp0r; \ \ y[0 * inc_y2] = res0r; \ y[0 * inc_y2 + 1] = res0i; \ diff --git a/kernel/mips/zgemv_t_msa.c b/kernel/mips/zgemv_t_msa.c index 6492f90bed..022bc60101 100644 --- a/kernel/mips/zgemv_t_msa.c +++ b/kernel/mips/zgemv_t_msa.c @@ -33,27 +33,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #undef OP2 #undef OP3 #undef OP4 +#undef OP5 -#if !defined(CONJ) - #if !defined(XCONJ) - #define OP0 -= - #define OP1 += - #define OP2 += - #else - #define OP0 += - #define OP1 += - #define OP2 -= - #endif +#if (!defined(CONJ) && !defined(XCONJ)) || (defined(CONJ) && defined(XCONJ)) + #define OP0 -= + #define OP1 += + #define OP2 += #else - #if !defined(XCONJ) - #define OP0 += - #define OP1 -= - #define OP2 += - #else - #define OP0 -= - #define OP1 -= - #define OP2 -= - #endif + #define OP0 += + #define OP1 += + #define OP2 -= +#endif + +#if !defined(XCONJ) + #define OP3 -= + #define OP4 += + #define OP5 += +#else + #define OP3 += + #define OP4 -= + #define OP5 += #endif #define ZGEMV_T_8x1() \ @@ -124,10 +123,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. res0i = y[0 * inc_y2 + 1]; \ \ res0r += alphar * temp0r; \ - res0r OP0 alphai * temp0i; \ + res0r OP3 alphai * temp0i; \ \ - res0i OP1 alphar * temp0i; \ - res0i OP2 alphai * temp0r; \ + res0i OP4 alphar * temp0i; \ + res0i OP5 alphai * temp0r; \ \ y[0 * inc_y2] = res0r; \ y[0 * inc_y2 + 1] = res0i; \ From c20caa49608feb0ecf5fc5926cd9216b11350c2f Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Mon, 15 Apr 2024 22:42:53 +0200 Subject: [PATCH 035/267] Fix uninitialized variables in the LAPACK tests for the new ?QP3RK functions (Reference-LAPACK PR961) --- lapack-netlib/TESTING/LIN/cchkqp3rk.f | 91 +++++++++------------------ lapack-netlib/TESTING/LIN/dchkqp3rk.f | 91 +++++++++------------------ lapack-netlib/TESTING/LIN/schkqp3rk.f | 91 +++++++++------------------ lapack-netlib/TESTING/LIN/zchkqp3rk.f | 91 +++++++++------------------ 4 files changed, 112 insertions(+), 252 deletions(-) diff --git a/lapack-netlib/TESTING/LIN/cchkqp3rk.f b/lapack-netlib/TESTING/LIN/cchkqp3rk.f index 79d6add72e..b794d4664c 100644 --- a/lapack-netlib/TESTING/LIN/cchkqp3rk.f +++ b/lapack-netlib/TESTING/LIN/cchkqp3rk.f @@ -608,6 +608,9 @@ SUBROUTINE CCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, CALL CLACPY( 'All', M, NRHS, COPYB, LDA, $ B, LDA ) CALL ICOPY( N, IWORK( 1 ), 1, IWORK( N+1 ), 1 ) + DO I = 1, NTESTS + RESULT( I ) = ZERO + END DO * ABSTOL = -1.0 RELTOl = -1.0 @@ -652,16 +655,6 @@ SUBROUTINE CCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, RESULT( 1 ) = CQRT12( M, N, A, LDA, S, WORK, $ LWORK , RWORK ) * - DO T = 1, 1 - IF( RESULT( T ).GE.THRESH ) THEN - IF( NFAIL.EQ.0 .AND. NERRS.EQ.0 ) - $ CALL ALAHD( NOUT, PATH ) - WRITE( NOUT, FMT = 9999 ) 'CGEQP3RK', M, N, - $ NRHS, KMAX, ABSTOL, RELTOL, NB, NX, - $ IMAT, T, RESULT( T ) - NFAIL = NFAIL + 1 - END IF - END DO NRUN = NRUN + 1 * * End test 1 @@ -675,7 +668,7 @@ SUBROUTINE CCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, * 1-norm( A*P - Q*R ) / ( max(M,N) * 1-norm(A) * EPS ) * RESULT( 2 ) = CQPT01( M, N, KFACT, COPYA, A, LDA, TAU, - $ IWORK( N+1 ), WORK, LWORK ) + $ IWORK( N+1 ), WORK, LWORK ) * * Compute test 3: * @@ -684,21 +677,8 @@ SUBROUTINE CCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, * 1-norm( Q**T * Q - I ) / ( M * EPS ) * RESULT( 3 ) = CQRT11( M, KFACT, A, LDA, TAU, WORK, - $ LWORK ) + $ LWORK ) * -* Print information about the tests that did not pass -* the threshold. -* - DO T = 2, 3 - IF( RESULT( T ).GE.THRESH ) THEN - IF( NFAIL.EQ.0 .AND. NERRS.EQ.0 ) - $ CALL ALAHD( NOUT, PATH ) - WRITE( NOUT, FMT = 9999 ) 'CGEQP3RK', M, N, - $ NRHS, KMAX, ABSTOL, RELTOL, - $ NB, NX, IMAT, T, RESULT( T ) - NFAIL = NFAIL + 1 - END IF - END DO NRUN = NRUN + 2 * * Compute test 4: @@ -717,8 +697,8 @@ SUBROUTINE CCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, * DO J = 1, KFACT-1, 1 * - DTEMP = (( ABS( A( (J-1)*M+J ) ) - - $ ABS( A( (J)*M+J+1 ) ) ) / + DTEMP = (( ABS( A( (J-1)*LDA+J ) ) - + $ ABS( A( (J)*LDA+J+1 ) ) ) / $ ABS( A(1) ) ) * IF( DTEMP.LT.ZERO ) THEN @@ -727,20 +707,6 @@ SUBROUTINE CCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, * END DO * -* Print information about the tests that did not -* pass the threshold. -* - DO T = 4, 4 - IF( RESULT( T ).GE.THRESH ) THEN - IF( NFAIL.EQ.0 .AND. NERRS.EQ.0 ) - $ CALL ALAHD( NOUT, PATH ) - WRITE( NOUT, FMT = 9999 ) 'CGEQP3RK', - $ M, N, NRHS, KMAX, ABSTOL, RELTOL, - $ NB, NX, IMAT, T, - $ RESULT( T ) - NFAIL = NFAIL + 1 - END IF - END DO NRUN = NRUN + 1 * * End test 4. @@ -762,42 +728,41 @@ SUBROUTINE CCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, * LWORK_MQR = MAX(1, NRHS) CALL CUNMQR( 'Left', 'Conjugate transpose', - $ M, NRHS, KFACT, A, LDA, TAU, B, LDA, - $ WORK, LWORK_MQR, INFO ) + $ M, NRHS, KFACT, A, LDA, TAU, B, LDA, + $ WORK, LWORK_MQR, INFO ) * DO I = 1, NRHS * * Compare N+J-th column of A and J-column of B. * CALL CAXPY( M, -CONE, A( ( N+I-1 )*LDA+1 ), 1, - $ B( ( I-1 )*LDA+1 ), 1 ) + $ B( ( I-1 )*LDA+1 ), 1 ) END DO * - RESULT( 5 ) = - $ ABS( - $ CLANGE( 'One-norm', M, NRHS, B, LDA, RDUMMY ) / - $ ( REAL( M )*SLAMCH( 'Epsilon' ) ) - $ ) -* -* Print information about the tests that did not pass -* the threshold. -* - DO T = 5, 5 - IF( RESULT( T ).GE.THRESH ) THEN - IF( NFAIL.EQ.0 .AND. NERRS.EQ.0 ) - $ CALL ALAHD( NOUT, PATH ) - WRITE( NOUT, FMT = 9999 ) 'CGEQP3RK', M, N, - $ NRHS, KMAX, ABSTOL, RELTOL, - $ NB, NX, IMAT, T, RESULT( T ) - NFAIL = NFAIL + 1 - END IF - END DO + RESULT( 5 ) = ABS( + $ CLANGE( 'One-norm', M, NRHS, B, LDA, RDUMMY ) / + $ ( REAL( M )*SLAMCH( 'Epsilon' ) ) ) +* NRUN = NRUN + 1 * * End compute test 5. * END IF * +* Print information about the tests that did not pass +* the threshold. +* + DO T = 1, NTESTS + IF( RESULT( T ).GE.THRESH ) THEN + IF( NFAIL.EQ.0 .AND. NERRS.EQ.0 ) + $ CALL ALAHD( NOUT, PATH ) + WRITE( NOUT, FMT = 9999 ) 'CGEQP3RK', M, N, + $ NRHS, KMAX, ABSTOL, RELTOL, + $ NB, NX, IMAT, T, RESULT( T ) + NFAIL = NFAIL + 1 + END IF + END DO +* * END DO KMAX = 1, MIN(M,N)+1 * END DO diff --git a/lapack-netlib/TESTING/LIN/dchkqp3rk.f b/lapack-netlib/TESTING/LIN/dchkqp3rk.f index 434d2067e2..5046e50a98 100644 --- a/lapack-netlib/TESTING/LIN/dchkqp3rk.f +++ b/lapack-netlib/TESTING/LIN/dchkqp3rk.f @@ -605,6 +605,9 @@ SUBROUTINE DCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, CALL DLACPY( 'All', M, NRHS, COPYB, LDA, $ B, LDA ) CALL ICOPY( N, IWORK( 1 ), 1, IWORK( N+1 ), 1 ) + DO I = 1, NTESTS + RESULT( I ) = ZERO + END DO * ABSTOL = -1.0 RELTOL = -1.0 @@ -648,16 +651,6 @@ SUBROUTINE DCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, RESULT( 1 ) = DQRT12( M, N, A, LDA, S, WORK, $ LWORK ) * - DO T = 1, 1 - IF( RESULT( T ).GE.THRESH ) THEN - IF( NFAIL.EQ.0 .AND. NERRS.EQ.0 ) - $ CALL ALAHD( NOUT, PATH ) - WRITE( NOUT, FMT = 9999 ) 'DGEQP3RK', M, N, - $ NRHS, KMAX, ABSTOL, RELTOL, NB, NX, - $ IMAT, T, RESULT( T ) - NFAIL = NFAIL + 1 - END IF - END DO NRUN = NRUN + 1 * * End test 1 @@ -671,7 +664,7 @@ SUBROUTINE DCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, * 1-norm( A*P - Q*R ) / ( max(M,N) * 1-norm(A) * EPS ) * RESULT( 2 ) = DQPT01( M, N, KFACT, COPYA, A, LDA, TAU, - $ IWORK( N+1 ), WORK, LWORK ) + $ IWORK( N+1 ), WORK, LWORK ) * * Compute test 3: * @@ -680,21 +673,8 @@ SUBROUTINE DCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, * 1-norm( Q**T * Q - I ) / ( M * EPS ) * RESULT( 3 ) = DQRT11( M, KFACT, A, LDA, TAU, WORK, - $ LWORK ) -* -* Print information about the tests that did not pass -* the threshold. + $ LWORK ) * - DO T = 2, 3 - IF( RESULT( T ).GE.THRESH ) THEN - IF( NFAIL.EQ.0 .AND. NERRS.EQ.0 ) - $ CALL ALAHD( NOUT, PATH ) - WRITE( NOUT, FMT = 9999 ) 'DGEQP3RK', M, N, - $ NRHS, KMAX, ABSTOL, RELTOL, - $ NB, NX, IMAT, T, RESULT( T ) - NFAIL = NFAIL + 1 - END IF - END DO NRUN = NRUN + 2 * * Compute test 4: @@ -713,8 +693,8 @@ SUBROUTINE DCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, * DO J = 1, KFACT-1, 1 - DTEMP = (( ABS( A( (J-1)*M+J ) ) - - $ ABS( A( (J)*M+J+1 ) ) ) / + DTEMP = (( ABS( A( (J-1)*LDA+J ) ) - + $ ABS( A( (J)*LDA+J+1 ) ) ) / $ ABS( A(1) ) ) * IF( DTEMP.LT.ZERO ) THEN @@ -723,20 +703,6 @@ SUBROUTINE DCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, * END DO * -* Print information about the tests that did not -* pass the threshold. -* - DO T = 4, 4 - IF( RESULT( T ).GE.THRESH ) THEN - IF( NFAIL.EQ.0 .AND. NERRS.EQ.0 ) - $ CALL ALAHD( NOUT, PATH ) - WRITE( NOUT, FMT = 9999 ) 'DGEQP3RK', - $ M, N, NRHS, KMAX, ABSTOL, RELTOL, - $ NB, NX, IMAT, T, - $ RESULT( T ) - NFAIL = NFAIL + 1 - END IF - END DO NRUN = NRUN + 1 * * End test 4. @@ -758,42 +724,41 @@ SUBROUTINE DCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, * LWORK_MQR = MAX(1, NRHS) CALL DORMQR( 'Left', 'Transpose', - $ M, NRHS, KFACT, A, LDA, TAU, B, LDA, - $ WORK, LWORK_MQR, INFO ) + $ M, NRHS, KFACT, A, LDA, TAU, B, LDA, + $ WORK, LWORK_MQR, INFO ) * DO I = 1, NRHS * * Compare N+J-th column of A and J-column of B. * CALL DAXPY( M, -ONE, A( ( N+I-1 )*LDA+1 ), 1, - $ B( ( I-1 )*LDA+1 ), 1 ) + $ B( ( I-1 )*LDA+1 ), 1 ) END DO * - RESULT( 5 ) = - $ ABS( - $ DLANGE( 'One-norm', M, NRHS, B, LDA, RDUMMY ) / - $ ( DBLE( M )*DLAMCH( 'Epsilon' ) ) - $ ) -* -* Print information about the tests that did not pass -* the threshold. -* - DO T = 5, 5 - IF( RESULT( T ).GE.THRESH ) THEN - IF( NFAIL.EQ.0 .AND. NERRS.EQ.0 ) - $ CALL ALAHD( NOUT, PATH ) - WRITE( NOUT, FMT = 9999 ) 'DGEQP3RK', M, N, - $ NRHS, KMAX, ABSTOL, RELTOL, - $ NB, NX, IMAT, T, RESULT( T ) - NFAIL = NFAIL + 1 - END IF - END DO + RESULT( 5 ) = ABS( + $ DLANGE( 'One-norm', M, NRHS, B, LDA, RDUMMY ) / + $ ( DBLE( M )*DLAMCH( 'Epsilon' ) ) ) +* NRUN = NRUN + 1 * * End compute test 5. * END IF * +* Print information about the tests that did not +* pass the threshold. +* + DO T = 1, NTESTS + IF( RESULT( T ).GE.THRESH ) THEN + IF( NFAIL.EQ.0 .AND. NERRS.EQ.0 ) + $ CALL ALAHD( NOUT, PATH ) + WRITE( NOUT, FMT = 9999 ) 'DGEQP3RK', M, N, + $ NRHS, KMAX, ABSTOL, RELTOL, NB, NX, + $ IMAT, T, RESULT( T ) + NFAIL = NFAIL + 1 + END IF + END DO +* * END DO KMAX = 1, MIN(M,N)+1 * END DO diff --git a/lapack-netlib/TESTING/LIN/schkqp3rk.f b/lapack-netlib/TESTING/LIN/schkqp3rk.f index 36cf9370ea..c5ce7ff609 100644 --- a/lapack-netlib/TESTING/LIN/schkqp3rk.f +++ b/lapack-netlib/TESTING/LIN/schkqp3rk.f @@ -604,6 +604,9 @@ SUBROUTINE SCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, CALL SLACPY( 'All', M, NRHS, COPYB, LDA, $ B, LDA ) CALL ICOPY( N, IWORK( 1 ), 1, IWORK( N+1 ), 1 ) + DO I = 1, NTESTS + RESULT( I ) = ZERO + END DO * ABSTOL = -1.0 RELTOL = -1.0 @@ -647,16 +650,6 @@ SUBROUTINE SCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, RESULT( 1 ) = SQRT12( M, N, A, LDA, S, WORK, $ LWORK ) * - DO T = 1, 1 - IF( RESULT( T ).GE.THRESH ) THEN - IF( NFAIL.EQ.0 .AND. NERRS.EQ.0 ) - $ CALL ALAHD( NOUT, PATH ) - WRITE( NOUT, FMT = 9999 ) 'SGEQP3RK', M, N, - $ NRHS, KMAX, ABSTOL, RELTOL, NB, NX, - $ IMAT, T, RESULT( T ) - NFAIL = NFAIL + 1 - END IF - END DO NRUN = NRUN + 1 * * End test 1 @@ -670,7 +663,7 @@ SUBROUTINE SCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, * 1-norm( A*P - Q*R ) / ( max(M,N) * 1-norm(A) * EPS ) * RESULT( 2 ) = SQPT01( M, N, KFACT, COPYA, A, LDA, TAU, - $ IWORK( N+1 ), WORK, LWORK ) + $ IWORK( N+1 ), WORK, LWORK ) * * Compute test 3: * @@ -679,21 +672,8 @@ SUBROUTINE SCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, * 1-norm( Q**T * Q - I ) / ( M * EPS ) * RESULT( 3 ) = SQRT11( M, KFACT, A, LDA, TAU, WORK, - $ LWORK ) + $ LWORK ) * -* Print information about the tests that did not pass -* the threshold. -* - DO T = 2, 3 - IF( RESULT( T ).GE.THRESH ) THEN - IF( NFAIL.EQ.0 .AND. NERRS.EQ.0 ) - $ CALL ALAHD( NOUT, PATH ) - WRITE( NOUT, FMT = 9999 ) 'SGEQP3RK', M, N, - $ NRHS, KMAX, ABSTOL, RELTOL, - $ NB, NX, IMAT, T, RESULT( T ) - NFAIL = NFAIL + 1 - END IF - END DO NRUN = NRUN + 2 * * Compute test 4: @@ -712,8 +692,8 @@ SUBROUTINE SCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, * DO J = 1, KFACT-1, 1 - DTEMP = (( ABS( A( (J-1)*M+J ) ) - - $ ABS( A( (J)*M+J+1 ) ) ) / + DTEMP = (( ABS( A( (J-1)*LDA+J ) ) - + $ ABS( A( (J)*LDA+J+1 ) ) ) / $ ABS( A(1) ) ) * IF( DTEMP.LT.ZERO ) THEN @@ -722,20 +702,6 @@ SUBROUTINE SCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, * END DO * -* Print information about the tests that did not -* pass the threshold. -* - DO T = 4, 4 - IF( RESULT( T ).GE.THRESH ) THEN - IF( NFAIL.EQ.0 .AND. NERRS.EQ.0 ) - $ CALL ALAHD( NOUT, PATH ) - WRITE( NOUT, FMT = 9999 ) 'SGEQP3RK', - $ M, N, NRHS, KMAX, ABSTOL, RELTOL, - $ NB, NX, IMAT, T, - $ RESULT( T ) - NFAIL = NFAIL + 1 - END IF - END DO NRUN = NRUN + 1 * * End test 4. @@ -757,42 +723,41 @@ SUBROUTINE SCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, * LWORK_MQR = MAX(1, NRHS) CALL SORMQR( 'Left', 'Transpose', - $ M, NRHS, KFACT, A, LDA, TAU, B, LDA, - $ WORK, LWORK_MQR, INFO ) + $ M, NRHS, KFACT, A, LDA, TAU, B, LDA, + $ WORK, LWORK_MQR, INFO ) * DO I = 1, NRHS * * Compare N+J-th column of A and J-column of B. * CALL SAXPY( M, -ONE, A( ( N+I-1 )*LDA+1 ), 1, - $ B( ( I-1 )*LDA+1 ), 1 ) + $ B( ( I-1 )*LDA+1 ), 1 ) END DO * - RESULT( 5 ) = - $ ABS( - $ SLANGE( 'One-norm', M, NRHS, B, LDA, RDUMMY ) / - $ ( REAL( M )*SLAMCH( 'Epsilon' ) ) - $ ) -* -* Print information about the tests that did not pass -* the threshold. -* - DO T = 5, 5 - IF( RESULT( T ).GE.THRESH ) THEN - IF( NFAIL.EQ.0 .AND. NERRS.EQ.0 ) - $ CALL ALAHD( NOUT, PATH ) - WRITE( NOUT, FMT = 9999 ) 'SGEQP3RK', M, N, - $ NRHS, KMAX, ABSTOL, RELTOL, - $ NB, NX, IMAT, T, RESULT( T ) - NFAIL = NFAIL + 1 - END IF - END DO + RESULT( 5 ) = ABS( + $ SLANGE( 'One-norm', M, NRHS, B, LDA, RDUMMY ) / + $ ( REAL( M )*SLAMCH( 'Epsilon' ) ) ) +* NRUN = NRUN + 1 * * End compute test 5. * END IF * +* Print information about the tests that did not pass +* the threshold. +* + DO T = 1, NTESTS + IF( RESULT( T ).GE.THRESH ) THEN + IF( NFAIL.EQ.0 .AND. NERRS.EQ.0 ) + $ CALL ALAHD( NOUT, PATH ) + WRITE( NOUT, FMT = 9999 ) 'SGEQP3RK', M, N, + $ NRHS, KMAX, ABSTOL, RELTOL, + $ NB, NX, IMAT, T, RESULT( T ) + NFAIL = NFAIL + 1 + END IF + END DO +* * END DO KMAX = 1, MIN(M,N)+1 * END DO diff --git a/lapack-netlib/TESTING/LIN/zchkqp3rk.f b/lapack-netlib/TESTING/LIN/zchkqp3rk.f index 302c7b1a87..5092058837 100644 --- a/lapack-netlib/TESTING/LIN/zchkqp3rk.f +++ b/lapack-netlib/TESTING/LIN/zchkqp3rk.f @@ -608,6 +608,9 @@ SUBROUTINE ZCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, CALL ZLACPY( 'All', M, NRHS, COPYB, LDA, $ B, LDA ) CALL ICOPY( N, IWORK( 1 ), 1, IWORK( N+1 ), 1 ) + DO I = 1, NTESTS + RESULT( I ) = ZERO + END DO * ABSTOL = -1.0 RELTOl = -1.0 @@ -652,16 +655,6 @@ SUBROUTINE ZCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, RESULT( 1 ) = ZQRT12( M, N, A, LDA, S, WORK, $ LWORK , RWORK ) * - DO T = 1, 1 - IF( RESULT( T ).GE.THRESH ) THEN - IF( NFAIL.EQ.0 .AND. NERRS.EQ.0 ) - $ CALL ALAHD( NOUT, PATH ) - WRITE( NOUT, FMT = 9999 ) 'ZGEQP3RK', M, N, - $ NRHS, KMAX, ABSTOL, RELTOL, NB, NX, - $ IMAT, T, RESULT( T ) - NFAIL = NFAIL + 1 - END IF - END DO NRUN = NRUN + 1 * * End test 1 @@ -675,7 +668,7 @@ SUBROUTINE ZCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, * 1-norm( A*P - Q*R ) / ( max(M,N) * 1-norm(A) * EPS ) * RESULT( 2 ) = ZQPT01( M, N, KFACT, COPYA, A, LDA, TAU, - $ IWORK( N+1 ), WORK, LWORK ) + $ IWORK( N+1 ), WORK, LWORK ) * * Compute test 3: * @@ -684,21 +677,8 @@ SUBROUTINE ZCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, * 1-norm( Q**T * Q - I ) / ( M * EPS ) * RESULT( 3 ) = ZQRT11( M, KFACT, A, LDA, TAU, WORK, - $ LWORK ) + $ LWORK ) * -* Print information about the tests that did not pass -* the threshold. -* - DO T = 2, 3 - IF( RESULT( T ).GE.THRESH ) THEN - IF( NFAIL.EQ.0 .AND. NERRS.EQ.0 ) - $ CALL ALAHD( NOUT, PATH ) - WRITE( NOUT, FMT = 9999 ) 'ZGEQP3RK', M, N, - $ NRHS, KMAX, ABSTOL, RELTOL, - $ NB, NX, IMAT, T, RESULT( T ) - NFAIL = NFAIL + 1 - END IF - END DO NRUN = NRUN + 2 * * Compute test 4: @@ -717,8 +697,8 @@ SUBROUTINE ZCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, * DO J = 1, KFACT-1, 1 * - DTEMP = (( ABS( A( (J-1)*M+J ) ) - - $ ABS( A( (J)*M+J+1 ) ) ) / + DTEMP = (( ABS( A( (J-1)*LDA+J ) ) - + $ ABS( A( (J)*LDA+J+1 ) ) ) / $ ABS( A(1) ) ) * IF( DTEMP.LT.ZERO ) THEN @@ -727,20 +707,6 @@ SUBROUTINE ZCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, * END DO * -* Print information about the tests that did not -* pass the threshold. -* - DO T = 4, 4 - IF( RESULT( T ).GE.THRESH ) THEN - IF( NFAIL.EQ.0 .AND. NERRS.EQ.0 ) - $ CALL ALAHD( NOUT, PATH ) - WRITE( NOUT, FMT = 9999 ) 'ZGEQP3RK', - $ M, N, NRHS, KMAX, ABSTOL, RELTOL, - $ NB, NX, IMAT, T, - $ RESULT( T ) - NFAIL = NFAIL + 1 - END IF - END DO NRUN = NRUN + 1 * * End test 4. @@ -762,42 +728,41 @@ SUBROUTINE ZCHKQP3RK( DOTYPE, NM, MVAL, NN, NVAL, NNS, NSVAL, * LWORK_MQR = MAX(1, NRHS) CALL ZUNMQR( 'Left', 'Conjugate transpose', - $ M, NRHS, KFACT, A, LDA, TAU, B, LDA, - $ WORK, LWORK_MQR, INFO ) + $ M, NRHS, KFACT, A, LDA, TAU, B, LDA, + $ WORK, LWORK_MQR, INFO ) * DO I = 1, NRHS * * Compare N+J-th column of A and J-column of B. * CALL ZAXPY( M, -CONE, A( ( N+I-1 )*LDA+1 ), 1, - $ B( ( I-1 )*LDA+1 ), 1 ) + $ B( ( I-1 )*LDA+1 ), 1 ) END DO * - RESULT( 5 ) = - $ ABS( - $ ZLANGE( 'One-norm', M, NRHS, B, LDA, RDUMMY ) / - $ ( DBLE( M )*DLAMCH( 'Epsilon' ) ) - $ ) -* -* Print information about the tests that did not pass -* the threshold. -* - DO T = 5, 5 - IF( RESULT( T ).GE.THRESH ) THEN - IF( NFAIL.EQ.0 .AND. NERRS.EQ.0 ) - $ CALL ALAHD( NOUT, PATH ) - WRITE( NOUT, FMT = 9999 ) 'ZGEQP3RK', M, N, - $ NRHS, KMAX, ABSTOL, RELTOL, - $ NB, NX, IMAT, T, RESULT( T ) - NFAIL = NFAIL + 1 - END IF - END DO + RESULT( 5 ) = ABS( + $ ZLANGE( 'One-norm', M, NRHS, B, LDA, RDUMMY ) / + $ ( DBLE( M )*DLAMCH( 'Epsilon' ) ) ) +* NRUN = NRUN + 1 * * End compute test 5. * END IF * +* Print information about the tests that did not pass +* the threshold. +* + DO T = 1, NTESTS + IF( RESULT( T ).GE.THRESH ) THEN + IF( NFAIL.EQ.0 .AND. NERRS.EQ.0 ) + $ CALL ALAHD( NOUT, PATH ) + WRITE( NOUT, FMT = 9999 ) 'ZGEQP3RK', M, N, + $ NRHS, KMAX, ABSTOL, RELTOL, + $ NB, NX, IMAT, T, RESULT( T ) + NFAIL = NFAIL + 1 + END IF + END DO +* * END DO KMAX = 1, MIN(M,N)+1 * END DO From 1d86e21cda4d1ca439291c7098127d3189fede9f Mon Sep 17 00:00:00 2001 From: Chip-Kerchner Date: Tue, 16 Apr 2024 15:18:19 -0500 Subject: [PATCH 036/267] Change xlf compiler to xlf_r in README for AIX. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a37459b82f..b52e485647 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,7 @@ Please read `GotoBLAS_01Readme.txt` for older CPU models already supported by th - **AIX**: Dynamic architecture with OpenXL and OpenMP. ```sh - make CC=ibm-clang_r FC=xlf TARGET=POWER7 BINARY=64 USE_OPENMP=1 INTERFACE64=1 DYNAMIC_ARCH=1 USE_THREAD=1 + make CC=ibm-clang_r FC=xlf_r TARGET=POWER7 BINARY=64 USE_OPENMP=1 INTERFACE64=1 DYNAMIC_ARCH=1 USE_THREAD=1 ``` #### IBM zEnterprise System From c57f9326d65a7d2044ef80aa5cf0ecb93e317a7f Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 17 Apr 2024 08:25:17 +0200 Subject: [PATCH 037/267] Add implementation of WhereAmI() to support NO_AFFINITY=0 on ARM64 (#4648) * Add preliminary implementation of WhereAmI() --- common_arm64.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/common_arm64.h b/common_arm64.h index 6ae6a35a30..d80b9e4345 100644 --- a/common_arm64.h +++ b/common_arm64.h @@ -55,6 +55,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef ASSEMBLER +static __inline int WhereAmI(void){ + uint64_t ret; + __asm__ volatile ( + " mrs x0, mpidr_el1 \n" + " and x0, x0, 0xff \n" + :"=r" (ret) + :: "memory" + ); + ret +=1; + if ((int)ret <0) ret = 0; + return (int)ret; +} + static __inline void blas_lock(volatile BLASULONG *address){ BLASULONG ret; From 51ab1903e7f1698b63ca0239031ae57e7b2cc5ab Mon Sep 17 00:00:00 2001 From: yamazaki-mitsufumi Date: Thu, 18 Apr 2024 18:20:25 +0900 Subject: [PATCH 038/267] Expanding the scop of 2D thread distribution --- driver/level3/level3_thread.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/driver/level3/level3_thread.c b/driver/level3/level3_thread.c index c7ccf84260..9fec7afca2 100644 --- a/driver/level3/level3_thread.c +++ b/driver/level3/level3_thread.c @@ -826,6 +826,16 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, IFLOAT *sa, IF if (nthreads_m * nthreads_n > args -> nthreads) { nthreads_n = blas_quickdivide(args -> nthreads, nthreads_m); } + /* The nthreads_m and nthreads_n are adjusted so that the submatrix */ + /* to be handled by each thread preferably becomes a square matrix */ + /* by minimizing an objective function 'n * nthreads_m + m * nthreads_n'. */ + /* Objective function come from sum of partitions in m and n. */ + /* (n / nthreads_n) + (m / nthreads_m) */ + /* = (n * nthreads_m + m * nthreads_n) / (nthreads_n * nthreads_m) */ + while (nthreads_m % 2 == 0 && n * nthreads_m + m * nthreads_n > n * (nthreads_m / 2) + m * (nthreads_n * 2)) { + nthreads_m /= 2; + nthreads_n *= 2; + } } /* Execute serial or parallel computation */ From 930ebed48b80cee741c4a3139a764cfb26c6821a Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 18 Apr 2024 15:59:24 +0200 Subject: [PATCH 039/267] Add option EMBEDDED --- Makefile.rule | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile.rule b/Makefile.rule index 2e777cdc67..bb239be2a4 100644 --- a/Makefile.rule +++ b/Makefile.rule @@ -173,6 +173,10 @@ NO_AFFINITY = 1 # If you are compiling for Linux and you have more than 16 numa nodes or more than 256 cpus # BIGNUMA = 1 +# If you are compiling for an embedded system ("bare metal") like Cortex M series +# Note that you will have to provide implementations of malloc() and free() in this case +# EMBEDDED = 1 + # Don't use AVX kernel on Sandy Bridge. It is compatible with old compilers # and OS. However, the performance is low. # NO_AVX = 1 From d6b0badc058010c368aae55570c7d8ad8c925d4a Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 18 Apr 2024 16:06:21 +0200 Subject: [PATCH 040/267] Fix declarations for EMBEDDED --- Makefile.system | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile.system b/Makefile.system index a4f41236b6..70a7436e9b 100644 --- a/Makefile.system +++ b/Makefile.system @@ -811,9 +811,11 @@ ifeq ($(ARCH), arm) NO_BINARY_MODE = 1 BINARY_DEFINED = 1 -ifneq ($(OS_EMBEDDED), 1) +ifneq ($(EMBEDDED), 1) CCOMMON_OPT += -marm FCOMMON_OPT += -marm +else +CCOMMON_OPT += -DOS_EMBEDDED endif # If softfp abi is mentioned on the command line, force it. From 0ec0746ae40d17a4e9d3d08b8fb60afe15b15783 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 18 Apr 2024 16:11:20 +0200 Subject: [PATCH 041/267] Update Makefile.system --- Makefile.system | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.system b/Makefile.system index 70a7436e9b..1f1cfd16c2 100644 --- a/Makefile.system +++ b/Makefile.system @@ -815,7 +815,7 @@ ifneq ($(EMBEDDED), 1) CCOMMON_OPT += -marm FCOMMON_OPT += -marm else -CCOMMON_OPT += -DOS_EMBEDDED +CCOMMON_OPT += -DOS_EMBEDDED -mthumb -mcpu=cortex-m4 -mfloat-abi=hard -mfpu=fpv4-sp-d16 endif # If softfp abi is mentioned on the command line, force it. From a5ab76c75e2f5d842e3be832dcf1c9f88c861618 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 18 Apr 2024 16:44:04 +0200 Subject: [PATCH 042/267] Update .cirrus.yml --- .cirrus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index 7a13d123fc..e8a0b85c02 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -156,7 +156,7 @@ FreeBSD_task: image_family: freebsd-13-2 install_script: - pkg update -f && pkg upgrade -y && pkg install -y gmake gcc - - ln -s /usr/local/lib/gcc12/libgfortran.so.5.0.0 /usr/lib/libgfortran.so + - ln -s /usr/local/lib/gcc13/libgfortran.so.5.0.0 /usr/lib/libgfortran.so compile_script: - gmake CC=clang FC=gfortran USE_OPENMP=1 CPP_THREAD_SAFETY_TEST=1 From ca64861ce8a04ebe8da9eab484ad7fe7d36d4610 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20B=C3=B6sz=C3=B6rm=C3=A9nyi?= Date: Fri, 19 Apr 2024 10:52:28 +0200 Subject: [PATCH 043/267] Add forgotten conditional uses of PREFETCH MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes a (cross-)compilation/linker error for PRESCOTT on Yocto. Signed-off-by: Zoltán Böszörményi --- kernel/x86_64/gemm_ncopy_4.S | 18 ++++++++++++++ kernel/x86_64/gemm_tcopy_4.S | 10 ++++++++ kernel/x86_64/zgemm_kernel_4x2_sse.S | 32 +++++++++++++++++++++++-- kernel/x86_64/zsymv_L_sse2.S | 8 +++++++ kernel/x86_64/zsymv_U_sse2.S | 8 +++++++ kernel/x86_64/ztrsm_kernel_LN_4x2_sse.S | 12 ++++++++-- kernel/x86_64/ztrsm_kernel_LT_4x2_sse.S | 12 ++++++++-- kernel/x86_64/ztrsm_kernel_RT_4x2_sse.S | 12 ++++++++-- 8 files changed, 104 insertions(+), 8 deletions(-) diff --git a/kernel/x86_64/gemm_ncopy_4.S b/kernel/x86_64/gemm_ncopy_4.S index 7192cecc20..d30e9d362f 100644 --- a/kernel/x86_64/gemm_ncopy_4.S +++ b/kernel/x86_64/gemm_ncopy_4.S @@ -189,12 +189,16 @@ movss %xmm6, 6 * SIZE(B) movss %xmm7, 7 * SIZE(B) +#ifdef PREFETCH PREFETCH RPREFETCHSIZE * SIZE(AO1) PREFETCH RPREFETCHSIZE * SIZE(AO2) PREFETCH RPREFETCHSIZE * SIZE(AO3) PREFETCH RPREFETCHSIZE * SIZE(AO4) +#endif +#ifdef PREFETCHW PREFETCHW WPREFETCHSIZE * SIZE(B) +#endif movss %xmm8, 8 * SIZE(B) movss %xmm9, 9 * SIZE(B) @@ -205,29 +209,39 @@ movss %xmm14, 14 * SIZE(B) movss %xmm15, 15 * SIZE(B) #else +#ifdef PREFETCH PREFETCH RPREFETCHSIZE * SIZE(AO1) +#endif movsd 0 * SIZE(AO1), %xmm0 movhpd 0 * SIZE(AO2), %xmm0 movsd 1 * SIZE(AO1), %xmm2 movhpd 1 * SIZE(AO2), %xmm2 +#ifdef PREFETCH PREFETCH RPREFETCHSIZE * SIZE(AO2) +#endif movsd 2 * SIZE(AO1), %xmm4 movhpd 2 * SIZE(AO2), %xmm4 movsd 3 * SIZE(AO1), %xmm6 movhpd 3 * SIZE(AO2), %xmm6 +#ifdef PREFETCH PREFETCH RPREFETCHSIZE * SIZE(AO3) +#endif movsd 0 * SIZE(AO3), %xmm1 movhpd 0 * SIZE(AO4), %xmm1 movsd 1 * SIZE(AO3), %xmm3 movhpd 1 * SIZE(AO4), %xmm3 +#ifdef PREFETCH PREFETCH RPREFETCHSIZE * SIZE(AO4) +#endif movsd 2 * SIZE(AO3), %xmm5 movhpd 2 * SIZE(AO4), %xmm5 movsd 3 * SIZE(AO3), %xmm7 movhpd 3 * SIZE(AO4), %xmm7 +#ifdef PREFETCHW PREFETCHW WPREFETCHSIZE * SIZE(B) +#endif movapd %xmm0, 0 * SIZE(B) movapd %xmm1, 2 * SIZE(B) movapd %xmm2, 4 * SIZE(B) @@ -342,10 +356,14 @@ movapd %xmm3, 6 * SIZE(B) #endif +#ifdef PREFETCH PREFETCH RPREFETCHSIZE * SIZE(AO1) PREFETCH RPREFETCHSIZE * SIZE(AO2) +#endif +#ifdef PREFETCHW PREFETCHW WPREFETCHSIZE * SIZE(B) +#endif addq $4 * SIZE, AO1 addq $4 * SIZE, AO2 diff --git a/kernel/x86_64/gemm_tcopy_4.S b/kernel/x86_64/gemm_tcopy_4.S index ba7714b4bd..177587c4b7 100644 --- a/kernel/x86_64/gemm_tcopy_4.S +++ b/kernel/x86_64/gemm_tcopy_4.S @@ -219,31 +219,41 @@ movaps %xmm3, 12 * SIZE(BO) #else +#ifdef PREFETCH PREFETCH RPREFETCHSIZE * SIZE(AO1) +#endif movsd 0 * SIZE(AO1), %xmm0 movhpd 1 * SIZE(AO1), %xmm0 movsd 2 * SIZE(AO1), %xmm1 movhpd 3 * SIZE(AO1), %xmm1 +#ifdef PREFETCH PREFETCH RPREFETCHSIZE * SIZE(AO2) +#endif movsd 0 * SIZE(AO2), %xmm2 movhpd 1 * SIZE(AO2), %xmm2 movsd 2 * SIZE(AO2), %xmm3 movhpd 3 * SIZE(AO2), %xmm3 +#ifdef PREFETCH PREFETCH RPREFETCHSIZE * SIZE(AO3) +#endif movsd 0 * SIZE(AO3), %xmm4 movhpd 1 * SIZE(AO3), %xmm4 movsd 2 * SIZE(AO3), %xmm5 movhpd 3 * SIZE(AO3), %xmm5 +#ifdef PREFETCH PREFETCH RPREFETCHSIZE * SIZE(AO4) +#endif movsd 0 * SIZE(AO4), %xmm6 movhpd 1 * SIZE(AO4), %xmm6 movsd 2 * SIZE(AO4), %xmm7 movhpd 3 * SIZE(AO4), %xmm7 +#ifdef PREFETCHW PREFETCHW WPREFETCHSIZE * SIZE(BO) +#endif movapd %xmm0, 0 * SIZE(BO) movapd %xmm1, 2 * SIZE(BO) movapd %xmm2, 4 * SIZE(BO) diff --git a/kernel/x86_64/zgemm_kernel_4x2_sse.S b/kernel/x86_64/zgemm_kernel_4x2_sse.S index 7d606aa6f4..5841f8b9e0 100644 --- a/kernel/x86_64/zgemm_kernel_4x2_sse.S +++ b/kernel/x86_64/zgemm_kernel_4x2_sse.S @@ -102,6 +102,14 @@ #define RPREFETCHSIZE (8 * 7 + 4) #define WPREFETCHSIZE (8 * 8 + 4) +#ifdef PREFETCH +#define PREFETCH_KERNEL1(xx) PREFETCH (PREFETCHSIZE + 0) * SIZE + 1 * (xx) * SIZE(AO, %rax, 4) ; +#define PREFETCH_KERNEL4(xx) PREFETCH (PREFETCHSIZE + 16) * SIZE + 1 * (xx) * SIZE(AO, %rax, 4) ; +#else +#define PREFETCH_KERNEL1(xx) +#define PREFETCH_KERNEL4(xx) +#endif + #ifndef GENERIC #define KERNEL1(xx) \ mulps %xmm0, %xmm1 ;\ @@ -111,7 +119,7 @@ addps %xmm3, %xmm9 ;\ movaps -28 * SIZE + 2 * (xx) * SIZE(BO, %rax, 8), %xmm3 ;\ mulps %xmm0, %xmm5 ;\ - PREFETCH (PREFETCHSIZE + 0) * SIZE + 1 * (xx) * SIZE(AO, %rax, 4) ;\ + PREFETCH_KERNEL1(xx) \ mulps -20 * SIZE + 2 * (xx) * SIZE(BO, %rax, 8), %xmm0 ;\ addps %xmm5, %xmm10 ;\ movaps -24 * SIZE + 2 * (xx) * SIZE(BO, %rax, 8), %xmm5 ;\ @@ -157,7 +165,7 @@ mulps -4 * SIZE + 2 * (xx) * SIZE(BO, %rax, 8), %xmm6 ;\ addps %xmm5, %xmm14 ;\ movaps 8 * SIZE + 2 * (xx) * SIZE(BO, %rax, 8), %xmm5 ;\ - PREFETCH (PREFETCHSIZE + 16) * SIZE + 1 * (xx) * SIZE(AO, %rax, 4) ;\ + PREFETCH_KERNEL4(xx) \ addps %xmm6, %xmm15 ;\ movaps -4 * SIZE + 1 * (xx) * SIZE(AO, %rax, 4), %xmm6 @@ -1026,7 +1034,9 @@ .L22: mulps %xmm0, %xmm1 +#ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) +#endif addps %xmm1, %xmm8 movaps -28 * SIZE(BO), %xmm1 mulps %xmm0, %xmm1 @@ -1079,7 +1089,9 @@ movaps 0 * SIZE(AO), %xmm0 mulps %xmm2, %xmm1 +#ifdef PREFETCH PREFETCH (PREFETCHSIZE + 16) * SIZE(AO) +#endif addps %xmm1, %xmm8 movaps 36 * SIZE(BO), %xmm1 mulps %xmm2, %xmm1 @@ -1285,7 +1297,9 @@ .L32: mulps %xmm0, %xmm1 +#ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) +#endif addps %xmm1, %xmm8 movaps -28 * SIZE(BO), %xmm1 mulps %xmm0, %xmm1 @@ -1679,7 +1693,9 @@ .L52: mulps %xmm0, %xmm1 +#ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) +#endif mulps -28 * SIZE(BO), %xmm0 addps %xmm1, %xmm8 movaps -32 * SIZE(BO), %xmm1 @@ -1705,7 +1721,9 @@ addps %xmm0, %xmm13 movaps 32 * SIZE(AO), %xmm0 +#ifdef PREFETCH PREFETCH (PREFETCHSIZE + 16) * SIZE(AO) +#endif mulps %xmm2, %xmm3 mulps -12 * SIZE(BO), %xmm2 @@ -1733,7 +1751,9 @@ addps %xmm2, %xmm13 movaps 48 * SIZE(AO), %xmm2 +#ifdef PREFETCH PREFETCH (PREFETCHSIZE + 32) * SIZE(AO) +#endif mulps %xmm4, %xmm5 mulps 4 * SIZE(BO), %xmm4 @@ -1761,7 +1781,9 @@ addps %xmm4, %xmm13 movaps 64 * SIZE(AO), %xmm4 +#ifdef PREFETCH PREFETCH (PREFETCHSIZE + 48) * SIZE(AO) +#endif mulps %xmm6, %xmm7 mulps 20 * SIZE(BO), %xmm6 @@ -1942,7 +1964,9 @@ .L62: mulps %xmm0, %xmm1 +#ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) +#endif mulps -28 * SIZE(BO), %xmm0 addps %xmm1, %xmm8 movaps -24 * SIZE(BO), %xmm1 @@ -1968,7 +1992,9 @@ addps %xmm0, %xmm11 movaps 0 * SIZE(AO), %xmm0 +#ifdef PREFETCH PREFETCH (PREFETCHSIZE + 16) * SIZE(AO) +#endif mulps %xmm2, %xmm5 mulps 4 * SIZE(BO), %xmm2 @@ -2130,7 +2156,9 @@ .L72: mulps %xmm0, %xmm1 +#ifdef PREFETCH PREFETCH (PREFETCHSIZE + 0) * SIZE(AO) +#endif addps %xmm1, %xmm8 movaps -28 * SIZE(BO), %xmm1 mulps %xmm0, %xmm1 diff --git a/kernel/x86_64/zsymv_L_sse2.S b/kernel/x86_64/zsymv_L_sse2.S index fa61ac9393..acc5ad592b 100644 --- a/kernel/x86_64/zsymv_L_sse2.S +++ b/kernel/x86_64/zsymv_L_sse2.S @@ -484,7 +484,9 @@ addpd a1, yy1 MOVDDUP(1 * SIZE, A1, a1) +#ifdef PREFETCH PREFETCH PREFETCHSIZE(A1) +#endif movapd xtemp3, xt1 mulpd a2, xt1 @@ -507,7 +509,9 @@ addpd a2, yy2 MOVDDUP(0 * SIZE, A2, a2) +#ifdef PREFETCH PREFETCH PREFETCHSIZE(XX) +#endif movapd xtemp3, xt1 movapd 12 * SIZE(XX), xtemp3 @@ -546,7 +550,9 @@ addpd a2, yy1 MOVDDUP(6 * SIZE, A2, a2) +#ifdef PREFETCH PREFETCH PREFETCHSIZE(A2) +#endif movlpd yy1, 0 * SIZE(YY) movhpd yy1, 1 * SIZE(YY) @@ -574,7 +580,9 @@ addpd a1, yy1 MOVDDUP(6 * SIZE, A1, a1) +#ifdef PREFETCHW PREFETCHW PREFETCHSIZE(YY) +#endif movapd xtemp4, xt1 mulpd a2, xt1 diff --git a/kernel/x86_64/zsymv_U_sse2.S b/kernel/x86_64/zsymv_U_sse2.S index 1657885c0e..fa1fe9fe9e 100644 --- a/kernel/x86_64/zsymv_U_sse2.S +++ b/kernel/x86_64/zsymv_U_sse2.S @@ -442,7 +442,9 @@ addpd a1, yy1 MOVDDUP(3 * SIZE, A2, a1) +#ifdef PREFETCH PREFETCH PREFETCHSIZE(A1) +#endif movapd xtemp3, xt1 mulpd a2, xt1 @@ -465,7 +467,9 @@ addpd a1, yy2 MOVDDUP(3 * SIZE, A1, a1) +#ifdef PREFETCH PREFETCH PREFETCHSIZE(XX) +#endif movapd xtemp3, xt1 movapd 12 * SIZE(XX), xtemp3 @@ -504,7 +508,9 @@ addpd a2, yy1 MOVDDUP(5 * SIZE, A1, a2) +#ifdef PREFETCH PREFETCH PREFETCHSIZE(A2) +#endif movlpd yy1, 0 * SIZE(YY) movhpd yy1, 1 * SIZE(YY) @@ -532,7 +538,9 @@ addpd a2, yy1 MOVDDUP(4 * SIZE, A2, a2) +#ifdef PREFETCH PREFETCHW PREFETCHSIZE(YY) +#endif movapd xtemp4, xt1 mulpd a3, xt1 diff --git a/kernel/x86_64/ztrsm_kernel_LN_4x2_sse.S b/kernel/x86_64/ztrsm_kernel_LN_4x2_sse.S index cd86db289f..02b5098a23 100644 --- a/kernel/x86_64/ztrsm_kernel_LN_4x2_sse.S +++ b/kernel/x86_64/ztrsm_kernel_LN_4x2_sse.S @@ -109,12 +109,20 @@ #define PREFETCHSIZE (8 * 6 + 4) #endif +#ifdef PREFETCH +#define PREFETCH_KERNEL1(xx) PREFETCH (PREFETCHSIZE + 0) * SIZE + 1 * (xx) * SIZE(AO) ; +#define PREFETCH_KERNEL5(xx) PREFETCH (PREFETCHSIZE + 8) * SIZE + 1 * (xx) * SIZE(AO) ; +#else +#define PREFETCH_KERNEL1(xx) +#define PREFETCH_KERNEL5(xx) +#endif + #define KERNEL1(xx) \ mulps %xmm8, %xmm9 ;\ addps %xmm9, %xmm0 ;\ movaps 0 * SIZE + 2 * (xx) * SIZE(BO), %xmm9 ;\ mulps %xmm8, %xmm11 ;\ - PREFETCH (PREFETCHSIZE + 0) * SIZE + 1 * (xx) * SIZE(AO) ;\ + PREFETCH_KERNEL1(xx) \ addps %xmm11, %xmm1 ;\ movaps 4 * SIZE + 2 * (xx) * SIZE(BO), %xmm11 ;\ mulps %xmm8, %xmm13 ;\ @@ -171,7 +179,7 @@ addps %xmm9, %xmm0 ;\ movaps 32 * SIZE + 2 * (xx) * SIZE(BO), %xmm9 ;\ mulps %xmm8, %xmm11 ;\ - PREFETCH (PREFETCHSIZE + 8) * SIZE + 1 * (xx) * SIZE(AO) ;\ + PREFETCH_KERNEL5(xx) \ addps %xmm11, %xmm1 ;\ movaps 36 * SIZE + 2 * (xx) * SIZE(BO), %xmm11 ;\ mulps %xmm8, %xmm13 ;\ diff --git a/kernel/x86_64/ztrsm_kernel_LT_4x2_sse.S b/kernel/x86_64/ztrsm_kernel_LT_4x2_sse.S index 53e5bb7f94..0c3a052a17 100644 --- a/kernel/x86_64/ztrsm_kernel_LT_4x2_sse.S +++ b/kernel/x86_64/ztrsm_kernel_LT_4x2_sse.S @@ -109,12 +109,20 @@ #define PREFETCHSIZE (8 * 6 + 4) #endif +#ifdef PREFETCH +#define PREFETCH_KERNEL1(xx) PREFETCH (PREFETCHSIZE + 0) * SIZE + 1 * (xx) * SIZE(AO) ; +#define PREFETCH_KERNEL5(xx) PREFETCH (PREFETCHSIZE + 8) * SIZE + 1 * (xx) * SIZE(AO) ; +#else +#define PREFETCH_KERNEL1(xx) +#define PREFETCH_KERNEL5(xx) +#endif + #define KERNEL1(xx) \ mulps %xmm8, %xmm9 ;\ addps %xmm9, %xmm0 ;\ movaps 0 * SIZE + 2 * (xx) * SIZE(BO), %xmm9 ;\ mulps %xmm8, %xmm11 ;\ - PREFETCH (PREFETCHSIZE + 0) * SIZE + 1 * (xx) * SIZE(AO) ;\ + PREFETCH_KERNEL1(xx) \ addps %xmm11, %xmm1 ;\ movaps 4 * SIZE + 2 * (xx) * SIZE(BO), %xmm11 ;\ mulps %xmm8, %xmm13 ;\ @@ -171,7 +179,7 @@ addps %xmm9, %xmm0 ;\ movaps 32 * SIZE + 2 * (xx) * SIZE(BO), %xmm9 ;\ mulps %xmm8, %xmm11 ;\ - PREFETCH (PREFETCHSIZE + 8) * SIZE + 1 * (xx) * SIZE(AO) ;\ + PREFETCH_KERNEL5(xx) \ addps %xmm11, %xmm1 ;\ movaps 36 * SIZE + 2 * (xx) * SIZE(BO), %xmm11 ;\ mulps %xmm8, %xmm13 ;\ diff --git a/kernel/x86_64/ztrsm_kernel_RT_4x2_sse.S b/kernel/x86_64/ztrsm_kernel_RT_4x2_sse.S index 20b93e1983..518e1b4fef 100644 --- a/kernel/x86_64/ztrsm_kernel_RT_4x2_sse.S +++ b/kernel/x86_64/ztrsm_kernel_RT_4x2_sse.S @@ -109,12 +109,20 @@ #define PREFETCHSIZE (8 * 6 + 4) #endif +#ifdef PREFETCH +#define PREFETCH_KERNEL1(xx) PREFETCH (PREFETCHSIZE + 0) * SIZE + 1 * (xx) * SIZE(AO) ; +#define PREFETCH_KERNEL5(xx) PREFETCH (PREFETCHSIZE + 8) * SIZE + 1 * (xx) * SIZE(AO) ; +#else +#define PREFETCH_KERNEL1(xx) +#define PREFETCH_KERNEL5(xx) +#endif + #define KERNEL1(xx) \ mulps %xmm8, %xmm9 ;\ addps %xmm9, %xmm0 ;\ movaps 0 * SIZE + 2 * (xx) * SIZE(BO), %xmm9 ;\ mulps %xmm8, %xmm11 ;\ - PREFETCH (PREFETCHSIZE + 0) * SIZE + 1 * (xx) * SIZE(AO) ;\ + PREFETCH_KERNEL1(xx) \ addps %xmm11, %xmm1 ;\ movaps 4 * SIZE + 2 * (xx) * SIZE(BO), %xmm11 ;\ mulps %xmm8, %xmm13 ;\ @@ -171,7 +179,7 @@ addps %xmm9, %xmm0 ;\ movaps 32 * SIZE + 2 * (xx) * SIZE(BO), %xmm9 ;\ mulps %xmm8, %xmm11 ;\ - PREFETCH (PREFETCHSIZE + 8) * SIZE + 1 * (xx) * SIZE(AO) ;\ + PREFETCH_KERNEL5(xx) \ addps %xmm11, %xmm1 ;\ movaps 36 * SIZE + 2 * (xx) * SIZE(BO), %xmm11 ;\ mulps %xmm8, %xmm13 ;\ From 94feadf2424f83176a10f52f3714c7c2bbb4111f Mon Sep 17 00:00:00 2001 From: Matti Picus Date: Wed, 17 Apr 2024 09:24:51 +1000 Subject: [PATCH 044/267] do not add LIBNAMESUFFIX to dylib --- exports/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exports/Makefile b/exports/Makefile index 27a291f341..239171d448 100644 --- a/exports/Makefile +++ b/exports/Makefile @@ -137,7 +137,7 @@ libgoto_hpl.def : $(GENSYM) ifeq ($(OSNAME), Darwin) ifeq ($(FIXED_LIBNAME),1) -INTERNALNAME = $(LIBPREFIX)$(LIBNAMESUFFIX).dylib +INTERNALNAME = $(LIBPREFIX).dylib else INTERNALNAME = $(LIBPREFIX).$(MAJOR_VERSION).dylib endif From 61214fcef7ada202a583bc574b7ff55eb1a710e1 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Mon, 22 Apr 2024 14:24:33 +0200 Subject: [PATCH 045/267] Fix utest_ext build on AIX (#4657) * Add all-in-one version of utest_ext for AIX --- utest/Makefile | 1 + utest/test_extensions/utest_main2.c | 712 ++++++++++++++++++++++++++++ 2 files changed, 713 insertions(+) create mode 100644 utest/test_extensions/utest_main2.c diff --git a/utest/Makefile b/utest/Makefile index ce0f5c4307..d4154e0c2c 100644 --- a/utest/Makefile +++ b/utest/Makefile @@ -58,6 +58,7 @@ OBJS = utest_main2.o endif ifeq ($(OSNAME), AIX) OBJS = utest_main2.o +OBJS_EXT = $(DIR_EXT)/utest_main2.o endif all : run_test diff --git a/utest/test_extensions/utest_main2.c b/utest/test_extensions/utest_main2.c new file mode 100644 index 0000000000..1a761cb75d --- /dev/null +++ b/utest/test_extensions/utest_main2.c @@ -0,0 +1,712 @@ +/***************************************************************************** +Copyright (c) 2011-2016, The OpenBLAS Project +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. Neither the name of the OpenBLAS project nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +**********************************************************************************/ + +#include +#include + + +#define CTEST_MAIN +#define CTEST_SEGFAULT +#define CTEST_ADD_TESTS_MANUALLY + +#include "cblas.h" +#include "openblas_utest.h" +#if 1 +CTEST(amax, samax){ + blasint N=3, inc=1; + float te_max=0.0, tr_max=0.0; + float x[]={-1.1, 2.2, -3.3}; + fprintf(stderr,"testing samax\n"); + te_max=BLASFUNC(samax)(&N, x, &inc); + tr_max=3.3; + + ASSERT_DBL_NEAR_TOL((double)(tr_max), (double)(te_max), SINGLE_EPS); +} + +CTEST(amax, damax){ + blasint N=3, inc=1; + double te_max=0.0, tr_max=0.0; + double x[]={-1.1, 2.2, -3.3}; + + fprintf(stderr,"testing damax\n"); + te_max=BLASFUNC(damax)(&N, x, &inc); + tr_max=3.3; + + ASSERT_DBL_NEAR_TOL((double)(tr_max), (double)(te_max), DOUBLE_EPS); +} + +CTEST (drotmg,rotmg) +{ + double te_d1, tr_d1; + double te_d2, tr_d2; + double te_x1, tr_x1; + double te_y1, tr_y1; + double te_param[5]; + double tr_param[5]; + int i=0; + // original test case for libGoto bug fixed by feb2014 rewrite + te_d1= 0.21149573940783739; + te_d2= 0.046892057172954082; + te_x1= -0.42272687517106533; + te_y1= 0.42211309121921659; + + + for(i=0; i<5; i++){ + te_param[i]=tr_param[i]=0.0; + } + + //reference values as calculated by netlib blas + + tr_d1= 0.1732048; + tr_d2= 0.03840234; + tr_x1= -0.516180; + tr_y1= 0.422113; + tr_d1= 0.17320483687975; + tr_d2= 0.03840233915037; + tr_x1= -0.51618034832329; + tr_y1= 0.42211309121922; + + tr_param[0]= 0.0; + tr_param[1]= 0.0; + tr_param[2]= 0.99854803659786; + tr_param[3]= -0.22139439665872; + tr_param[4]= 0.0; + + BLASFUNC(drotmg)(&te_d1, &te_d2, &te_x1, &te_y1, te_param); + ASSERT_DBL_NEAR_TOL(tr_d1, te_d1, DOUBLE_EPS); + ASSERT_DBL_NEAR_TOL(tr_d2, te_d2, DOUBLE_EPS); + ASSERT_DBL_NEAR_TOL(tr_x1, te_x1, DOUBLE_EPS); + ASSERT_DBL_NEAR_TOL(tr_y1, te_y1, DOUBLE_EPS); + + for(i=0; i<5; i++){ + ASSERT_DBL_NEAR_TOL(tr_param[i], te_param[i], DOUBLE_EPS); + } +} + +CTEST (drotmg,rotmg_issue1452) +{ + double te_d1, tr_d1; + double te_d2, tr_d2; + double te_x1, tr_x1; + double te_y1, tr_y1; + double te_param[5]; + double tr_param[5]; + int i=0; + + // from issue #1452 + te_d1 = 5.9e-8; + te_d2 = 5.960464e-8; + te_x1 = 1.0; + te_y1 = 150.0; + + for(i=0; i<5; i++){ + te_param[i]=tr_param[i]=0.0; + } + te_param[3]=1./4096.; + //reference values as calculated by gonum blas with rotmg rewritten to Hopkins' algorithm + tr_d1= 0.99995592822897; + tr_d2= 0.98981219860583; + tr_x1= 0.03662270484346; + tr_y1= 150.000000000000; + + tr_param[0]= -1.0; + tr_param[1]= 0.00000161109346; + tr_param[2]= -0.00024414062500; + tr_param[3]= 0.00024414062500; + tr_param[4]= 0.00000162760417; + + //OpenBLAS + BLASFUNC(drotmg)(&te_d1, &te_d2, &te_x1, &te_y1, te_param); + + ASSERT_DBL_NEAR_TOL(tr_d1, te_d1, DOUBLE_EPS); + ASSERT_DBL_NEAR_TOL(tr_d2, te_d2, DOUBLE_EPS); + ASSERT_DBL_NEAR_TOL(tr_x1, te_x1, DOUBLE_EPS); + ASSERT_DBL_NEAR_TOL(tr_y1, te_y1, DOUBLE_EPS); + + for(i=0; i<5; i++){ + ASSERT_DBL_NEAR_TOL(tr_param[i], te_param[i], DOUBLE_EPS); + } + +} + +CTEST(drotmg, rotmg_D1eqD2_X1eqX2) +{ + double te_d1, tr_d1; + double te_d2, tr_d2; + double te_x1, tr_x1; + double te_y1, tr_y1; + double te_param[5]; + double tr_param[5]; + int i=0; + te_d1= tr_d1=2.; + te_d2= tr_d2=2.; + te_x1= tr_x1=8.; + te_y1= tr_y1=8.; + + for(i=0; i<5; i++){ + te_param[i]=tr_param[i]=0.0; + } + + //reference values as calculated by netlib blas + tr_d1= 1.0; + tr_d2= 1.0; + tr_x1= 16.0; + tr_y1= 8.0; + + tr_param[0]=1.0; + tr_param[1]=1.0; + tr_param[2]=0.0; + tr_param[3]=0.0; + tr_param[4]=1.0; + + //OpenBLAS + BLASFUNC(drotmg)(&te_d1, &te_d2, &te_x1, &te_y1, te_param); + + ASSERT_DBL_NEAR_TOL(tr_d1, te_d1, DOUBLE_EPS); + ASSERT_DBL_NEAR_TOL(tr_d2, te_d2, DOUBLE_EPS); + ASSERT_DBL_NEAR_TOL(tr_x1, te_x1, DOUBLE_EPS); + ASSERT_DBL_NEAR_TOL(tr_y1, te_y1, DOUBLE_EPS); + + for(i=0; i<5; i++){ + ASSERT_DBL_NEAR_TOL(tr_param[i], te_param[i], DOUBLE_EPS); + } +} + +CTEST(drotmg, drotmg_D1_big_D2_big_flag_zero) +{ + double te_d1, tr_d1; + double te_d2, tr_d2; + double te_x1, tr_x1; + double te_y1, tr_y1; + double te_param[5]={1.,4096.,-4096.,1.,4096.}; + double tr_param[5]={-1.,4096.,-3584.,1792.,4096.}; + int i=0; + te_d1= tr_d1=1600000000.; + te_d2= tr_d2=800000000.; + te_x1= tr_x1=8.; + te_y1= tr_y1=7.; + + + //reference values as calculated by gonum + tr_d1= 68.96627824858757; + tr_d2= 34.483139124293785; + tr_x1= 45312.; + tr_y1= 7.0; + + + //OpenBLAS + BLASFUNC(drotmg)(&te_d1, &te_d2, &te_x1, &te_y1, te_param); + + ASSERT_DBL_NEAR_TOL(tr_d1, te_d1, DOUBLE_EPS); + ASSERT_DBL_NEAR_TOL(tr_d2, te_d2, DOUBLE_EPS); + ASSERT_DBL_NEAR_TOL(tr_x1, te_x1, DOUBLE_EPS); + ASSERT_DBL_NEAR_TOL(tr_y1, te_y1, DOUBLE_EPS); + + for(i=0; i<5; i++){ + ASSERT_DBL_NEAR_TOL(tr_param[i], te_param[i], DOUBLE_EPS); + } +} + +CTEST(axpy,daxpy_inc_0) +{ + blasint i; + blasint N=8,incX=0,incY=0; + double a=0.25; + double x1[]={1.0,3.0,5.0,7.0,1.0,3.0,5.0,7.0}; + double y1[]={2.0,4.0,6.0,8.0,2.0,4.0,6.0,8.0}; + + double x2[]={1.0,3.0,5.0,7.0,1.0,3.0,5.0,7.0}; + double y2[]={4.0,4.0,6.0,8.0,2.0,4.0,6.0,8.0}; + + //OpenBLAS + BLASFUNC(daxpy)(&N,&a,x1,&incX,y1,&incY); + + for(i=0; i Date: Mon, 15 Apr 2024 10:31:33 +0800 Subject: [PATCH 046/267] loongarch64: Fixed clang compilation issues --- Makefile.system | 6 + c_check | 15 +- kernel/loongarch64/cgemv_n_4_lsx.S | 4 +- kernel/loongarch64/cgemv_n_8_lasx.S | 4 +- kernel/loongarch64/cgemv_t_4_lsx.S | 4 +- kernel/loongarch64/cgemv_t_8_lasx.S | 4 +- kernel/loongarch64/dgemm_ncopy_8_lsx.S | 4 +- kernel/loongarch64/dgemm_tcopy_4_lsx.S | 4 +- kernel/loongarch64/dgemm_tcopy_8_lsx.S | 4 +- kernel/loongarch64/dgemv_n_8_lasx.S | 4 +- kernel/loongarch64/dgemv_t_8_lasx.S | 4 +- .../loongarch64/dtrsm_kernel_LN_16x4_lasx.S | 4 +- .../loongarch64/dtrsm_kernel_LT_16x4_lasx.S | 157 +++--- .../loongarch64/dtrsm_kernel_RN_16x4_lasx.S | 207 ++++---- .../loongarch64/dtrsm_kernel_RT_16x4_lasx.S | 206 ++++---- kernel/loongarch64/loongarch64_asm.S | 470 +++++++++++------- kernel/loongarch64/sgemm_kernel_16x8_lasx.S | 4 +- kernel/loongarch64/sgemm_ncopy_16_lasx.S | 4 +- kernel/loongarch64/sgemm_ncopy_8_lasx.S | 4 +- kernel/loongarch64/sgemm_tcopy_16_lasx.S | 4 +- kernel/loongarch64/sgemm_tcopy_8_lasx.S | 4 +- kernel/loongarch64/sgemv_n_8_lasx.S | 4 +- kernel/loongarch64/sgemv_t_8_lasx.S | 4 +- kernel/loongarch64/zgemv_n_2_lsx.S | 4 +- kernel/loongarch64/zgemv_n_4_lasx.S | 4 +- kernel/loongarch64/zgemv_t_2_lsx.S | 4 +- kernel/loongarch64/zgemv_t_4_lasx.S | 4 +- 27 files changed, 638 insertions(+), 507 deletions(-) diff --git a/Makefile.system b/Makefile.system index 902b536052..eae1376e18 100644 --- a/Makefile.system +++ b/Makefile.system @@ -955,12 +955,18 @@ endif ifeq ($(ARCH), loongarch64) LA64_ABI=$(shell $(CC) -mabi=lp64d -c $(TOPDIR)/cpuid_loongarch64.c -o /dev/null > /dev/null 2> /dev/null && echo lp64d) +LA64_ARCH=$(shell $(CC) -march=loongarch64 -c $(TOPDIR)/cpuid_loongarch64.c -o /dev/null > /dev/null 2> /dev/null && echo loongarch64) ifneq ($(LA64_ABI), lp64d) LA64_ABI=lp64 endif +ifneq ($(LA64_ARCH), loongarch64) +CCOMMON_OPT += -mabi=$(LA64_ABI) +FCOMMON_OPT += -mabi=$(LA64_ABI) +else CCOMMON_OPT += -march=loongarch64 -mabi=$(LA64_ABI) FCOMMON_OPT += -march=loongarch64 -mabi=$(LA64_ABI) endif +endif endif diff --git a/c_check b/c_check index 59ab9bb13a..114eed60c2 100755 --- a/c_check +++ b/c_check @@ -197,10 +197,22 @@ fi no_lsx=0 no_lasx=0 if [ "$architecture" = "loongarch64" ]; then + lasx_flags='-march=loongarch64' + lsx_flags='-march=loongarch64' + tmpd="$(mktemp -d)" + tmparch="$tmpd/arch.c" + printf "void main(void){ }\n" >> "$tmparch" + args="-march=loongarch64 -o $tmparch.o $tmparch" + { + $compiler_name $flags $args >/dev/null 2>&1 + } || { + lasx_flags='' + lsx_flags='' + } + tmplsx="$tmpd/lsx.c" codelsx='"vadd.b $vr0, $vr0, $vr0"' - lsx_flags='-march=loongarch64' printf "void main(void){ __asm__ volatile(%s);}\n" "$codelsx" >> "$tmplsx" args="$lsx_flags -o $tmplsx.o $tmplsx" { @@ -211,7 +223,6 @@ if [ "$architecture" = "loongarch64" ]; then tmplasx="$tmpd/lasx.c" codelasx='"xvadd.b $xr0, $xr0, $xr0"' - lasx_flags='-march=loongarch64' printf "void main(void){ __asm__ volatile(%s);}\n" "$codelasx" >> "$tmplasx" args="$lasx_flags -o $tmplasx.o $tmplasx" { diff --git a/kernel/loongarch64/cgemv_n_4_lsx.S b/kernel/loongarch64/cgemv_n_4_lsx.S index cf82737976..a3626191b0 100644 --- a/kernel/loongarch64/cgemv_n_4_lsx.S +++ b/kernel/loongarch64/cgemv_n_4_lsx.S @@ -279,7 +279,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE PTR_LD INC_Y, $sp, 0 - push_if_used 17 + 7, 31 + push_if_used 7, 7 PTR_ADDI K, $r0, 0x01 PTR_SUB I, INC_X, K PTR_SUB J, INC_Y, K @@ -318,6 +318,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .L_GAP_1_1: /* if (inc_x != 1) && (incy != 1) */ CGEMV_N_LSX GAP_1_1, X_4_GAP, X_1, Y_4_GAP, Y_1 .L_END: - pop_if_used 17 + 7, 31 + pop_if_used 7, 7 jirl $r0, $r1, 0x0 EPILOGUE diff --git a/kernel/loongarch64/cgemv_n_8_lasx.S b/kernel/loongarch64/cgemv_n_8_lasx.S index ba38a9573a..44e59d0a70 100644 --- a/kernel/loongarch64/cgemv_n_8_lasx.S +++ b/kernel/loongarch64/cgemv_n_8_lasx.S @@ -336,7 +336,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE PTR_LD INC_Y, $sp, 0 - push_if_used 17 + 7, 31 + push_if_used 7, 7 PTR_ADDI K, $r0, 0x01 PTR_SUB I, INC_X, K PTR_SUB J, INC_Y, K @@ -378,6 +378,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .L_GAP_1_1: /* if (inc_x != 1) && (incy != 1) */ CGEMV_N_LASX GAP_1_1, X_8_GAP, X_1, Y_8_GAP, Y_1 .L_END: - pop_if_used 17 + 7, 31 + pop_if_used 7, 7 jirl $r0, $r1, 0x0 EPILOGUE diff --git a/kernel/loongarch64/cgemv_t_4_lsx.S b/kernel/loongarch64/cgemv_t_4_lsx.S index ada3493641..6acf8c63bc 100644 --- a/kernel/loongarch64/cgemv_t_4_lsx.S +++ b/kernel/loongarch64/cgemv_t_4_lsx.S @@ -255,7 +255,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE PTR_LD INC_Y, $sp, 0 - push_if_used 17 + 8, 30 + push_if_used 8, 6 PTR_ADDI K, $r0, 0x01 PTR_SUB I, INC_X, K maskeqz I, K, I /* if(inc_x == 1) I = 0; else I = 1; */ @@ -285,6 +285,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .L_GAP_1: /* if (incx != 1) */ CGEMV_T_LSX GAP_1, X4_GAP .L_END: - pop_if_used 17 + 8, 30 + pop_if_used 8, 6 jirl $r0, $r1, 0x0 EPILOGUE diff --git a/kernel/loongarch64/cgemv_t_8_lasx.S b/kernel/loongarch64/cgemv_t_8_lasx.S index 94e4bd2ebf..f8a0ad124b 100644 --- a/kernel/loongarch64/cgemv_t_8_lasx.S +++ b/kernel/loongarch64/cgemv_t_8_lasx.S @@ -304,7 +304,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE PTR_LD INC_Y, $sp, 0 - push_if_used 17 + 8, 30 + push_if_used 8, 6 PTR_ADDI K, $r0, 0x01 PTR_SUB I, INC_X, K maskeqz I, K, I /* if(inc_x == 1) I = 0; else I = 1; */ @@ -337,6 +337,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .L_GAP_1: /* if (incx != 1) */ CGEMV_T_LASX GAP_1, X8_GAP .L_END: - pop_if_used 17 + 8, 30 + pop_if_used 8, 6 jirl $r0, $r1, 0x0 EPILOGUE diff --git a/kernel/loongarch64/dgemm_ncopy_8_lsx.S b/kernel/loongarch64/dgemm_ncopy_8_lsx.S index 30bebe8df5..203c3eb27d 100644 --- a/kernel/loongarch64/dgemm_ncopy_8_lsx.S +++ b/kernel/loongarch64/dgemm_ncopy_8_lsx.S @@ -79,7 +79,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define D7 $vr15 PROLOGUE - push_if_used 26, 32 + push_if_used 0, 0 move TD, DST move TS, SRC slli.d TL, LDA, 0x03 @@ -278,6 +278,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addi.d M, M, -1 blt ZERO, M, .L_M1 .L_N0: - pop_if_used 26, 32 + pop_if_used 0, 0 jirl $r0, $r1, 0x00 EPILOGUE diff --git a/kernel/loongarch64/dgemm_tcopy_4_lsx.S b/kernel/loongarch64/dgemm_tcopy_4_lsx.S index 1340664717..d9a442e576 100644 --- a/kernel/loongarch64/dgemm_tcopy_4_lsx.S +++ b/kernel/loongarch64/dgemm_tcopy_4_lsx.S @@ -66,7 +66,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define U7 $vr7 PROLOGUE - push_if_used 18, 8 + push_if_used 1, 0 move S0, SRC move P0, DST @@ -274,7 +274,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. fst.d F0, P3, 0x00 .L_M0: - pop_if_used 18, 8 + pop_if_used 1, 0 jirl $r0, $r1, 0x00 EPILOGUE diff --git a/kernel/loongarch64/dgemm_tcopy_8_lsx.S b/kernel/loongarch64/dgemm_tcopy_8_lsx.S index a7e3ef69cb..b4106e6a96 100644 --- a/kernel/loongarch64/dgemm_tcopy_8_lsx.S +++ b/kernel/loongarch64/dgemm_tcopy_8_lsx.S @@ -76,7 +76,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define U7 $vr7 PROLOGUE - push_if_used 24, 8 + push_if_used 7, 0 move S0, SRC move P0, DST @@ -592,6 +592,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addi.d S1, S1, 0x08 addi.d P4, P4, 0x08 .L_M0: - pop_if_used 24, 8 + pop_if_used 7, 0 jirl $r0, $r1, 0x00 EPILOGUE diff --git a/kernel/loongarch64/dgemv_n_8_lasx.S b/kernel/loongarch64/dgemv_n_8_lasx.S index a49bf9bb14..9fe4bfddd5 100644 --- a/kernel/loongarch64/dgemv_n_8_lasx.S +++ b/kernel/loongarch64/dgemv_n_8_lasx.S @@ -509,7 +509,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE PTR_LD INC_Y, $sp, 0 - push_if_used 17 + 7, 24 + 4 + push_if_used 7, 4 PTR_ADDI K, $r0, 0x01 PTR_SUB I, INC_X, K PTR_SUB J, INC_Y, K @@ -549,6 +549,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .L_GAP_1_1: /* if (inc_x != 1) && (incy != 1) */ DGEMV_N_LASX GAP_1_1, X_8_GAP, X_4_GAP, X_2_GAP, X_1, Y_8_GAP, Y_4_GAP, Y_1 .L_END: - pop_if_used 17 + 7, 24 + 4 + pop_if_used 7, 4 jirl $r0, $r1, 0x0 EPILOGUE diff --git a/kernel/loongarch64/dgemv_t_8_lasx.S b/kernel/loongarch64/dgemv_t_8_lasx.S index 71f942b0f0..2c29bebece 100644 --- a/kernel/loongarch64/dgemv_t_8_lasx.S +++ b/kernel/loongarch64/dgemv_t_8_lasx.S @@ -445,7 +445,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE PTR_LD INC_Y, $sp, 0 - push_if_used 17 + 8, 24 + 3 + push_if_used 8, 3 PTR_ADDI K, $r0, 0x01 PTR_SUB I, INC_X, K maskeqz I, K, I /* if(inc_x == 1) I = 0; else I = 1; */ @@ -476,6 +476,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .L_GAP_1: /* if (incx != 1) */ DGEMV_T_LASX GAP_1, X8_GAP, X4_GAP .L_END: - pop_if_used 17 + 8, 24 + 3 + pop_if_used 8, 3 jirl $r0, $r1, 0x0 EPILOGUE diff --git a/kernel/loongarch64/dtrsm_kernel_LN_16x4_lasx.S b/kernel/loongarch64/dtrsm_kernel_LN_16x4_lasx.S index 3315daccbc..e71fa7d30f 100644 --- a/kernel/loongarch64/dtrsm_kernel_LN_16x4_lasx.S +++ b/kernel/loongarch64/dtrsm_kernel_LN_16x4_lasx.S @@ -1029,7 +1029,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .endm PROLOGUE - push_if_used 26, 32 + push_if_used 9, 8 PTR_SLLI LDC, LDC, 3 /* if (!(N >> 2)) goto L_N3 */ PTR_SRAI J, N, 2 /* J = bn >> 2 */ @@ -1361,6 +1361,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. blt ZERO, I, .L_N1_I1 .L_N1_M0: .L_N0: - pop_if_used 26, 32 + pop_if_used 9, 8 jirl $r0, $r1, 0x0 EPILOGUE diff --git a/kernel/loongarch64/dtrsm_kernel_LT_16x4_lasx.S b/kernel/loongarch64/dtrsm_kernel_LT_16x4_lasx.S index 0e2cacccff..7fc62857a9 100644 --- a/kernel/loongarch64/dtrsm_kernel_LT_16x4_lasx.S +++ b/kernel/loongarch64/dtrsm_kernel_LT_16x4_lasx.S @@ -128,31 +128,31 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "dtrsm_kernel_macro.S" -.macro ldrepl_macro start, end, stride +.macro ldrepl_macro stride:req, index:req, more:vararg // Load Ux (x = 0...15) -.if \start <= \end - GLDREPL xv, d, $xr\start, A0, \stride * 8 - ldrepl_macro %start + 1, \end, %stride + 1 + GLDREPL xv, d, $xr\index, A0, \index * 8 - \stride * 8 +.ifnb \more + ldrepl_macro \stride, \more .endif .endm -.macro nmsub_macro start0, end0, start1, reg +.macro nmsub_macro reg:req, start0:req, start1:req, more:vararg // Gx -= reg * Ux -.if \start0 <= \end0 xvfnmsub.d $xr\start0, \reg, $xr\start1, $xr\start0 - nmsub_macro %start0 + 1, \end0, %start1 + 1, \reg +.ifnb \more + nmsub_macro \reg, \more .endif .endm -.macro B_st_macro start, end, stride, N +.macro B_st_macro N:req, stride:req, start:req, more:vararg // Store Gx(x = 16...31) -.if \start <= \end .if \N == 4 - xvst $xr\start, B0, \stride * 0x20 + xvst $xr\start, B0, \start * 0x20 - \stride * 0x20 .elseif \N == 2 - vst $vr\start, B0, \stride * 0x10 + vst $vr\start, B0, \start * 0x10 - \stride * 0x10 .elseif \N == 1 - fst.d $f\start, B0, \stride * 0x08 + fst.d $f\start, B0, \start * 0x08 - \stride * 0x08 .endif - B_st_macro %start + 1, \end, %stride + 1, \N +.ifnb \more + B_st_macro \N, \stride, \more .endif .endm @@ -194,86 +194,93 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // 255 // Sequentially extract data from A in row order // Load 0 - ldrepl_macro 0, 15, 0 + ldrepl_macro 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 GMUL xvf, d, G0, G0, U0 - nmsub_macro 17, 31, 1, G0 + nmsub_macro G0, 17, 1, 18, 2, 19, 3, 20, 4, 21, 5, 22, 6, 23, 7, 24, 8, \ + 25, 9, 26, 10, 27, 11, 28, 12, 29, 13, 30, 14, 31, 15 PTR_ADDI A0, A0, 17 * 8 // Load 1 - ldrepl_macro 1, 15, 0 + ldrepl_macro 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 GMUL xvf, d, G1, G1, U1 - nmsub_macro 18, 31, 2, G1 + nmsub_macro G1, 18, 2, 19, 3, 20, 4, 21, 5, 22, 6, 23, 7, 24, 8, \ + 25, 9, 26, 10, 27, 11, 28, 12, 29, 13, 30, 14, 31, 15 PTR_ADDI A0, A0, 17 * 8 // Load 2 - ldrepl_macro 2, 15, 0 + ldrepl_macro 2, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 GMUL xvf, d, G2, G2, U2 - nmsub_macro 19, 31, 3, G2 + nmsub_macro G2, 19, 3, 20, 4, 21, 5, 22, 6, 23, 7, 24, 8, 25, 9, 26, \ + 10, 27, 11, 28, 12, 29, 13, 30, 14, 31, 15 PTR_ADDI A0, A0, 17 * 8 // Load 3 - ldrepl_macro 3, 15, 0 + ldrepl_macro 3, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 GMUL xvf, d, G3, G3, U3 - nmsub_macro 20, 31, 4, G3 + nmsub_macro G3, 20, 4, 21, 5, 22, 6, 23, 7, 24, 8, 25, 9, 26, 10, \ + 27, 11, 28, 12, 29, 13, 30, 14, 31, 15 PTR_ADDI A0, A0, 17 * 8 // Load 4 - ldrepl_macro 4, 15, 0 + ldrepl_macro 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 GMUL xvf, d, G4, G4, U4 - nmsub_macro 21, 31, 5, G4 + nmsub_macro G4, 21, 5, 22, 6, 23, 7, 24, 8, 25, 9, 26, 10, 27, 11, \ + 28, 12, 29, 13, 30, 14, 31, 15 PTR_ADDI A0, A0, 17 * 8 // Load 5 - ldrepl_macro 5, 15, 0 + ldrepl_macro 5, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 GMUL xvf, d, G5, G5, U5 - nmsub_macro 22, 31, 6, G5 + nmsub_macro G5, 22, 6, 23, 7, 24, 8, 25, 9, 26, 10, 27, 11, 28, 12, \ + 29, 13, 30, 14, 31, 15 PTR_ADDI A0, A0, 17 * 8 // Load 6 - ldrepl_macro 6, 15, 0 + ldrepl_macro 6, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 GMUL xvf, d, G6, G6, U6 - nmsub_macro 23, 31, 7, G6 + nmsub_macro G6, 23, 7, 24, 8, 25, 9, 26, 10, 27, 11, 28, 12, 29, 13, \ + 30, 14, 31, 15 PTR_ADDI A0, A0, 17 * 8 // Load 7 - ldrepl_macro 7, 15, 0 + ldrepl_macro 7, 7, 8, 9, 10, 11, 12, 13, 14, 15 GMUL xvf, d, G7, G7, U7 - nmsub_macro 24, 31, 8, G7 + nmsub_macro G7, 24, 8, 25, 9, 26, 10, 27, 11, 28, 12, 29, 13, 30, 14, 31, 15 PTR_ADDI A0, A0, 17 * 8 // Load 8 - ldrepl_macro 8, 15, 0 + ldrepl_macro 8, 8, 9, 10, 11, 12, 13, 14, 15 GMUL xvf, d, G8, G8, U8 - nmsub_macro 25, 31, 9, G8 + nmsub_macro G8, 25, 9, 26, 10, 27, 11, 28, 12, 29, 13, 30, 14, 31, 15 PTR_ADDI A0, A0, 17 * 8 // Load 9 - ldrepl_macro 9, 15, 0 + ldrepl_macro 9, 9, 10, 11, 12, 13, 14, 15 GMUL xvf, d, G9, G9, U9 - nmsub_macro 26, 31, 10, G9 + nmsub_macro G9, 26, 10, 27, 11, 28, 12, 29, 13, 30, 14, 31, 15 PTR_ADDI A0, A0, 17 * 8 // Load 10 - ldrepl_macro 10, 15, 0 + ldrepl_macro 10, 10, 11, 12, 13, 14, 15 GMUL xvf, d, G10, G10, U10 - nmsub_macro 27, 31, 11, G10 + nmsub_macro G10, 27, 11, 28, 12, 29, 13, 30, 14, 31, 15 PTR_ADDI A0, A0, 17 * 8 // Load 11 - ldrepl_macro 11, 15, 0 + ldrepl_macro 11, 11, 12, 13, 14, 15 GMUL xvf, d, G11, G11, U11 - nmsub_macro 28, 31, 12, G11 + nmsub_macro G11, 28, 12, 29, 13, 30, 14, 31, 15 PTR_ADDI A0, A0, 17 * 8 // Load 12 - ldrepl_macro 12, 15, 0 + ldrepl_macro 12, 12, 13, 14, 15 GMUL xvf, d, G12, G12, U12 - nmsub_macro 29, 31, 13, G12 + nmsub_macro G12, 29, 13, 30, 14, 31, 15 PTR_ADDI A0, A0, 17 * 8 // Load 13 - ldrepl_macro 13, 15, 0 + ldrepl_macro 13, 13, 14, 15 GMUL xvf, d, G13, G13, U13 - nmsub_macro 30, 31, 14, G13 + nmsub_macro G13, 30, 14, 31, 15 PTR_ADDI A0, A0, 17 * 8 // Load 14 - ldrepl_macro 14, 15, 0 + ldrepl_macro 14, 14, 15 GMUL xvf, d, G14, G14, U14 - nmsub_macro 31, 31, 15, G14 + nmsub_macro G14, 31, 15 PTR_ADDI A0, A0, 17 * 8 // Load 15 - ldrepl_macro 15, 15, 0 + ldrepl_macro 15, 15 GMUL xvf, d, G15, G15, U15 // Finally, We can store the result. // For B, stored sequentially, and C, first transpose and then store - B_st_macro 16, 31, 0, \N + B_st_macro \N, 16, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 GTRANSPOSE4x4_D G0, G1, G2, G3, G0, G1, G2, G3, U0, U1 GTRANSPOSE4x4_D G4, G5, G6, G7, G4, G5, G6, G7, U0, U1 GTRANSPOSE4x4_D G8, G9, G10, G11, G8, G9, G10, G11, U0, U1 @@ -334,46 +341,46 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // 63 // Sequentially extract data from A in row order // Load 0 - ldrepl_macro 0, 7, 0 + ldrepl_macro 0, 0, 1, 2, 3, 4, 5, 6, 7 GMUL xvf, d, G0, G0, U0 - nmsub_macro 17, 23, 1, G0 + nmsub_macro G0, 17, 1, 18, 2, 19, 3, 20, 4, 21, 5, 22, 6, 23, 7 PTR_ADDI A0, A0, 9 * 8 // Load 1 - ldrepl_macro 1, 7, 0 + ldrepl_macro 1, 1, 2, 3, 4, 5, 6, 7 GMUL xvf, d, G1, G1, U1 - nmsub_macro 18, 23, 2, G1 + nmsub_macro G1, 18, 2, 19, 3, 20, 4, 21, 5, 22, 6, 23, 7 PTR_ADDI A0, A0, 9 * 8 // Load 2 - ldrepl_macro 2, 7, 0 + ldrepl_macro 2, 2, 3, 4, 5, 6, 7 GMUL xvf, d, G2, G2, U2 - nmsub_macro 19, 23, 3, G2 + nmsub_macro G2, 19, 3, 20, 4, 21, 5, 22, 6, 23, 7 PTR_ADDI A0, A0, 9 * 8 // Load 3 - ldrepl_macro 3, 7, 0 + ldrepl_macro 3, 3, 4, 5, 6, 7 GMUL xvf, d, G3, G3, U3 - nmsub_macro 20, 23, 4, G3 + nmsub_macro G3, 20, 4, 21, 5, 22, 6, 23, 7 PTR_ADDI A0, A0, 9 * 8 // Load 4 - ldrepl_macro 4, 7, 0 + ldrepl_macro 4, 4, 5, 6, 7 GMUL xvf, d, G4, G4, U4 - nmsub_macro 21, 23, 5, G4 + nmsub_macro G4, 21, 5, 22, 6, 23, 7 PTR_ADDI A0, A0, 9 * 8 // Load 5 - ldrepl_macro 5, 7, 0 + ldrepl_macro 5, 5, 6, 7 GMUL xvf, d, G5, G5, U5 - nmsub_macro 22, 23, 6, G5 + nmsub_macro G5, 22, 6, 23, 7 PTR_ADDI A0, A0, 9 * 8 // Load 6 - ldrepl_macro 6, 7, 0 + ldrepl_macro 6, 6, 7 GMUL xvf, d, G6, G6, U6 - nmsub_macro 23, 23, 7, G6 + nmsub_macro G6, 23, 7 PTR_ADDI A0, A0, 9 * 8 // Load 7 - ldrepl_macro 7, 7, 0 + ldrepl_macro 7, 7 GMUL xvf, d, G7, G7, U7 // Finally, We can store the result. // For B, stored sequentially, and C, first transpose and then store - B_st_macro 16, 23, 0, \N + B_st_macro \N, 16, 16, 17, 18, 19, 20, 21, 22, 23 GTRANSPOSE4x4_D G0, G1, G2, G3, G0, G1, G2, G3, U0, U1 GTRANSPOSE4x4_D G4, G5, G6, G7, G4, G5, G6, G7, U0, U1 .if \N == 4 @@ -437,26 +444,26 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // 15 // Sequentially extract data from A in row order // Load 0 - ldrepl_macro 0, 3, 0 + ldrepl_macro 0, 0, 1, 2, 3 GMUL xvf, d, G0, G0, U0 - nmsub_macro 17, 19, 1, G0 + nmsub_macro G0, 17, 1, 18, 2, 19, 3 PTR_ADDI A0, A0, 5 * 8 // Load 1 - ldrepl_macro 1, 3, 0 + ldrepl_macro 1, 1, 2, 3 GMUL xvf, d, G1, G1, U1 - nmsub_macro 18, 19, 2, G1 + nmsub_macro G1, 18, 2, 19, 3 PTR_ADDI A0, A0, 5 * 8 // Load 2 - ldrepl_macro 2, 3, 0 + ldrepl_macro 2, 2, 3 GMUL xvf, d, G2, G2, U2 - nmsub_macro 19, 19, 3, G2 + nmsub_macro G2, 19, 3 PTR_ADDI A0, A0, 5 * 8 // Load 3 - ldrepl_macro 3, 3, 0 + ldrepl_macro 3, 3 GMUL xvf, d, G3, G3, U3 // Finally, We can store the result. // For B, stored sequentially, and C, first transpose and then store - B_st_macro 16, 19, 0, \N + B_st_macro \N, 16, 16, 17, 18, 19 GTRANSPOSE4x4_D G0, G1, G2, G3, G0, G1, G2, G3, U0, U1 .if \N == 4 GST xv, , G0, C0, 0x00, G1, C1, 0x00, G2, C2, 0x00, G3, C3, 0x00 @@ -501,16 +508,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // 3 // Sequentially extract data from A in row order // Load 0 - ldrepl_macro 0, 1, 0 + ldrepl_macro 0, 0, 1 GMUL xvf, d, G0, G0, U0 - nmsub_macro 17, 17, 1, G0 + nmsub_macro G0, 17, 1 PTR_ADDI A0, A0, 3 * 8 // Load 1 - ldrepl_macro 1, 1, 0 + ldrepl_macro 1, 1 GMUL xvf, d, G1, G1, U1 // Finally, We can store the result. // For B, stored sequentially, and C, first transpose and then store - B_st_macro 16, 17, 0, \N + B_st_macro \N, 16, 16, 17 GSBUTTERFLY xv, d, U0, U1, G1, G0 .if \N == 4 vst $vr0, C0, 0x00 @@ -717,7 +724,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .endm PROLOGUE - push_if_used 26, 32 + push_if_used 9, 8 PTR_SLLI LDC, LDC, 3 /* if (!(N >> 2)) goto L_N3 */ PTR_SRAI J, N, 2 /* J = bn >> 2 */ @@ -954,6 +961,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PTR_ADD AA, AA, T0 // aa += 1 * k .L_N1_M0: .L_N0: - pop_if_used 26, 32 + pop_if_used 9, 8 jirl $r0, $r1, 0x0 EPILOGUE diff --git a/kernel/loongarch64/dtrsm_kernel_RN_16x4_lasx.S b/kernel/loongarch64/dtrsm_kernel_RN_16x4_lasx.S index 4213397360..be378631b5 100644 --- a/kernel/loongarch64/dtrsm_kernel_RN_16x4_lasx.S +++ b/kernel/loongarch64/dtrsm_kernel_RN_16x4_lasx.S @@ -128,33 +128,31 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "dtrsm_kernel_macro.S" -.macro ldrepl_macro start, end, stride +.macro ldrepl_macro stride:req, index:req, more:vararg // Load Ux (x = 0...15) -.if \start <= \end - GLDREPL xv, d, $xr\start, B0, \stride * 8 - ldrepl_macro %start + 1, \end, %stride + 1 + GLDREPL xv, d, $xr\index, B0, \index * 8 - \stride * 8 +.ifnb \more + ldrepl_macro \stride, \more .endif .endm - -.macro nmsub_macro start0, end0, start1, reg -// Ux -= reg * Dx -.if \start0 <= \end0 +.macro nmsub_macro reg:req, start0:req, start1:req, more:vararg +// Gx -= reg * Ux xvfnmsub.d $xr\start0, \reg, $xr\start1, $xr\start0 - nmsub_macro %start0 + 1, \end0, %start1 + 1, \reg +.ifnb \more + nmsub_macro \reg, \more .endif .endm - -.macro A_st_macro start, end, stride, N -// Store Ux(x = 0...15) -.if \start <= \end +.macro A_st_macro N:req, stride:req, start:req, more:vararg +// Store Gx(x = 16...31) .if \N == 4 - xvst $xr\start, A0, \stride * 0x20 + xvst $xr\start, A0, \start * 0x20 - \stride * 0x20 .elseif \N == 2 - vst $vr\start, A0, \stride * 0x10 + vst $vr\start, A0, \start * 0x10 - \stride * 0x10 .elseif \N == 1 - fst.d $f\start, A0, \stride * 0x08 + fst.d $f\start, A0, \start * 0x08 - \stride * 0x08 .endif - A_st_macro %start + 1, \end, %stride + 1, \N +.ifnb \more + A_st_macro \N, \stride, \more .endif .endm @@ -167,22 +165,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // 10 11 // 15 // Sequentially extract data from B in row order - ldrepl_macro 16, 19, 0 + ldrepl_macro 16, 16, 17, 18, 19 GMUL xvf, d, U0, D0, U0, U1, D0, U1, U2, D0, U2, U3, D0, U3 - ldrepl_macro 20, 22, 5 - nmsub_macro 4, 7, 0, D1 - ldrepl_macro 23, 24, 10 + ldrepl_macro 15, 20, 21, 22 + + nmsub_macro D1, 4, 0, 5, 1, 6, 2, 7, 3 + ldrepl_macro 13, 23, 24 GMUL xvf, d, U4, D4, U4, U5, D4, U5, U6, D4, U6, U7, D4, U7 - ldrepl_macro 25, 25, 15 - nmsub_macro 8, 11, 0, D2 - nmsub_macro 8, 11, 4, D5 + ldrepl_macro 10, 25 + nmsub_macro D2, 8, 0, 9, 1, 10, 2, 11, 3 + nmsub_macro D5, 8, 4, 9, 5, 10, 6, 11, 7 GMUL xvf, d, U8, D7, U8, U9, D7, U9, U10, D7, U10, U11, D7, U11 - nmsub_macro 12, 15, 0, D3 - nmsub_macro 12, 15, 4, D6 - nmsub_macro 12, 15, 8, D8 + nmsub_macro D3, 12, 0, 13, 1, 14, 2, 15, 3 + nmsub_macro D6, 12, 4, 13, 5, 14, 6, 15, 7 + nmsub_macro D8, 12, 8, 13, 9, 14, 10, 15, 11 GMUL xvf, d, U12, D9, U12, U13, D9, U13, U14, D9, U14, U15, D9, U15 // Store A - A_st_macro 0, 15, 0, 4 + A_st_macro 4, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 // Store C GST xv, , U0, C0, 0x00, U1, C0, 0x20, U2, C0, 0x40, U3, C0, 0x60, \ U4, C1, 0x00, U5, C1, 0x20, U6, C1, 0x40, U7, C1, 0x60, \ @@ -197,13 +196,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //0 1 // 3 // Sequentially extract data from B in row order - ldrepl_macro 16, 17, 0 + ldrepl_macro 16, 16, 17 GMUL xvf, d, U0, D0, U0, U1, D0, U1, U2, D0, U2, U3, D0, U3 - ldrepl_macro 18, 18, 3 - nmsub_macro 4, 7, 0, D1 + ldrepl_macro 15, 18 + nmsub_macro D1, 4, 0, 5, 1, 6, 2, 7, 3 GMUL xvf, d, U4, D2, U4, U5, D2, U5, U6, D2, U6, U7, D2, U7 // Store A - A_st_macro 0, 7, 0, 4 + A_st_macro 4, 0, 0, 1, 2, 3, 4, 5, 6, 7 // Store C GST xv, , U0, C0, 0x00, U1, C0, 0x20, U2, C0, 0x40, U3, C0, 0x60, \ U4, C1, 0x00, U5, C1, 0x20, U6, C1, 0x40, U7, C1, 0x60 @@ -218,22 +217,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // 10 11 // 15 // Sequentially extract data from B in row order - ldrepl_macro 16, 19, 0 + ldrepl_macro 16, 16, 17, 18, 19 GMUL xvf, d, U0, D0, U0, U1, D0, U1 - ldrepl_macro 20, 22, 5 - nmsub_macro 2, 3, 0, D1 - ldrepl_macro 23, 24, 10 + ldrepl_macro 15, 20, 21, 22 + nmsub_macro D1, 2, 0, 3, 1 + ldrepl_macro 13, 23, 24 GMUL xvf, d, U2, D4, U2, U3, D4, U3 - ldrepl_macro 25, 25, 15 - nmsub_macro 4, 5, 0, D2 - nmsub_macro 4, 5, 2, D5 + ldrepl_macro 10, 25 + nmsub_macro D2, 4, 0, 5, 1 + nmsub_macro D5, 4, 2, 5, 3 GMUL xvf, d, U4, D7, U4, U5, D7, U5 - nmsub_macro 6, 7, 0, D3 - nmsub_macro 6, 7, 2, D6 - nmsub_macro 6, 7, 4, D8 + nmsub_macro D3, 6, 0, 7, 1 + nmsub_macro D6, 6, 2, 7, 3 + nmsub_macro D8, 6, 4, 7, 5 GMUL xvf, d, U6, D9, U6, U7, D9, U7 // Store A - A_st_macro 0, 7, 0, 4 + A_st_macro 4, 0, 0, 1, 2, 3, 4, 5, 6, 7 // Store C GST xv, , U0, C0, 0x00, U1, C0, 0x20, \ U2, C1, 0x00, U3, C1, 0x20, \ @@ -248,13 +247,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //0 1 // 3 // Sequentially extract data from B in row order - ldrepl_macro 16, 17, 0 + ldrepl_macro 16, 16, 17 GMUL xvf, d, U0, D0, U0, U1, D0, U1 - ldrepl_macro 18, 18, 3 - nmsub_macro 2, 3, 0, D1 + ldrepl_macro 15, 18 + nmsub_macro D1, 2, 0, 3, 1 GMUL xvf, d, U2, D2, U2, U3, D2, U3 // Store A - A_st_macro 0, 3, 0, 4 + A_st_macro 4, 0, 0, 1, 2, 3 // Store C GST xv, , U0, C0, 0x00, U1, C0, 0x20, \ U2, C1, 0x00, U3, C1, 0x20 @@ -269,22 +268,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // 10 11 // 15 // Sequentially extract data from B in row order - ldrepl_macro 16, 19, 0 + ldrepl_macro 16, 16, 17, 18, 19 GMUL xvf, d, U0, D0, U0 - ldrepl_macro 20, 22, 5 - nmsub_macro 1, 1, 0, D1 - ldrepl_macro 23, 24, 10 + ldrepl_macro 15, 20, 21, 22 + nmsub_macro D1, 1, 0 + ldrepl_macro 13, 23, 24 GMUL xvf, d, U1, D4, U1 - ldrepl_macro 25, 25, 15 - nmsub_macro 2, 2, 0, D2 - nmsub_macro 2, 2, 1, D5 + ldrepl_macro 10, 25 + nmsub_macro D2, 2, 0 + nmsub_macro D5, 2, 1 GMUL xvf, d, U2, D7, U2 - nmsub_macro 3, 3, 0, D3 - nmsub_macro 3, 3, 1, D6 - nmsub_macro 3, 3, 2, D8 + nmsub_macro D3, 3, 0 + nmsub_macro D6, 3, 1 + nmsub_macro D8, 3, 2 GMUL xvf, d, U3, D9, U3 // Store A - A_st_macro 0, 3, 0, 4 + A_st_macro 4, 0, 0, 1, 2, 3 // Store C GST xv, , U0, C0, 0x00, U1, C1, 0x00, U2, C2, 0x00, U3, C3, 0x00 .endm @@ -296,13 +295,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //0 1 // 3 // Sequentially extract data from B in row order - ldrepl_macro 16, 17, 0 + ldrepl_macro 16, 16, 17 GMUL xvf, d, U0, D0, U0 - ldrepl_macro 18, 18, 3 - nmsub_macro 1, 1, 0, D1 + ldrepl_macro 15, 18 + nmsub_macro D1, 1, 0 GMUL xvf, d, U1, D2, U1 // Store A - A_st_macro 0, 1, 0, 4 + A_st_macro 4, 0, 0, 1 // Store C GST xv, , U0, C0, 0x00, U1, C1, 0x00 .endm @@ -316,23 +315,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // 10 11 // 15 // Sequentially extract data from B in row order - ldrepl_macro 16, 19, 0 + ldrepl_macro 16, 16, 17, 18, 19 GMUL xvf, d, U0, D0, U0 - ldrepl_macro 20, 22, 5 - nmsub_macro 1, 1, 0, D1 - ldrepl_macro 23, 24, 10 + ldrepl_macro 15, 20, 21, 22 + nmsub_macro D1, 1, 0 + ldrepl_macro 13, 23, 24 GMUL xvf, d, U1, D4, U1 - ldrepl_macro 25, 25, 15 - nmsub_macro 2, 2, 0, D2 - nmsub_macro 2, 2, 1, D5 + ldrepl_macro 10, 25 + nmsub_macro D2, 2, 0 + nmsub_macro D5, 2, 1 GMUL xvf, d, U2, D7, U2 - nmsub_macro 3, 3, 0, D3 - nmsub_macro 3, 3, 1, D6 - nmsub_macro 3, 3, 2, D8 + nmsub_macro D3, 3, 0 + nmsub_macro D6, 3, 1 + nmsub_macro D8, 3, 2 GMUL xvf, d, U3, D9, U3 // Store A - A_st_macro 0, 3, 0, 2 + A_st_macro 2, 0, 0, 1, 2, 3 // Store C GST v, , $vr0, C0, 0x00, $vr1, C1, 0x00, $vr2, C2, 0x00, $vr3, C3, 0x00, .endm @@ -344,13 +343,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //0 1 // 3 // Sequentially extract data from B in row order - ldrepl_macro 16, 17, 0 + ldrepl_macro 16, 16, 17 GMUL xvf, d, U0, D0, U0 - ldrepl_macro 18, 18, 3 - nmsub_macro 1, 1, 0, D1 + ldrepl_macro 15, 18 + nmsub_macro D1, 1, 0 GMUL xvf, d, U1, D2, U1 // Store A - A_st_macro 0, 1, 0, 2 + A_st_macro 2, 0, 0, 1 // Store C GST v, , $vr0, C0, 0x00, $vr1, C1, 0x00 .endm @@ -364,23 +363,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // 10 11 // 15 // Sequentially extract data from B in row order - ldrepl_macro 16, 19, 0 + ldrepl_macro 16, 16, 17, 18, 19 GMUL xvf, d, U0, D0, U0 - ldrepl_macro 20, 22, 5 - nmsub_macro 1, 1, 0, D1 - ldrepl_macro 23, 24, 10 + ldrepl_macro 15, 20, 21, 22 + nmsub_macro D1, 1, 0 + ldrepl_macro 13, 23, 24 GMUL xvf, d, U1, D4, U1 - ldrepl_macro 25, 25, 15 - nmsub_macro 2, 2, 0, D2 - nmsub_macro 2, 2, 1, D5 + ldrepl_macro 10, 25 + nmsub_macro D2, 2, 0 + nmsub_macro D5, 2, 1 GMUL xvf, d, U2, D7, U2 - nmsub_macro 3, 3, 0, D3 - nmsub_macro 3, 3, 1, D6 - nmsub_macro 3, 3, 2, D8 + nmsub_macro D3, 3, 0 + nmsub_macro D6, 3, 1 + nmsub_macro D8, 3, 2 GMUL xvf, d, U3, D9, U3 // Store A - A_st_macro 0, 3, 0, 1 + A_st_macro 1, 0, 0, 1, 2, 3 // Store C GST f, d, $f0, C0, 0x00, $f1, C1, 0x00, $f2, C2, 0x00, $f3, C3, 0x00, .endm @@ -392,13 +391,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //0 1 // 3 // Sequentially extract data from B in row order - ldrepl_macro 16, 17, 0 + ldrepl_macro 16, 16, 17 GMUL xvf, d, U0, D0, U0 - ldrepl_macro 18, 18, 3 - nmsub_macro 1, 1, 0, D1 + ldrepl_macro 15, 18 + nmsub_macro D1, 1, 0 GMUL xvf, d, U1, D2, U1 // Store A - A_st_macro 0, 1, 0, 1 + A_st_macro 1, 0, 0, 1 // Store C GST f, d, $f0, C0, 0x00, $f1, C1, 0x00 .endm @@ -582,10 +581,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. xvld U2, C0, 0x40 xvld U3, C0, 0x60 .L_dsolve_16x1: - ldrepl_macro 16, 16, 0 + ldrepl_macro 16, 16 GMUL xvf, d, U0, D0, U0, U1, D0, U1, U2, D0, U2, U3, D0, U3 // Store A - A_st_macro 0, 3, 0, 4 + A_st_macro 4, 0, 0, 1, 2, 3 // Strore C GST xv, , U0, C0, 0x00, U1, C0, 0x20, U2, C0, 0x40, U3, C0, 0x60 .endm @@ -599,10 +598,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. xvld U0, C0, 0x00 xvld U1, C0, 0x20 .L_dsolve_8x1: - ldrepl_macro 16, 16, 0 + ldrepl_macro 16, 16 GMUL xvf, d, U0, D0, U0, U1, D0, U1 // Store A - A_st_macro 0, 1, 0, 4 + A_st_macro 4, 0, 0, 1 // Strore C GST xv, , U0, C0, 0x00, U1, C0, 0x20 .endm @@ -615,10 +614,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /* Load C0 */ xvld U0, C0, 0x00 .L_dsolve_4x1: - ldrepl_macro 16, 16, 0 + ldrepl_macro 16, 16 GMUL xvf, d, U0, D0, U0 // Store A - A_st_macro 0, 0, 0, 4 + A_st_macro 4, 0, 0 // Strore C GST xv, , U0, C0, 0x00 .endm @@ -631,10 +630,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /* Load C0 */ xvld U0, C0, 0x00 .L_dsolve_2x1: - ldrepl_macro 16, 16, 0 + ldrepl_macro 16, 16 GMUL xvf, d, U0, D0, U0 // Store A - A_st_macro 0, 0, 0, 2 + A_st_macro 2, 0, 0 // Strore C GST v, , $vr0, C0, 0x00 .endm @@ -647,16 +646,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // Load C fld.d $f0, C0, 0x00 .L_dsolve_1x1: - ldrepl_macro 16, 16, 0 + ldrepl_macro 16, 16 GMUL xvf, d, U0, D0, U0 // Store A - A_st_macro 0, 0, 0, 1 + A_st_macro 1, 0, 0 // Strore C GST f, d, $f0, C0, 0x00 .endm PROLOGUE - push_if_used 26, 32 + push_if_used 9, 8 PTR_SLLI LDC, LDC, 3 PTR_SUB KK, ZERO, OFFSET /* if (!(N >> 2)) goto L_N3 */ @@ -877,6 +876,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PTR_ADD AA, AA, T0 // aa += 1 * k .L_N1_M0: .L_N0: - pop_if_used 26, 32 + pop_if_used 9, 8 jirl $r0, $r1, 0x0 EPILOGUE diff --git a/kernel/loongarch64/dtrsm_kernel_RT_16x4_lasx.S b/kernel/loongarch64/dtrsm_kernel_RT_16x4_lasx.S index 5f86d75b52..fb08775238 100644 --- a/kernel/loongarch64/dtrsm_kernel_RT_16x4_lasx.S +++ b/kernel/loongarch64/dtrsm_kernel_RT_16x4_lasx.S @@ -111,33 +111,31 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "dtrsm_kernel_macro.S" -.macro ldrepl_macro start, end, stride +.macro ldrepl_macro stride:req, index:req, more:vararg // Load Ux (x = 0...15) -.if \start <= \end - GLDREPL xv, d, $xr\start, B0, \stride * 8 - ldrepl_macro %start + 1, \end, %stride + 1 + GLDREPL xv, d, $xr\index, B0, \index * 8 - \stride * 8 +.ifnb \more + ldrepl_macro \stride, \more .endif .endm - -.macro nmsub_macro start0, end0, start1, reg -// Ux -= reg * Dx -.if \start0 <= \end0 +.macro nmsub_macro reg:req, start0:req, start1:req, more:vararg +// Gx -= reg * Ux xvfnmsub.d $xr\start0, \reg, $xr\start1, $xr\start0 - nmsub_macro %start0 + 1, \end0, %start1 + 1, \reg +.ifnb \more + nmsub_macro \reg, \more .endif .endm - -.macro A_st_macro start, end, stride, N -// Store Ux(x = 0...15) -.if \start <= \end +.macro A_st_macro N:req, stride:req, start:req, more:vararg +// Store Gx(x = 16...31) .if \N == 4 - xvst $xr\start, A0, \stride * 0x20 + xvst $xr\start, A0, \start * 0x20 - \stride * 0x20 .elseif \N == 2 - vst $vr\start, A0, \stride * 0x10 + vst $vr\start, A0, \start * 0x10 - \stride * 0x10 .elseif \N == 1 - fst.d $f\start, A0, \stride * 0x08 + fst.d $f\start, A0, \start * 0x08 - \stride * 0x08 .endif - A_st_macro %start + 1, \end, %stride + 1, \N +.ifnb \more + A_st_macro \N, \stride, \more .endif .endm @@ -148,13 +146,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //0 //2 3 // Sequentially extract data from B in row order - ldrepl_macro 16, 16, 0 - ldrepl_macro 17, 18, 2 + ldrepl_macro 16, 16 + ldrepl_macro 15, 17, 18 GMUL xvf, d, U4, D2, U4, U5, D2, U5, U6, D2, U6, U7, D2, U7 - nmsub_macro 0, 3, 4, D1 + nmsub_macro D1, 0, 4, 1, 5, 2, 6, 3, 7 GMUL xvf, d, U0, D0, U0, U1, D0, U1, U2, D0, U2, U3, D0, U3 // Store A - A_st_macro 0, 7, 0, 4 + A_st_macro 4, 0, 0, 1, 2, 3, 4, 5, 6, 7 // Store C GST xv, , U0, C0, 0x00, U1, C0, 0x20, U2, C0, 0x40, U3, C0, 0x60, \ U4, C1, 0x00, U5, C1, 0x20, U6, C1, 0x40, U7, C1, 0x60 @@ -167,13 +165,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //0 //2 3 // Sequentially extract data from B in row order - ldrepl_macro 16, 16, 0 - ldrepl_macro 17, 18, 2 + ldrepl_macro 16, 16 + ldrepl_macro 15, 17, 18 GMUL xvf, d, U2, D2, U2, U3, D2, U3 - nmsub_macro 0, 1, 2, D1 + nmsub_macro D1, 0, 2, 1, 3 GMUL xvf, d, U0, D0, U0, U1, D0, U1 // Store A - A_st_macro 0, 3, 0, 4 + A_st_macro 4, 0, 0, 1, 2, 3 // Store C GST xv, , U0, C0, 0x00, U1, C0, 0x20, \ U2, C1, 0x00, U3, C1, 0x20 @@ -186,13 +184,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //0 //2 3 // Sequentially extract data from B in row order - ldrepl_macro 16, 16, 0 - ldrepl_macro 17, 18, 2 + ldrepl_macro 16, 16 + ldrepl_macro 15, 17, 18 GMUL xvf, d, U1, D2, U1 - nmsub_macro 0, 0, 1, D1 + nmsub_macro D1, 0, 1 GMUL xvf, d, U0, D0, U0 // Store A - A_st_macro 0, 1, 0, 4 + A_st_macro 4, 0, 0, 1 // Store C GST xv, , U0, C0, 0x00, U1, C1, 0x00 .endm @@ -204,13 +202,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //0 //2 3 // Sequentially extract data from B in row order - ldrepl_macro 16, 16, 0 - ldrepl_macro 17, 18, 2 + ldrepl_macro 16, 16 + ldrepl_macro 15, 17, 18 GMUL xvf, d, U1, D2, U1 - nmsub_macro 0, 0, 1, D1 + nmsub_macro D1, 0, 1 GMUL xvf, d, U0, D0, U0 // Store A - A_st_macro 0, 1, 0, 2 + A_st_macro 2, 0, 0, 1 // Store C GST v, , $vr0, C0, 0x00, $vr1, C1, 0x00 .endm @@ -222,13 +220,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //0 //2 3 // Sequentially extract data from B in row order - ldrepl_macro 16, 16, 0 - ldrepl_macro 17, 18, 2 + ldrepl_macro 16, 16 + ldrepl_macro 15, 17, 18 GMUL xvf, d, U1, D2, U1 - nmsub_macro 0, 0, 1, D1 + nmsub_macro D1, 0, 1 GMUL xvf, d, U0, D0, U0 // Store A - A_st_macro 0, 1, 0, 1 + A_st_macro 1, 0, 0, 1 // Store C GST f, d, $f0, C0, 0x00, $f1, C1, 0x00 .endm @@ -242,22 +240,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //8 9 10 //12 13 14 15 // Sequentially extract data from B in row order - ldrepl_macro 22, 25, 12 + ldrepl_macro 10, 22, 23, 24, 25 GMUL xvf, d, U12, D9, U12, U13, D9, U13, U14, D9, U14, U15, D9, U15 - ldrepl_macro 19, 21, 8 - nmsub_macro 8, 11, 12, D8 - ldrepl_macro 17, 18, 4 + ldrepl_macro 11, 19, 20, 21 + nmsub_macro D8, 8, 12, 9, 13, 10, 14, 11, 15 + ldrepl_macro 13, 17, 18 GMUL xvf, d, U8, D5, U8, U9, D5, U9, U10, D5, U10, U11, D5, U11 - ldrepl_macro 16, 16, 0 - nmsub_macro 4, 7, 12, D7 - nmsub_macro 4, 7, 8, D4 + ldrepl_macro 16, 16 + nmsub_macro D7, 4, 12, 5, 13, 6, 14, 7, 15 + nmsub_macro D4, 4, 8, 5, 9, 6, 10, 7, 11 GMUL xvf, d, U4, D2, U4, U5, D2, U5, U6, D2, U6, U7, D2, U7 - nmsub_macro 0, 3, 12, D6 - nmsub_macro 0, 3, 8, D3 - nmsub_macro 0, 3, 4, D1 + nmsub_macro D6, 0, 12, 1, 13, 2, 14, 3, 15 + nmsub_macro D3, 0, 8, 1, 9, 2, 10, 3, 11 + nmsub_macro D1, 0, 4, 1, 5, 2, 6, 3, 7 GMUL xvf, d, U0, D0, U0, U1, D0, U1, U2, D0, U2, U3, D0, U3 // Store A - A_st_macro 0, 15, 0, 4 + A_st_macro 4, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 // Store C GST xv, , U0, C0, 0x00, U1, C0, 0x20, U2, C0, 0x40, U3, C0, 0x60, \ U4, C1, 0x00, U5, C1, 0x20, U6, C1, 0x40, U7, C1, 0x60, \ @@ -274,22 +272,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //8 9 10 //12 13 14 15 // Sequentially extract data from B in row order - ldrepl_macro 22, 25, 12 + ldrepl_macro 10, 22, 23, 24, 25 GMUL xvf, d, U6, D9, U6, U7, D9, U7 - ldrepl_macro 19, 21, 8 - nmsub_macro 4, 5, 6, D8 - ldrepl_macro 17, 18, 4 + ldrepl_macro 11, 19, 20, 21 + nmsub_macro D8, 4, 6, 5, 7 + ldrepl_macro 13, 17, 18 GMUL xvf, d, U4, D5, U4, U5, D5, U5 - ldrepl_macro 16, 16, 0 - nmsub_macro 2, 3, 6, D7 - nmsub_macro 2, 3, 4, D4 + ldrepl_macro 16, 16 + nmsub_macro D7, 2, 6, 3, 7 + nmsub_macro D4, 2, 4, 3, 5 GMUL xvf, d, U2, D2, U2, U3, D2, U3 - nmsub_macro 0, 1, 6, D6 - nmsub_macro 0, 1, 4, D3 - nmsub_macro 0, 1, 2, D1 + nmsub_macro D6, 0, 6, 1, 7 + nmsub_macro D3, 0, 4, 1, 5 + nmsub_macro D1, 0, 2, 1, 3 GMUL xvf, d, U0, D0, U0, U1, D0, U1 // Store A - A_st_macro 0, 7, 0, 4 + A_st_macro 4, 0, 0, 1, 2, 3, 4, 5, 6, 7 // Store C GST xv, , U0, C0, 0x00, U1, C0, 0x20, \ U2, C1, 0x00, U3, C1, 0x20, \ @@ -306,22 +304,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //8 9 10 //12 13 14 15 // Sequentially extract data from B in row order - ldrepl_macro 22, 25, 12 + ldrepl_macro 10, 22, 23, 24, 25 GMUL xvf, d, U3, D9, U3 - ldrepl_macro 19, 21, 8 - nmsub_macro 2, 2, 3, D8 - ldrepl_macro 17, 18, 4 + ldrepl_macro 11, 19, 20, 21 + nmsub_macro D8, 2, 3 + ldrepl_macro 13, 17, 18 GMUL xvf, d, U2, D5, U2 - ldrepl_macro 16, 16, 0 - nmsub_macro 1, 1, 3, D7 - nmsub_macro 1, 1, 2, D4 + ldrepl_macro 16, 16 + nmsub_macro D7, 1, 3 + nmsub_macro D4, 1, 2 GMUL xvf, d, U1, D2, U1 - nmsub_macro 0, 0, 3, D6 - nmsub_macro 0, 0, 2, D3 - nmsub_macro 0, 0, 1, D1 + nmsub_macro D6, 0, 3 + nmsub_macro D3, 0, 2 + nmsub_macro D1, 0, 1 GMUL xvf, d, U0, D0, U0 // Store A - A_st_macro 0, 3, 0, 4 + A_st_macro 4, 0, 0, 1, 2, 3 // Store C GST xv, , U0, C0, 0x00, U1, C1, 0x00, U2, C2, 0x00, U3, C3, 0x00 .endm @@ -335,22 +333,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //8 9 10 //12 13 14 15 // Sequentially extract data from B in row order - ldrepl_macro 22, 25, 12 + ldrepl_macro 10, 22, 23, 24, 25 GMUL xvf, d, U3, D9, U3 - ldrepl_macro 19, 21, 8 - nmsub_macro 2, 2, 3, D8 - ldrepl_macro 17, 18, 4 + ldrepl_macro 11, 19, 20, 21 + nmsub_macro D8, 2, 3 + ldrepl_macro 13, 17, 18 GMUL xvf, d, U2, D5, U2 - ldrepl_macro 16, 16, 0 - nmsub_macro 1, 1, 3, D7 - nmsub_macro 1, 1, 2, D4 + ldrepl_macro 16, 16 + nmsub_macro D7, 1, 3 + nmsub_macro D4, 1, 2 GMUL xvf, d, U1, D2, U1 - nmsub_macro 0, 0, 3, D6 - nmsub_macro 0, 0, 2, D3 - nmsub_macro 0, 0, 1, D1 + nmsub_macro D6, 0, 3 + nmsub_macro D3, 0, 2 + nmsub_macro D1, 0, 1 GMUL xvf, d, U0, D0, U0 // Store A - A_st_macro 0, 3, 0, 2 + A_st_macro 2, 0, 0, 1, 2, 3 // Store C GST v, , $vr0, C0, 0x00, $vr1, C1, 0x00, $vr2, C2, 0x00, $vr3, C3, 0x00 .endm @@ -364,22 +362,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //8 9 10 //12 13 14 15 // Sequentially extract data from B in row order - ldrepl_macro 22, 25, 12 + ldrepl_macro 10, 22, 23, 24, 25 GMUL xvf, d, U3, D9, U3 - ldrepl_macro 19, 21, 8 - nmsub_macro 2, 2, 3, D8 - ldrepl_macro 17, 18, 4 + ldrepl_macro 11, 19, 20, 21 + nmsub_macro D8, 2, 3 + ldrepl_macro 13, 17, 18 GMUL xvf, d, U2, D5, U2 - ldrepl_macro 16, 16, 0 - nmsub_macro 1, 1, 3, D7 - nmsub_macro 1, 1, 2, D4 + ldrepl_macro 16, 16 + nmsub_macro D7, 1, 3 + nmsub_macro D4, 1, 2 GMUL xvf, d, U1, D2, U1 - nmsub_macro 0, 0, 3, D6 - nmsub_macro 0, 0, 2, D3 - nmsub_macro 0, 0, 1, D1 + nmsub_macro D6, 0, 3 + nmsub_macro D3, 0, 2 + nmsub_macro D1, 0, 1 GMUL xvf, d, U0, D0, U0 // Store A - A_st_macro 0, 3, 0, 1 + A_st_macro 1, 0, 0, 1, 2, 3 // Store C GST f, d, $f0, C0, 0x00, $f1, C1, 0x00, $f2, C2, 0x00, $f3, C3, 0x00, .endm @@ -399,10 +397,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .L_dsolve_16x1: PTR_ADDI A0, T1, -16 * 8 PTR_ADDI B0, T2, -1 * 8 - ldrepl_macro 16, 16, 0 + ldrepl_macro 16, 16 GMUL xvf, d, U0, D0, U0, U1, D0, U1, U2, D0, U2, U3, D0, U3 // Store A - A_st_macro 0, 3, 0, 4 + A_st_macro 4, 0, 0, 1, 2, 3 // Strore C GST xv, , U0, C0, 0x00, U1, C0, 0x20, U2, C0, 0x40, U3, C0, 0x60 .endm @@ -420,10 +418,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .L_dsolve_8x1: PTR_ADDI A0, T1, -8 * 8 PTR_ADDI B0, T2, -1 * 8 - ldrepl_macro 16, 16, 0 + ldrepl_macro 16, 16 GMUL xvf, d, U0, D0, U0, U1, D0, U1 // Store A - A_st_macro 0, 1, 0, 4 + A_st_macro 4, 0, 0, 1 // Strore C GST xv, , U0, C0, 0x00, U1, C0, 0x20 .endm @@ -440,10 +438,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .L_dsolve_4x1: PTR_ADDI A0, T1, -4 * 8 PTR_ADDI B0, T2, -1 * 8 - ldrepl_macro 16, 16, 0 + ldrepl_macro 16, 16 GMUL xvf, d, U0, D0, U0 // Store A - A_st_macro 0, 0, 0, 4 + A_st_macro 4, 0, 0 // Strore C GST xv, , U0, C0, 0x00 .endm @@ -460,10 +458,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .L_dsolve_2x1: PTR_ADDI A0, T1, -2 * 8 PTR_ADDI B0, T2, -1 * 8 - ldrepl_macro 16, 16, 0 + ldrepl_macro 16, 16 GMUL xvf, d, U0, D0, U0 // Store A - A_st_macro 0, 0, 0, 2 + A_st_macro 2, 0, 0 // Strore C GST v, , $vr0, C0, 0x00 .endm @@ -480,10 +478,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .L_dsolve_1x1: PTR_ADDI A0, T1, -1 * 8 PTR_ADDI B0, T2, -1 * 8 - ldrepl_macro 16, 16, 0 + ldrepl_macro 16, 16 GMUL xvf, d, U0, D0, U0 // Store A - A_st_macro 0, 0, 0, 1 + A_st_macro 1, 0, 0 // Strore C GST f, d, $f0, C0, 0x00 .endm @@ -697,7 +695,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .endm PROLOGUE - push_if_used 26, 32 + push_if_used 9, 8 PTR_SLLI LDC, LDC, 3 PTR_SUB KK, N, OFFSET PTR_MUL T0, N, LDC @@ -948,6 +946,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PTR_ADDI KK, KK, -4 bnez J, .L_J1 .L_N0: - pop_if_used 26, 32 + pop_if_used 9, 8 jirl $r0, $r1, 0x0 EPILOGUE diff --git a/kernel/loongarch64/loongarch64_asm.S b/kernel/loongarch64/loongarch64_asm.S index d097b3045e..a2221491b2 100644 --- a/kernel/loongarch64/loongarch64_asm.S +++ b/kernel/loongarch64/loongarch64_asm.S @@ -90,57 +90,175 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define PTR_FST fst.d #endif -// The max registers available to the user which -// do not need to be preserved across calls. -// Ref: https://loongson.github.io/LoongArch-Documentation/LoongArch-ELF-ABI-CN.html -#define MAX_INT_CALLER_SAVED 17 -#define MAX_FP_CALLER_SAVED 24 - .altmacro // Enable alternate macro mode +/* + * Pushing and popping static registers into/from the stack. + * regs : number of static general-purpose registers, greater than or equal to 0, less than or equal to 9 + * fregs: number of static floating-point registers, greater than or equal to 0, less than or equal to 8 + */ .macro push_if_used regs, fregs -.if \regs > MAX_INT_CALLER_SAVED - PTR_ADDI $sp, $sp, -((\regs - MAX_INT_CALLER_SAVED) << REG_LOG) - push_regs 0, \regs - MAX_INT_CALLER_SAVED - 1 +.if \regs > 0 + PTR_ADDI $sp, $sp, -(\regs << REG_LOG) + push_regs 0, \regs - 1 .endif -.if \fregs > MAX_FP_CALLER_SAVED - PTR_ADDI $sp, $sp, -((\fregs - MAX_FP_CALLER_SAVED) << FREG_LOG) - push_fregs 0, \fregs - MAX_FP_CALLER_SAVED - 1 +.if \fregs > 0 + PTR_ADDI $sp, $sp, -(\fregs << FREG_LOG) + push_fregs 0, \fregs - 1 .endif .endm // End push_if_used + .macro pop_if_used regs, fregs -.if \fregs > MAX_FP_CALLER_SAVED - pop_fregs 0, \fregs - MAX_FP_CALLER_SAVED - 1 - PTR_ADDI $sp, $sp, (\fregs - MAX_FP_CALLER_SAVED) << FREG_LOG +.if \fregs > 0 + pop_fregs 0, \fregs - 1 + PTR_ADDI $sp, $sp, \fregs << FREG_LOG .endif -.if \regs > MAX_INT_CALLER_SAVED - pop_regs 0, \regs - MAX_INT_CALLER_SAVED - 1 - PTR_ADDI $sp, $sp, (\regs - MAX_INT_CALLER_SAVED) << REG_LOG +.if \regs > 0 + pop_regs 0, \regs - 1 + PTR_ADDI $sp, $sp, \regs << REG_LOG .endif .endm // End pop_if_used + .macro push_regs from, to - PTR_ST $s\()\from, $sp, \from << REG_LOG +#ifdef __clang__ +.if \to >= 0 + PTR_ST $s0, $sp, 0 << REG_LOG +.endif +.if \to >= 1 + PTR_ST $s1, $sp, 1 << REG_LOG +.endif +.if \to >= 2 + PTR_ST $s2, $sp, 2 << REG_LOG +.endif +.if \to >= 3 + PTR_ST $s3, $sp, 3 << REG_LOG +.endif +.if \to >= 4 + PTR_ST $s4, $sp, 4 << REG_LOG +.endif +.if \to >= 5 + PTR_ST $s5, $sp, 5 << REG_LOG +.endif +.if \to >= 6 + PTR_ST $s6, $sp, 6 << REG_LOG +.endif +.if \to >= 7 + PTR_ST $s7, $sp, 7 << REG_LOG +.endif +.if \to >= 8 + PTR_ST $s8, $sp, 8 << REG_LOG +.endif +#else + PTR_ST $s\()\from, $sp, \from << REG_LOG .if \to - \from push_regs %from + 1, \to .endif +#endif .endm // End push_regs + .macro pop_regs from, to +#ifdef __clang__ +.if \to >= 0 + PTR_LD $s0, $sp, 0 << REG_LOG +.endif +.if \to >= 1 + PTR_LD $s1, $sp, 1 << REG_LOG +.endif +.if \to >= 2 + PTR_LD $s2, $sp, 2 << REG_LOG +.endif +.if \to >= 3 + PTR_LD $s3, $sp, 3 << REG_LOG +.endif +.if \to >= 4 + PTR_LD $s4, $sp, 4 << REG_LOG +.endif +.if \to >= 5 + PTR_LD $s5, $sp, 5 << REG_LOG +.endif +.if \to >= 6 + PTR_LD $s6, $sp, 6 << REG_LOG +.endif +.if \to >= 7 + PTR_LD $s7, $sp, 7 << REG_LOG +.endif +.if \to >= 8 + PTR_LD $s8, $sp, 8 << REG_LOG +.endif +#else PTR_LD $s\()\from, $sp, \from << REG_LOG .if \to - \from pop_regs %from + 1, \to .endif +#endif .endm // End pop_regs + .macro push_fregs from, to +#ifdef __clang__ +.if \to >= 0 + PTR_FST $fs0, $sp, 0 << FREG_LOG +.endif +.if \to >= 1 + PTR_FST $fs1, $sp, 1 << FREG_LOG +.endif +.if \to >= 2 + PTR_FST $fs2, $sp, 2 << FREG_LOG +.endif +.if \to >= 3 + PTR_FST $fs3, $sp, 3 << FREG_LOG +.endif +.if \to >= 4 + PTR_FST $fs4, $sp, 4 << FREG_LOG +.endif +.if \to >= 5 + PTR_FST $fs5, $sp, 5 << FREG_LOG +.endif +.if \to >= 6 + PTR_FST $fs6, $sp, 6 << FREG_LOG +.endif +.if \to >= 7 + PTR_FST $fs7, $sp, 7 << FREG_LOG +.endif +#else PTR_FST $fs\()\from, $sp, \from << FREG_LOG .if \to - \from push_fregs %from + 1, \to .endif +#endif .endm // End push_fregs + .macro pop_fregs from, to +#ifdef __clang__ +.if \to >= 0 + PTR_FLD $fs0, $sp, 0 << FREG_LOG +.endif +.if \to >= 1 + PTR_FLD $fs1, $sp, 1 << FREG_LOG +.endif +.if \to >= 2 + PTR_FLD $fs2, $sp, 2 << FREG_LOG +.endif +.if \to >= 3 + PTR_FLD $fs3, $sp, 3 << FREG_LOG +.endif +.if \to >= 4 + PTR_FLD $fs4, $sp, 4 << FREG_LOG +.endif +.if \to >= 5 + PTR_FLD $fs5, $sp, 5 << FREG_LOG +.endif +.if \to >= 6 + PTR_FLD $fs6, $sp, 6 << FREG_LOG +.endif +.if \to >= 7 + PTR_FLD $fs7, $sp, 7 << FREG_LOG +.endif +#else PTR_FLD $fs\()\from, $sp, \from << FREG_LOG .if \to - \from pop_fregs %from + 1, \to .endif +#endif .endm // End pop_fregs // @@ -275,7 +393,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // GXOR // .macro GXOR pre_op:req, suf_op:req, out:req, in0:req, in1:req, more:vararg - \pre_op\()xor.\suf_op \out, \in0, \in1 +.ifnb \pre_op + \pre_op\()xor.v \out, \in0, \in1 +.else + xor.\suf_op \out, \in0, \in1 +.endif .ifnb \more GXOR \pre_op, \suf_op, \more .endif @@ -307,6 +429,48 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. GPRELD \more .endif .endm +// +// GPACKEV +// +.macro GPACKEV pre_op:req, suf_op:req, out:req, in0:req, in1:req, more:vararg + \pre_op\()packev.\suf_op \out, \in0, \in1 +.ifnb \more + GPACKEV \pre_op, \suf_op, \more +.endif +.endm +// +// GPACKOD +// +.macro GPACKOD pre_op:req, suf_op:req, out:req, in0:req, in1:req, more:vararg + \pre_op\()packod.\suf_op \out, \in0, \in1 +.ifnb \more + GPACKOD \pre_op, \suf_op, \more +.endif +.endm +// +// GSHUF4I +// +.macro GSHUF4I pre_op:req, suf_op:req, out:req, in0:req, in1:req /* imm */, more:vararg + \pre_op\()shuf4i.\suf_op \out, \in0, \in1 +.ifnb \more + GSHUF4I \pre_op, \suf_op, \more +.endif +.endm + +.macro TRANSF2G name, pre_op:req, suf_op:req, more:vararg +.ifeqs "\pre_op\()\suf_op", "vfs" + \name v, w, \more +.endif +.ifeqs "\pre_op\()\suf_op", "vfd" + \name v, d, \more +.endif +.ifeqs "\pre_op\()\suf_op", "xvfs" + \name xv, w, \more +.endif +.ifeqs "\pre_op\()\suf_op", "xvfd" + \name xv, d, \more +.endif +.endm // // Compound instructions @@ -314,61 +478,96 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // GACC: Accumulate the values of vector registers // .macro GACC pre_op:req, suf_op:req, out:req, in:req, more:vararg -.ifeqs "\pre_op", "xvf" +.ifeqs "\pre_op\()\suf_op", "xvfd" + xvpermi.q \out, \in, 0x01 + \pre_op\()add.\suf_op \in, \out, \in + xvpackod.d \out, \in, \in + \pre_op\()add.\suf_op \out, \out, \in +.endif +.ifeqs "\pre_op\()\suf_op", "xvfs" xvpermi.q \out, \in, 0x01 \pre_op\()add.\suf_op \in, \out, \in xvpackod.d \out, \in, \in \pre_op\()add.\suf_op \out, \out, \in -.ifeqs "\suf_op", "s" xvpackod.w \in, \out, \out \pre_op\()add.\suf_op \out, \out, \in .endif +.ifeqs "\pre_op\()\suf_op", "vfd" + vpackod.d \out, \in, \in + \pre_op\()add.\suf_op \out, \out, \in .endif - -.ifeqs "\pre_op", "vf" +.ifeqs "\pre_op\()\suf_op", "vfs" vpackod.d \out, \in, \in \pre_op\()add.\suf_op \out, \out, \in -.ifeqs "\suf_op", "s" vpackod.w \in, \out, \out \pre_op\()add.\suf_op \out, \out, \in .endif -.endif -.ifeqs "\pre_op", "xv" +.ifeqs "\pre_op\()\suf_op", "xvd" + xvpermi.q \out, \in, 0x01 + \pre_op\()add.\suf_op \in, \out, \in + xvpackod.d \out, \in, \in + \pre_op\()add.\suf_op \out, \out, \in +.endif +.ifeqs "\pre_op\()\suf_op", "xvw" + xvpermi.q \out, \in, 0x01 + \pre_op\()add.\suf_op \in, \out, \in + xvpackod.d \out, \in, \in + \pre_op\()add.\suf_op \out, \out, \in + xvpackod.w \in, \out, \out + \pre_op\()add.\suf_op \out, \out, \in +.endif +.ifeqs "\pre_op\()\suf_op", "xvh" + xvpermi.q \out, \in, 0x01 + \pre_op\()add.\suf_op \in, \out, \in + xvpackod.d \out, \in, \in + \pre_op\()add.\suf_op \out, \out, \in + xvpackod.w \in, \out, \out + \pre_op\()add.\suf_op \out, \out, \in + xvpackod.h \in, \out, \out + \pre_op\()add.\suf_op \out, \out, \in +.endif +.ifeqs "\pre_op\()\suf_op", "xvb" xvpermi.q \out, \in, 0x01 \pre_op\()add.\suf_op \in, \out, \in xvpackod.d \out, \in, \in \pre_op\()add.\suf_op \out, \out, \in -.ifnc "\suf_op", "d" xvpackod.w \in, \out, \out \pre_op\()add.\suf_op \out, \out, \in -.ifnc "\suf_op", "w" xvpackod.h \in, \out, \out \pre_op\()add.\suf_op \out, \out, \in -.ifnc "\suf_op", "h" xvpackod.b \in, \out, \out \pre_op\()add.\suf_op \out, \out, \in .endif + +.ifeqs "\pre_op\()\suf_op", "vd" + vpackod.d \out, \in, \in + \pre_op\()add.\suf_op \out, \out, \in .endif +.ifeqs "\pre_op\()\suf_op", "vw" + vpackod.d \out, \in, \in + \pre_op\()add.\suf_op \out, \out, \in + vpackod.w \in, \out, \out + \pre_op\()add.\suf_op \out, \out, \in .endif +.ifeqs "\pre_op\()\suf_op", "vh" + vpackod.d \out, \in, \in + \pre_op\()add.\suf_op \out, \out, \in + vpackod.w \in, \out, \out + \pre_op\()add.\suf_op \out, \out, \in + vpackod.h \in, \out, \out + \pre_op\()add.\suf_op \out, \out, \in .endif - -.ifeqs "\pre_op", "v" +.ifeqs "\pre_op\()\suf_op", "vb" vpackod.d \out, \in, \in \pre_op\()add.\suf_op \out, \out, \in -.ifnc "\suf_op", "d" vpackod.w \in, \out, \out \pre_op\()add.\suf_op \out, \out, \in -.ifnc "\suf_op", "w" vpackod.h \in, \out, \out \pre_op\()add.\suf_op \out, \out, \in -.ifnc "\suf_op", "h" vpackod.b \in, \out, \out \pre_op\()add.\suf_op \out, \out, \in .endif -.endif -.endif -.endif .ifnb \more GACC \pre_op, \suf_op, \more @@ -391,27 +590,27 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // Note: When "pre_op = xvf && suf_op = s", in will be modified. // .macro GCOMPLEXACC pre_op:req, suf_op:req, out:req, in:req, more:vararg -.ifeqs "\pre_op", "xvf" +.ifeqs "\pre_op\()\suf_op", "xvfd" + xvpermi.q \out, \in, 0x01 + \pre_op\()add.\suf_op \out, \out, \in +.endif + +.ifeqs "\pre_op\()\suf_op", "xvfs" xvpermi.q \out, \in, 0x01 -.ifeqs "\suf_op", "s" \pre_op\()add.\suf_op \in, \out, \in xvpackod.d \out, \in, \in \pre_op\()add.\suf_op \out, \out, \in -.else - \pre_op\()add.\suf_op \out, \out, \in .endif + +.ifeqs "\pre_op\()\suf_op", "vfd" + vor.v \out, \in, \in .endif -.ifeqs "\pre_op", "vf" -.ifeqs "\suf_op", "s" +.ifeqs "\pre_op\()\suf_op", "vfs" vpackod.d \out, \in, \in \pre_op\()add.\suf_op \out, \out, \in -.else - vor.v \out, \in, \in -.endif .endif - .ifnb \more GCOMPLEXACC \pre_op, \suf_op, \more .endif @@ -430,56 +629,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // suf_op: s or d, differentiate between single precision or double precision complex numbers // .macro GCOMPLEXMUL xconj=0, pre_op:req, suf_op:req, out:req, in0:req, in1:req, tmp0:req, tmp1:req, tmp2:req, more:vararg -.ifeqs "\pre_op", "xvf" - xvxor.v \tmp1, \tmp1, \tmp1 -.ifeqs "\suf_op", "s" - xvpackev.w \tmp0, \in0, \in0 -.else - xvpackev.d \tmp0, \in0, \in0 -.endif -.else - vxor.v \tmp1, \tmp1, \tmp1 -.ifeqs "\suf_op", "s" - vpackev.w \tmp0, \in0, \in0 -.else - vpackev.d \tmp0, \in0, \in0 -.endif -.endif + TRANSF2G GXOR, \pre_op, s, \tmp1, \tmp1, \tmp1 + TRANSF2G GPACKEV, \pre_op, \suf_op, \tmp0, \in0, \in0 \pre_op\()sub.\suf_op \tmp1, \tmp1, \in0 -.ifeqs "\pre_op", "xvf" -.ifeqs "\suf_op", "s" .ifeqs "\xconj", "0" - xvpackod.w \tmp1, \in0, \tmp1 + TRANSF2G GPACKOD, \pre_op, \suf_op, \tmp1, \in0, \tmp1 .else - xvpackod.w \tmp1, \tmp1, \in0 + TRANSF2G GPACKOD, \pre_op, \suf_op, \tmp1, \tmp1, \in0 .endif - xvshuf4i.w \tmp2, \in1, 0xb1 -.else -.ifeqs "\xconj", "0" - xvpackod.d \tmp1, \in0, \tmp1 -.else - xvpackod.d \tmp1, \tmp1, \in0 -.endif - xvshuf4i.d \tmp2, \in1, 0x0b -.endif -.else + .ifeqs "\suf_op", "s" -.ifeqs "\xconj", "0" - vpackod.w \tmp1, \in0, \tmp1 -.else - vpackod.w \tmp1, \tmp1, \in0 -.endif - vshuf4i.w \tmp2, \in1, 0xb1 -.else -.ifeqs "\xconj", "0" - vpackod.d \tmp1, \in0, \tmp1 + TRANSF2G GSHUF4I, \pre_op, \suf_op, \tmp2, \in1, 0xb1 .else - vpackod.d \tmp1, \tmp1, \in0 -.endif - vshuf4i.d \tmp2, \in1, 0x0b -.endif + TRANSF2G GSHUF4I, \pre_op, \suf_op, \tmp2, \in1, 0x0b .endif \pre_op\()mul.\suf_op \out, \tmp0, \in1 @@ -512,112 +676,58 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // suf_op: s or d, differentiate between single precision or double precision complex numbers // .macro GCOMPLEXMADD xconj=0, conj=0, pre_op:req, suf_op:req, out:req, in0:req, in1:req, in2:req, tmp0:req, tmp1:req, tmp2:req, more:vararg -.ifeqs "\pre_op", "xvf" - xvxor.v \tmp1, \tmp1, \tmp1 -.ifeqs "\suf_op", "s" - xvpackev.w \tmp0, \in0, \in0 -.else - xvpackev.d \tmp0, \in0, \in0 -.endif -.else - vxor.v \tmp1, \tmp1, \tmp1 -.ifeqs "\suf_op", "s" - vpackev.w \tmp0, \in0, \in0 -.else - vpackev.d \tmp0, \in0, \in0 -.endif -.endif + TRANSF2G GXOR, \pre_op, s, \tmp1, \tmp1, \tmp1 + TRANSF2G GPACKEV, \pre_op, \suf_op, \tmp0, \in0, \in0 \pre_op\()madd.\suf_op \tmp2, \tmp0, \in1, \in2 -.ifeqs "\conj", "1" + +.ifeqs "\conj\()\suf_op", "1s" \pre_op\()nmsub.\suf_op \tmp0, \tmp0, \in1, \in2 -.ifeqs "\pre_op", "xvf" -.ifeqs "\suf_op", "s" - xvshuf4i.w \tmp0, \tmp0, 0xb1 - xvpackev.w \out, \tmp0, \tmp2 -.else - xvshuf4i.d \tmp0, \tmp0, 0x0b - xvpackev.d \out, \tmp0, \tmp2 + TRANSF2G GSHUF4I, \pre_op, \suf_op, \tmp0, \tmp0, 0xb1 + TRANSF2G GPACKEV, \pre_op, \suf_op, \out, \tmp0, \tmp2 .endif -.else -.ifeqs "\suf_op", "s" - vshuf4i.w \tmp0, \tmp0, 0xb1 - vpackev.w \out, \tmp0, \tmp2 -.else - vshuf4i.d \tmp0, \tmp0, 0x0b - vpackev.d \out, \tmp0, \tmp2 +.ifeqs "\conj\()\suf_op", "1d" + \pre_op\()nmsub.\suf_op \tmp0, \tmp0, \in1, \in2 + TRANSF2G GSHUF4I, \pre_op, \suf_op, \tmp0, \tmp0, 0x0b + TRANSF2G GPACKEV, \pre_op, \suf_op, \out, \tmp0, \tmp2 .endif -.endif /* pre_op = xvf */ -.else +.ifeqs "\conj", "0" \pre_op\()add.\suf_op \out, \tmp2, \tmp1 -.endif /* conj = 1 */ +.endif \pre_op\()sub.\suf_op \tmp1, \tmp1, \in0 -.ifeqs "\pre_op", "xvf" -.ifeqs "\suf_op", "s" -.ifeqs "\conj", "0" -.ifeqs "\xconj", "0" - xvpackod.w \tmp1, \in0, \tmp1 -.else - xvpackod.w \tmp1, \tmp1, \in0 +.ifeqs "\xconj\()\conj\()\suf_op", "00s" + TRANSF2G GPACKOD, \pre_op, \suf_op, \tmp1, \in0, \tmp1 + TRANSF2G GSHUF4I, \pre_op, \suf_op, \tmp2, \in1, 0xb1 .endif -.else -.ifeqs "\xconj", "0" - xvpackod.w \tmp1, \in0, \in0 -.else - xvpackod.w \tmp1, \tmp1, \tmp1 +.ifeqs "\xconj\()\conj\()\suf_op", "10s" + TRANSF2G GPACKOD, \pre_op, \suf_op, \tmp1, \tmp1, \in0 + TRANSF2G GSHUF4I, \pre_op, \suf_op, \tmp2, \in1, 0xb1 .endif +.ifeqs "\xconj\()\conj\()\suf_op", "01s" + TRANSF2G GPACKOD, \pre_op, \suf_op, \tmp1, \in0, \in0 + TRANSF2G GSHUF4I, \pre_op, \suf_op, \tmp2, \in1, 0xb1 .endif - xvshuf4i.w \tmp2, \in1, 0xb1 -.else -.ifeqs "\conj", "0" -.ifeqs "\xconj", "0" - xvpackod.d \tmp1, \in0, \tmp1 -.else - xvpackod.d \tmp1, \tmp1, \in0 -.endif -.else -.ifeqs "\xconj", "0" - xvpackod.d \tmp1, \in0, \in0 -.else - xvpackod.d \tmp1, \tmp1, \tmp1 -.endif -.endif - xvshuf4i.d \tmp2, \in1, 0x0b -.endif -.else -.ifeqs "\suf_op", "s" -.ifeqs "\conj", "0" -.ifeqs "\xconj", "0" - vpackod.w \tmp1, \in0, \tmp1 -.else - vpackod.w \tmp1, \tmp1, \in0 +.ifeqs "\xconj\()\conj\()\suf_op", "11s" + TRANSF2G GPACKOD, \pre_op, \suf_op, \tmp1, \tmp1, \tmp1 + TRANSF2G GSHUF4I, \pre_op, \suf_op, \tmp2, \in1, 0xb1 .endif -.else -.ifeqs "\xconj", "0" - vpackod.w \tmp1, \in0, \in0 -.else - vpackod.w \tmp1, \tmp1, \tmp1 -.endif -.endif - vshuf4i.w \tmp2, \in1, 0xb1 -.else -.ifeqs "\conj", "0" -.ifeqs "\xconj", "0" - vpackod.d \tmp1, \in0, \tmp1 -.else - vpackod.d \tmp1, \tmp1, \in0 -.endif -.else -.ifeqs "\xconj", "0" - vpackod.d \tmp1, \in0, \in0 -.else - vpackod.d \tmp1, \tmp1, \tmp1 +.ifeqs "\xconj\()\conj\()\suf_op", "00d" + TRANSF2G GPACKOD, \pre_op, \suf_op, \tmp1, \in0, \tmp1 + TRANSF2G GSHUF4I, \pre_op, \suf_op, \tmp2, \in1, 0x0b .endif +.ifeqs "\xconj\()\conj\()\suf_op", "10d" + TRANSF2G GPACKOD, \pre_op, \suf_op, \tmp1, \tmp1, \in0 + TRANSF2G GSHUF4I, \pre_op, \suf_op, \tmp2, \in1, 0x0b .endif - vshuf4i.d \tmp2, \in1, 0x0b +.ifeqs "\xconj\()\conj\()\suf_op", "01d" + TRANSF2G GPACKOD, \pre_op, \suf_op, \tmp1, \in0, \in0 + TRANSF2G GSHUF4I, \pre_op, \suf_op, \tmp2, \in1, 0x0b .endif +.ifeqs "\xconj\()\conj\()\suf_op", "11d" + TRANSF2G GPACKOD, \pre_op, \suf_op, \tmp1, \tmp1, \tmp1 + TRANSF2G GSHUF4I, \pre_op, \suf_op, \tmp2, \in1, 0x0b .endif \pre_op\()madd.\suf_op \out, \tmp1, \tmp2, \out diff --git a/kernel/loongarch64/sgemm_kernel_16x8_lasx.S b/kernel/loongarch64/sgemm_kernel_16x8_lasx.S index bd609394e8..c6d1aeaef7 100644 --- a/kernel/loongarch64/sgemm_kernel_16x8_lasx.S +++ b/kernel/loongarch64/sgemm_kernel_16x8_lasx.S @@ -837,7 +837,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .endm PROLOGUE - push_if_used 26, 32 + push_if_used 9, 8 xvreplve0.w VALPHA, $xr0 #if defined (TRMMKERNEL) && !defined(LEFT) PTR_SUB OFF, ZERO, OFFSET @@ -2343,6 +2343,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif // #if defined(TRMMKERNEL) .L_N1_M0: .L_N0: - pop_if_used 26, 32 + pop_if_used 9, 8 jirl $r0, $r1, 0x0 EPILOGUE diff --git a/kernel/loongarch64/sgemm_ncopy_16_lasx.S b/kernel/loongarch64/sgemm_ncopy_16_lasx.S index 266c07c5cd..1a81ce601d 100644 --- a/kernel/loongarch64/sgemm_ncopy_16_lasx.S +++ b/kernel/loongarch64/sgemm_ncopy_16_lasx.S @@ -135,7 +135,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //.L_N0 PROLOGUE - push_if_used 26, 32 + push_if_used 9, 8 move TD, DST move TS, SRC @@ -458,6 +458,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PTR_ADDI M, M, -1 blt ZERO, M, .L_N1_M1 .L_N0: - pop_if_used 26, 32 + pop_if_used 9, 8 jirl $r0, $r1, 0x0 EPILOGUE diff --git a/kernel/loongarch64/sgemm_ncopy_8_lasx.S b/kernel/loongarch64/sgemm_ncopy_8_lasx.S index 5c173568bd..db36827d5e 100644 --- a/kernel/loongarch64/sgemm_ncopy_8_lasx.S +++ b/kernel/loongarch64/sgemm_ncopy_8_lasx.S @@ -110,7 +110,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //.L_N0 PROLOGUE - push_if_used 17, 20 + push_if_used 0, 0 move TD, DST move TS, SRC @@ -293,6 +293,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PTR_ADDI M, M, -1 blt ZERO, M, .L_N1_M1 .L_N0: - pop_if_used 17, 20 + pop_if_used 0, 0 jirl $r0, $r1, 0x0 EPILOGUE diff --git a/kernel/loongarch64/sgemm_tcopy_16_lasx.S b/kernel/loongarch64/sgemm_tcopy_16_lasx.S index d9789bdcd4..fc42ae8c26 100644 --- a/kernel/loongarch64/sgemm_tcopy_16_lasx.S +++ b/kernel/loongarch64/sgemm_tcopy_16_lasx.S @@ -118,7 +118,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //.L_M0 PROLOGUE - push_if_used 24, 8 + push_if_used 7, 0 move S0, SRC move P0, DST @@ -521,6 +521,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PTR_ADDI S1, S1, 0x04 PTR_ADDI P5, P5, 0x04 .L_M0: - pop_if_used 24, 8 + pop_if_used 7, 0 jirl $r0, $r1, 0x00 EPILOGUE diff --git a/kernel/loongarch64/sgemm_tcopy_8_lasx.S b/kernel/loongarch64/sgemm_tcopy_8_lasx.S index 725a47a604..73d08fb8bb 100644 --- a/kernel/loongarch64/sgemm_tcopy_8_lasx.S +++ b/kernel/loongarch64/sgemm_tcopy_8_lasx.S @@ -110,7 +110,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. //.L_M0 PROLOGUE - push_if_used 23, 8 + push_if_used 6, 0 move S0, SRC move P0, DST @@ -401,6 +401,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PTR_ADDI S1, S1, 0x04 PTR_ADDI P4, P4, 0x04 .L_M0: - pop_if_used 23, 8 + pop_if_used 6, 0 jirl $r0, $r1, 0x00 EPILOGUE diff --git a/kernel/loongarch64/sgemv_n_8_lasx.S b/kernel/loongarch64/sgemv_n_8_lasx.S index 52ffc320ea..8648c22124 100644 --- a/kernel/loongarch64/sgemv_n_8_lasx.S +++ b/kernel/loongarch64/sgemv_n_8_lasx.S @@ -418,7 +418,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE PTR_LD INC_Y, $sp, 0 - push_if_used 17 + 7, 19 + push_if_used 7, 0 PTR_ADDI K, $r0, 0x01 PTR_SUB I, INC_X, K PTR_SUB J, INC_Y, K @@ -458,6 +458,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .L_GAP_1_1: /* if (inc_x != 1) && (incy != 1) */ SGEMV_N_LASX GAP_1_1, X_8_GAP, X_4_GAP, X_2_GAP, X_1, Y_8_GAP, Y_4_GAP, Y_1 .L_END: - pop_if_used 17 + 7, 19 + pop_if_used 7, 0 jirl $r0, $r1, 0x0 EPILOGUE diff --git a/kernel/loongarch64/sgemv_t_8_lasx.S b/kernel/loongarch64/sgemv_t_8_lasx.S index f4bfffb425..1f843cadb5 100644 --- a/kernel/loongarch64/sgemv_t_8_lasx.S +++ b/kernel/loongarch64/sgemv_t_8_lasx.S @@ -369,7 +369,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE PTR_LD INC_Y, $sp, 0 - push_if_used 17 + 8, 18 + push_if_used 8, 0 PTR_ADDI K, $r0, 0x01 PTR_SUB I, INC_X, K maskeqz I, K, I /* if(inc_x == 1) I = 0; else I = 1; */ @@ -400,6 +400,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .L_GAP_1: /* if (incx != 1) */ SGEMV_T_LASX GAP_1, X8_GAP, X4_GAP .L_END: - pop_if_used 17 + 8, 18 + pop_if_used 8, 0 jirl $r0, $r1, 0x0 EPILOGUE diff --git a/kernel/loongarch64/zgemv_n_2_lsx.S b/kernel/loongarch64/zgemv_n_2_lsx.S index efb3761187..d681540085 100644 --- a/kernel/loongarch64/zgemv_n_2_lsx.S +++ b/kernel/loongarch64/zgemv_n_2_lsx.S @@ -253,7 +253,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE PTR_LD INC_Y, $sp, 0 - push_if_used 17 + 7, 31 + push_if_used 7, 7 PTR_ADDI K, $r0, 0x01 PTR_SUB I, INC_X, K PTR_SUB J, INC_Y, K @@ -291,6 +291,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .L_GAP_1_1: /* if (inc_x != 1) && (incy != 1) */ ZGEMV_N_LSX GAP_1_1, X_2_GAP, X_1, Y_2_GAP, Y_1 .L_END: - pop_if_used 17 + 7, 31 + pop_if_used 7, 7 jirl $r0, $r1, 0x0 EPILOGUE diff --git a/kernel/loongarch64/zgemv_n_4_lasx.S b/kernel/loongarch64/zgemv_n_4_lasx.S index 26edf1ed7f..2e0e0a06d9 100644 --- a/kernel/loongarch64/zgemv_n_4_lasx.S +++ b/kernel/loongarch64/zgemv_n_4_lasx.S @@ -298,7 +298,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE PTR_LD INC_Y, $sp, 0 - push_if_used 17 + 7, 31 + push_if_used 7, 7 PTR_ADDI K, $r0, 0x01 PTR_SUB I, INC_X, K PTR_SUB J, INC_Y, K @@ -337,7 +337,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .L_GAP_1_1: /* if (inc_x != 1) && (incy != 1) */ ZGEMV_N_LASX GAP_1_1, X_4_GAP, X_1, Y_4_GAP, Y_1 .L_END: - pop_if_used 17 + 7, 31 + pop_if_used 7, 7 jirl $r0, $r1, 0x0 EPILOGUE diff --git a/kernel/loongarch64/zgemv_t_2_lsx.S b/kernel/loongarch64/zgemv_t_2_lsx.S index 2a0fc172e9..cae2a0ce4c 100644 --- a/kernel/loongarch64/zgemv_t_2_lsx.S +++ b/kernel/loongarch64/zgemv_t_2_lsx.S @@ -234,7 +234,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE PTR_LD INC_Y, $sp, 0 - push_if_used 17 + 8, 30 + push_if_used 8, 6 PTR_ADDI K, $r0, 0x01 PTR_SUB I, INC_X, K maskeqz I, K, I /* if(inc_x == 1) I = 0; else I = 1; */ @@ -263,6 +263,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .L_GAP_1: /* if (incx != 1) */ ZGEMV_T_LSX GAP_1, X2_GAP .L_END: - pop_if_used 17 + 8, 30 + pop_if_used 8, 6 jirl $r0, $r1, 0x0 EPILOGUE diff --git a/kernel/loongarch64/zgemv_t_4_lasx.S b/kernel/loongarch64/zgemv_t_4_lasx.S index 4d33b8f96c..50dd73adc9 100644 --- a/kernel/loongarch64/zgemv_t_4_lasx.S +++ b/kernel/loongarch64/zgemv_t_4_lasx.S @@ -264,7 +264,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE PTR_LD INC_Y, $sp, 0 - push_if_used 17 + 8, 30 + push_if_used 8, 6 PTR_ADDI K, $r0, 0x01 PTR_SUB I, INC_X, K maskeqz I, K, I /* if(inc_x == 1) I = 0; else I = 1; */ @@ -294,6 +294,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .L_GAP_1: /* if (incx != 1) */ ZGEMV_T_LASX GAP_1, X4_GAP .L_END: - pop_if_used 17 + 8, 30 + pop_if_used 8, 6 jirl $r0, $r1, 0x0 EPILOGUE From 1cdad097603ea9a2cae78dcceb9d63cd0300034c Mon Sep 17 00:00:00 2001 From: gxw Date: Mon, 15 Apr 2024 11:04:08 +0800 Subject: [PATCH 047/267] CI: Add clang test for loongarch64 --- .github/workflows/loongarch64_clang.yml | 122 ++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 .github/workflows/loongarch64_clang.yml diff --git a/.github/workflows/loongarch64_clang.yml b/.github/workflows/loongarch64_clang.yml new file mode 100644 index 0000000000..b32e4d7319 --- /dev/null +++ b/.github/workflows/loongarch64_clang.yml @@ -0,0 +1,122 @@ +name: loongarch64 clang qemu test + +on: [push, pull_request] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + TEST: + if: "github.repository == 'OpenMathLib/OpenBLAS'" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - target: LOONGSONGENERIC + opts: NO_SHARED=1 DYNAMIC_ARCH=1 TARGET=LOONGSONGENERIC + - target: LOONGSON3R5 + opts: NO_SHARED=1 DYNAMIC_ARCH=1 TARGET=LOONGSON3R5 + - target: LOONGSON2K1000 + opts: NO_SHARED=1 DYNAMIC_ARCH=1 TARGET=LOONGSON2K1000 + - target: DYNAMIC_ARCH + opts: NO_SHARED=1 DYNAMIC_ARCH=1 TARGET=GENERIC + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install libffi6 + run: | + wget http://ftp.ca.debian.org/debian/pool/main/libf/libffi/libffi6_3.2.1-9_amd64.deb + sudo dpkg -i libffi6_3.2.1-9_amd64.deb + + - name: Install APT deps + run: | + sudo add-apt-repository ppa:savoury1/virtualisation + sudo apt-get update + sudo apt-get install autoconf automake autotools-dev ninja-build make ccache \ + qemu-user-static + + - name: Download and install loongarch64-toolchain + run: | + wget http://ftp.loongnix.cn/toolchain/llvm/llvm8/clang+llvm_8.0.1-6_amd64-linux-gnu_debian-10.tar.gz + wget http://ftp.loongnix.cn/toolchain/gcc/release/loongarch/gcc8/loongson-gnu-toolchain-8.3-x86_64-loongarch64-linux-gnu-rc1.3.tar.xz + tar -xf clang+llvm_8.0.1-6_amd64-linux-gnu_debian-10.tar.gz -C /opt + tar -xf loongson-gnu-toolchain-8.3-x86_64-loongarch64-linux-gnu-rc1.3.tar.xz -C /opt + + - name: Set env + run: | + echo "PATH=$GITHUB_WORKSPACE:/opt/clang+llvm_8.0.1-6_amd64-linux-gnu_debian-10/bin:/opt/loongson-gnu-toolchain-8.3-x86_64-loongarch64-linux-gnu-rc1.3/bin:$PATH" >> $GITHUB_ENV + + - name: Compilation cache + uses: actions/cache@v3 + with: + path: ~/.ccache + key: ccache-${{ runner.os }}-${{ matrix.target }}-${{ github.ref }}-${{ github.sha }} + restore-keys: | + ccache-${{ runner.os }}-${{ matrix.target }}-${{ github.ref }} + ccache-${{ runner.os }}-${{ matrix.target }} + + - name: Configure ccache + run: | + test -d ~/.ccache || mkdir -p ~/.ccache + echo "max_size = 300M" > ~/.ccache/ccache.conf + echo "compression = true" >> ~/.ccache/ccache.conf + ccache -s + + - name: Disable utest dsdot:dsdot_n_1 + run: | + echo -n > utest/test_dsdot.c + echo "Due to the qemu versions 7.2 causing utest cases to fail," + echo "the utest dsdot:dsdot_n_1 have been temporarily disabled." + + - name: Build OpenBLAS + run: make CC='ccache clang --target=loongarch64-linux-gnu --sysroot=/opt/loongson-gnu-toolchain-8.3-x86_64-loongarch64-linux-gnu-rc1.3/loongarch64-linux-gnu/sysroot/ -static' FC='ccache loongarch64-linux-gnu-gfortran -static' HOSTCC='ccache clang' CROSS_SUFFIX=llvm- NO_SHARED=1 ${{ matrix.opts }} -j$(nproc) + + - name: Test + run: | + qemu-loongarch64-static ./utest/openblas_utest + qemu-loongarch64-static ./utest/openblas_utest_ext + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xscblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xdcblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xccblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xzcblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xscblat2 < ./ctest/sin2 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xdcblat2 < ./ctest/din2 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xccblat2 < ./ctest/cin2 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xzcblat2 < ./ctest/zin2 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xscblat3 < ./ctest/sin3 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xdcblat3 < ./ctest/din3 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xccblat3 < ./ctest/cin3 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xzcblat3 < ./ctest/zin3 + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/sblat1 + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/dblat1 + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/cblat1 + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/zblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/sblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/dblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/cblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/zblat1 + rm -f ./test/?BLAT2.SUMM + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/sblat2 < ./test/sblat2.dat + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/dblat2 < ./test/dblat2.dat + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/cblat2 < ./test/cblat2.dat + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/zblat2 < ./test/zblat2.dat + rm -f ./test/?BLAT2.SUMM + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/sblat2 < ./test/sblat2.dat + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/dblat2 < ./test/dblat2.dat + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/cblat2 < ./test/cblat2.dat + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/zblat2 < ./test/zblat2.dat + rm -f ./test/?BLAT3.SUMM + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/sblat3 < ./test/sblat3.dat + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/dblat3 < ./test/dblat3.dat + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/cblat3 < ./test/cblat3.dat + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/zblat3 < ./test/zblat3.dat + rm -f ./test/?BLAT3.SUMM + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/sblat3 < ./test/sblat3.dat + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/dblat3 < ./test/dblat3.dat + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/cblat3 < ./test/cblat3.dat + OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/zblat3 < ./test/zblat3.dat + From 992b71fea2e3916c294dd6a90062b4c31740cd85 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 23 Apr 2024 21:52:26 +0200 Subject: [PATCH 048/267] remove stray comma --- kernel/loongarch64/dot_lasx.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/loongarch64/dot_lasx.S b/kernel/loongarch64/dot_lasx.S index 0715b63119..11c896cb9a 100644 --- a/kernel/loongarch64/dot_lasx.S +++ b/kernel/loongarch64/dot_lasx.S @@ -165,7 +165,7 @@ PROLOGUE /* store dot in s1 $f8 */ #ifdef DSDOT xvfadd.d $xr8, $xr8, $xr9 - fsub.s s2, s2, s2, /* set s2 to 0.0 */ + fsub.s s2, s2, s2 /* set s2 to 0.0 */ xvpermi.q $xr0, $xr8, 0x1 vfadd.d $vr8, $vr8, $vr0 vpackod.d $vr0, $vr8, $vr8 From 015042f7b5552d93a502e32faac3812378cba332 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 23 Apr 2024 21:55:57 +0200 Subject: [PATCH 049/267] Fix Loongson compiler flag test --- cmake/cc.cmake | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmake/cc.cmake b/cmake/cc.cmake index 2da941afb0..0a498510aa 100644 --- a/cmake/cc.cmake +++ b/cmake/cc.cmake @@ -2,6 +2,7 @@ ## Author: Hank Anderson ## Description: Ported from portion of OpenBLAS/Makefile.system ## Sets C related variables. +include(CheckCCompilerFlag) if (${CMAKE_C_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_C_COMPILER_ID} STREQUAL "LSB" OR ${CMAKE_C_COMPILER_ID} MATCHES "Clang") @@ -36,14 +37,14 @@ if (${CMAKE_C_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_C_COMPILER_ID} STREQUAL "LS if (LOONGARCH64) if (BINARY64) - CHECK_CXX_COMPILER_FLAG("-mabi=lp64d" COMPILER_SUPPORT_LP64D_ABI) + CHECK_C_COMPILER_FLAG("-mabi=lp64d" COMPILER_SUPPORT_LP64D_ABI) if(COMPILER_SUPPORT_LP64D_ABI) set(CCOMMON_OPT "${CCOMMON_OPT} -mabi=lp64d") else() set(CCOMMON_OPT "${CCOMMON_OPT} -mabi=lp64") endif () else () - CHECK_CXX_COMPILER_FLAG("-mabi=ilp32d" COMPILER_SUPPORT_ILP32D_ABI) + CHECK_C_COMPILER_FLAG("-mabi=ilp32d" COMPILER_SUPPORT_ILP32D_ABI) if(COMPILER_SUPPORT_ILP32D_ABI) set(CCOMMON_OPT "${CCOMMON_OPT} -mabi=ilp32d") else() From 69aa93e34f053611ed09da28862da9c22fab9b55 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 23 Apr 2024 21:57:42 +0200 Subject: [PATCH 050/267] Fix Loongson compiler flag check --- cmake/fc.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/fc.cmake b/cmake/fc.cmake index 4d3da1a29b..176d3d4388 100644 --- a/cmake/fc.cmake +++ b/cmake/fc.cmake @@ -61,14 +61,14 @@ if (${F_COMPILER} STREQUAL "GFORTRAN" OR ${F_COMPILER} STREQUAL "F95" OR CMAKE_F endif () if (LOONGARCH64) if (BINARY64) - CHECK_CXX_COMPILER_FLAG("-mabi=lp64d" COMPILER_SUPPORT_LP64D_ABI) + CHECK_C_COMPILER_FLAG("-mabi=lp64d" COMPILER_SUPPORT_LP64D_ABI) if(COMPILER_SUPPORT_LP64D_ABI) set(FCOMMON_OPT "${FCOMMON_OPT} -mabi=lp64d") else() set(FCOMMON_OPT "${FCOMMON_OPT} -mabi=lp64") endif () else () - CHECK_CXX_COMPILER_FLAG("-mabi=ilp32d" COMPILER_SUPPORT_ILP32D_ABI) + CHECK_C_COMPILER_FLAG("-mabi=ilp32d" COMPILER_SUPPORT_ILP32D_ABI) if(COMPILER_SUPPORT_ILP32D_ABI) set(FCOMMON_OPT "${FCOMMON_OPT} -mabi=ilp32d") else() From faf7b3d1bb34854ad05e6da0a2f658276490b316 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 24 Apr 2024 17:49:40 +0200 Subject: [PATCH 051/267] Document the two HUGETLB options for buffer allocation --- Makefile.rule | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Makefile.rule b/Makefile.rule index bb239be2a4..d25299775d 100644 --- a/Makefile.rule +++ b/Makefile.rule @@ -219,6 +219,16 @@ NO_AFFINITY = 1 # to the user space. If bigphysarea is enabled, it will use it. # DEVICEDRIVER_ALLOCATION = 1 +# Use large page allocation (called hugepage support in Linux context) +# for the thread buffers (with access by shared memory operations) +# HUGETLB_ALLOCATION = 1 + +# Use large page allocation called hugepages in Linux) based on mmap accessing +# a memory-backed pseudofile (requires hugetlbfs to be mounted in the system, +# the example below has it mounted on /hugepages. OpenBLAS will create the backing +# file as gotoblas.processid in that path) +# HUGETLBFILE_ALLOCATION = /hugepages + # If you need to synchronize FP CSR between threads (for x86/x86_64 and aarch64 only). # CONSISTENT_FPCSR = 1 From f4ee0a423b62c1a7f02baa748f8a491bd548fda9 Mon Sep 17 00:00:00 2001 From: Ayappan P Date: Thu, 25 Apr 2024 07:32:21 -0400 Subject: [PATCH 052/267] Fix openblas_utest_ext build in AIX --- utest/test_extensions/utest_main2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utest/test_extensions/utest_main2.c b/utest/test_extensions/utest_main2.c index 1a761cb75d..41269f0e17 100644 --- a/utest/test_extensions/utest_main2.c +++ b/utest/test_extensions/utest_main2.c @@ -40,7 +40,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define CTEST_ADD_TESTS_MANUALLY #include "cblas.h" -#include "openblas_utest.h" +#include "utest/openblas_utest.h" #if 1 CTEST(amax, samax){ blasint N=3, inc=1; From b1d722fc0cf563298de8c5ae66fc04a1901d4bf1 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Mon, 29 Apr 2024 15:37:26 +0200 Subject: [PATCH 053/267] Fix cast to work with INTERFACE64 (especially on big-endian) --- utest/test_extensions/xerbla.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utest/test_extensions/xerbla.c b/utest/test_extensions/xerbla.c index 9487b20a64..8c90d4ce75 100644 --- a/utest/test_extensions/xerbla.c +++ b/utest/test_extensions/xerbla.c @@ -39,7 +39,7 @@ static char *rout; static void F77_xerbla(char *srname, void *vinfo) { - int info=*(int*)vinfo; + blasint info=*(blasint*)vinfo; if (link_xerbla) { @@ -85,4 +85,4 @@ void set_xerbla(char* current_rout, int expected_info){ lerr = TRUE; _info = expected_info; rout = current_rout; -} \ No newline at end of file +} From edacf9b3973634d2540d1b7d076715d3f37d9c79 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 30 Apr 2024 08:50:47 +0200 Subject: [PATCH 054/267] Work around spurious BLAS3 test errors on LOONGSON3R3/4 (#4667) Force compilation with gfortran to use O0 on older Loongson hardware to avoid spurious test failures --- ctest/Makefile | 3 +++ test/Makefile | 3 +++ 2 files changed, 6 insertions(+) diff --git a/ctest/Makefile b/ctest/Makefile index 6c7cc1ed56..46885f3bb7 100644 --- a/ctest/Makefile +++ b/ctest/Makefile @@ -25,6 +25,9 @@ endif override CFLAGS += -DADD$(BU) -DCBLAS ifeq ($(F_COMPILER),GFORTRAN) +ifneq (, $(filter $(CORE),LOONGSON3R3 LOONGSON3R4)) + override FFLAGS = $(filter_out(-O2 -O3,$(FFLAGS)) -O0 +endif override FFLAGS += -fno-tree-vectorize endif override TARGET_ARCH= diff --git a/test/Makefile b/test/Makefile index 6a50b6c983..85bdca1030 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,6 +1,9 @@ TOPDIR = .. include ../Makefile.system ifeq ($(F_COMPILER),GFORTRAN) +ifneq (, $(filter $(CORE),LOONGSON3R3 LOONGSON3R4)) + override FFLAGS = $(filter_out(-O2 -O3,$(FFLAGS)) -O0 +endif override FFLAGS += -fno-tree-vectorize endif From 7c915e64cad5bc12e85a2d8db01a3a4cb3996040 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 30 Apr 2024 17:48:14 +0200 Subject: [PATCH 055/267] Silence a GCC14 warning/error in the f2c-converted LAPACK --- Makefile.system | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile.system b/Makefile.system index 840389db73..620a462e3f 100644 --- a/Makefile.system +++ b/Makefile.system @@ -356,6 +356,9 @@ OBJCONV = $(CROSS_SUFFIX)objconv ifeq ($(NOFORTRAN), 1) C_LAPACK = 1 override FEXTRALIB = +ifeq ($(C_COMPILER), GCC) +CCOMMON_OPT += -Wno-error=incompatible-pointer-types +endif endif ifeq ($(C_COMPILER), GCC) From 3d26837a35129c2de021bab79dc1dbe1c1df00fb Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 30 Apr 2024 19:05:18 +0200 Subject: [PATCH 056/267] Suppress GCC14 error exit in the f2c-converted LAPACK --- cmake/system.cmake | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmake/system.cmake b/cmake/system.cmake index 7166cc07f5..6fec04bfcf 100644 --- a/cmake/system.cmake +++ b/cmake/system.cmake @@ -160,6 +160,12 @@ else() endif () endif () +if (C_LAPACK) + if (${CMAKE_C_COMPILER_ID} STREQUAL "GNU") + set(CCOMMON_OPT "${CCOMMON_OPT} -Wno-error=incompatible-pointer-types") + endif () +endif () + include("${PROJECT_SOURCE_DIR}/cmake/prebuild.cmake") if (DEFINED TARGET) if (${TARGET} STREQUAL COOPERLAKE AND NOT NO_AVX512) From 9c4e10fbd155daa0ee8206a8313436f14f7f7282 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 4 May 2024 14:48:02 +0200 Subject: [PATCH 057/267] sort hugetlb and shm alloc options --- Makefile.system | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/Makefile.system b/Makefile.system index 840389db73..67830f2a3e 100644 --- a/Makefile.system +++ b/Makefile.system @@ -355,7 +355,10 @@ OBJCONV = $(CROSS_SUFFIX)objconv # When fortran support was either not detected or actively deselected, only build BLAS. ifeq ($(NOFORTRAN), 1) C_LAPACK = 1 -override FEXTRALIB = +override FEXTRALIB = +ifeq ($(C_COMPILER), GCC) +CCOMMON_OPT += -Wno-error=incompatible-pointer-types +endif endif ifeq ($(C_COMPILER), GCC) @@ -959,18 +962,12 @@ endif ifeq ($(ARCH), loongarch64) LA64_ABI=$(shell $(CC) -mabi=lp64d -c $(TOPDIR)/cpuid_loongarch64.c -o /dev/null > /dev/null 2> /dev/null && echo lp64d) -LA64_ARCH=$(shell $(CC) -march=loongarch64 -c $(TOPDIR)/cpuid_loongarch64.c -o /dev/null > /dev/null 2> /dev/null && echo loongarch64) ifneq ($(LA64_ABI), lp64d) LA64_ABI=lp64 endif -ifneq ($(LA64_ARCH), loongarch64) -CCOMMON_OPT += -mabi=$(LA64_ABI) -FCOMMON_OPT += -mabi=$(LA64_ABI) -else CCOMMON_OPT += -march=loongarch64 -mabi=$(LA64_ABI) FCOMMON_OPT += -march=loongarch64 -mabi=$(LA64_ABI) endif -endif endif @@ -1589,13 +1586,23 @@ ifdef FUNCTION_PROFILE CCOMMON_OPT += -DFUNCTION_PROFILE endif +ifdef SHMEM_ALLOCATION +ifneq ($(SHMEM_ALLOCATION), 0) +CCOMMON_OPT += -DALLOC_SHM +endif +endif + ifdef HUGETLB_ALLOCATION +ifneq ($(HUGETLB_ALLOCATION), 0) CCOMMON_OPT += -DALLOC_HUGETLB endif +endif ifdef HUGETLBFILE_ALLOCATION +ifneq ($(HUGETLBFILE_ALLOCATION), 0) CCOMMON_OPT += -DALLOC_HUGETLBFILE -DHUGETLB_FILE_NAME=$(HUGETLBFILE_ALLOCATION) endif +endif ifdef STATIC_ALLOCATION CCOMMON_OPT += -DALLOC_STATIC From dc99b613805fd15f1e71836b0f3e37c17770d697 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 4 May 2024 14:49:00 +0200 Subject: [PATCH 058/267] sort unwanted interdependencies of alloc_shm and alloc_hugetlb --- driver/others/memory.c | 58 ++++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/driver/others/memory.c b/driver/others/memory.c index c55688aced..94b2f1a6ad 100644 --- a/driver/others/memory.c +++ b/driver/others/memory.c @@ -1165,11 +1165,10 @@ void *blas_memory_alloc(int procpos){ #ifdef ALLOC_DEVICEDRIVER alloc_devicedirver, #endif -/* Hugetlb implicitly assumes ALLOC_SHM */ -#ifdef ALLOC_SHM +#ifdef ALLOC_SHM && !defined(ALLOC_HUGETLB) alloc_shm, #endif -#if ((defined ALLOC_SHM) && (defined OS_LINUX || defined OS_AIX || defined __sun__ || defined OS_WINDOWS)) +#if ((defined ALLOC_HUGETLB) && (defined OS_LINUX || defined OS_AIX || defined __sun__ || defined OS_WINDOWS)) alloc_hugetlb, #endif #ifdef ALLOC_MMAP @@ -1190,7 +1189,6 @@ void *blas_memory_alloc(int procpos){ struct alloc_t * alloc_info; struct alloc_t ** alloc_table; - #if defined(SMP) && !defined(USE_OPENMP) int mi; LOCK_COMMAND(&alloc_lock); @@ -1282,7 +1280,7 @@ UNLOCK_COMMAND(&alloc_lock); } #endif -#if (defined ALLOC_SHM) && (defined OS_LINUX || defined OS_AIX || defined __sun__ || defined OS_WINDOWS) +#if (defined ALLOC_HUGETLB) && (defined OS_LINUX || defined OS_AIX || defined __sun__ || defined OS_WINDOWS) if ((*func == alloc_hugetlb) && (map_address != (void *)-1)) hugetlb_allocated = 1; #endif @@ -2494,7 +2492,7 @@ static void *alloc_devicedirver(void *address){ #endif -#ifdef ALLOC_SHM +#if defined(ALLOC_SHM) && !defined(ALLOC_HUGETLB) static void alloc_shm_free(struct release_t *release){ @@ -2506,7 +2504,9 @@ static void alloc_shm_free(struct release_t *release){ static void *alloc_shm(void *address){ void *map_address; int shmid; - +#ifdef DEBUG + fprintf(stderr,"alloc_shm got called\n"); +#endif shmid = shmget(IPC_PRIVATE, BUFFER_SIZE,IPC_CREAT | 0600); map_address = (void *)shmat(shmid, address, 0); @@ -2533,6 +2533,7 @@ static void *alloc_shm(void *address){ return map_address; } +#endif #if defined OS_LINUX || defined OS_AIX || defined __sun__ || defined OS_WINDOWS @@ -2562,6 +2563,10 @@ static void *alloc_hugetlb(void *address){ void *map_address = (void *)-1; +#ifdef DEBUG +fprintf(stderr,"alloc_hugetlb got called\n"); +#endif + #if defined(OS_LINUX) || defined(OS_AIX) int shmid; @@ -2583,7 +2588,7 @@ static void *alloc_hugetlb(void *address){ if (map_address != (void *)-1){ shmctl(shmid, IPC_RMID, 0); - } + }else printf("alloc_hugetlb failed\n"); } #endif @@ -2645,7 +2650,6 @@ static void *alloc_hugetlb(void *address){ } #endif -#endif #ifdef ALLOC_HUGETLBFILE @@ -2762,11 +2766,10 @@ void *blas_memory_alloc(int procpos){ #ifdef ALLOC_DEVICEDRIVER alloc_devicedirver, #endif -/* Hugetlb implicitly assumes ALLOC_SHM */ -#ifdef ALLOC_SHM +#ifdef ALLOC_SHM && !defined(ALLOC_HUGETLB) alloc_shm, #endif -#if ((defined ALLOC_SHM) && (defined OS_LINUX || defined OS_AIX || defined __sun__ || defined OS_WINDOWS)) +#if ((defined ALLOC_HUGETLB) && (defined OS_LINUX || defined OS_AIX || defined __sun__ || defined OS_WINDOWS)) alloc_hugetlb, #endif #ifdef ALLOC_MMAP @@ -2945,8 +2948,22 @@ void *blas_memory_alloc(int procpos){ } #endif -#if (defined ALLOC_SHM) && (defined OS_LINUX || defined OS_AIX || defined __sun__ || defined OS_WINDOWS) +#if (defined ALLOC_HUGETLB) && (defined OS_LINUX || defined OS_AIX || defined __sun__ || defined OS_WINDOWS) if ((*func == alloc_hugetlb) && (map_address != (void *)-1)) hugetlb_allocated = 1; +#ifdef DEBUG + if (hugetlb_allocated) printf("allocating via shared memory with large page support (hugetlb)\n"); +#endif +#endif + +#if (defined ALLOC_SHM) && (defined OS_LINUX || defined OS_AIX || defined __sun__ || defined OS_WINDOWS) +#ifdef DEBUG + printf("allocating via shared memory\n"); +#endif + if ((*func == alloc_shm) && (map_address == (void *)-1)) { +#ifndef OS_WINDOWS + fprintf(stderr, "OpenBLAS Warning ... shared memory allocation was failed.\n"); +#endif + } #endif func ++; @@ -3061,10 +3078,23 @@ void *blas_memory_alloc(int procpos){ } #endif -#if (defined ALLOC_SHM) && (defined OS_LINUX || defined OS_AIX || defined __sun__ || defined OS_WINDOWS) +#if (defined ALLOC_HUGETLB) && (defined OS_LINUX || defined OS_AIX || defined __sun__ || defined OS_WINDOWS) +#ifdef DEBUG + fprintf(stderr,"OpenBLAS: allocating via shared memory with large page support (hugetlb)\n"); +#endif if ((*func == alloc_hugetlb) && (map_address != (void *)-1)) hugetlb_allocated = 1; #endif +#if (defined ALLOC_SHM) && (defined OS_LINUX || defined OS_AIX || defined __sun__ || defined OS_WINDOWS) +#ifdef DEBUG + fprintf(stderr,"allocating via shared memory\n"); +#endif + if ((*func == alloc_shm) && (map_address == (void *)-1)) { +#ifndef OS_WINDOWS + fprintf(stderr, "OpenBLAS Warning ... shared memory allocation was failed.\n"); +#endif + } +#endif func ++; } From a82ecadc11c10cbf82b2f9647bf76f9bb764185f Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 4 May 2024 15:48:48 +0200 Subject: [PATCH 059/267] mention LD_LIBRARY_PATH --- docs/user_manual.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user_manual.md b/docs/user_manual.md index 0855f81194..4d5fa9eaa1 100644 --- a/docs/user_manual.md +++ b/docs/user_manual.md @@ -49,7 +49,7 @@ For more information, please read [Installation Guide](install.md). gcc -o test test.c -I/your_path/OpenBLAS/include/ -L/your_path/OpenBLAS/lib -Wl,-rpath,/your_path/OpenBLAS/lib -lopenblas ``` -The `-Wl,-rpath,/your_path/OpenBLAS/lib` option to linker can be omitted if you ran `ldconfig` to update linker cache, put `/your_path/OpenBLAS/lib` in `/etc/ld.so.conf` or a file in `/etc/ld.so.conf.d`, or installed OpenBLAS in a location part of `ld.so` default search path. Otherwise, linking at runtime will fail. +The `-Wl,-rpath,/your_path/OpenBLAS/lib` option to linker can be omitted if you ran `ldconfig` to update linker cache, put `/your_path/OpenBLAS/lib` in `/etc/ld.so.conf` or a file in `/etc/ld.so.conf.d`, or installed OpenBLAS in a location that is part of the `ld.so` default search path (usually /lib,/usr/lib and /usr/local/lib). Alternatively, you can set the environment variable LD_LIBRARY_PATH to point to the folder that contains libopenblas.so. Otherwise, linking at runtime will fail with a message like `cannot open shared object file: no such file or directory` If the library is multithreaded, please add `-lpthread`. If the library contains LAPACK functions, please add `-lgfortran` or other Fortran libs, although if you only make calls to LAPACKE routines, i.e. your code has `#include "lapacke.h"` and makes calls to methods like `LAPACKE_dgeqrf`, `-lgfortran` is not needed. From bbd227ce4a657a137bc08d93ec23e49f31dff095 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Mon, 6 May 2024 00:11:44 +0200 Subject: [PATCH 060/267] Add Intel Meteor Lake and Emerald Rapids --- cpuid_x86.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cpuid_x86.c b/cpuid_x86.c index 6cf4d6503f..40735c5634 100644 --- a/cpuid_x86.c +++ b/cpuid_x86.c @@ -1535,6 +1535,7 @@ int get_cpuname(void){ return CPUTYPE_SANDYBRIDGE; else return CPUTYPE_NEHALEM; + case 0: // Meteor Lake case 7: // Rocket Lake if(support_avx512()) return CPUTYPE_SKYLAKEX; @@ -1560,6 +1561,19 @@ int get_cpuname(void){ return CPUTYPE_NEHALEM; } break; + case 12: //family 6 exmodel 12 + switch (model) { + case 15: + if(support_avx512()) + return CPUTYPE_SAPPHIRERAPIDS; + if(support_avx2()) + return CPUTYPE_HASWELL; + if(support_avx()) + return CPUTYPE_SANDYBRIDGE; + else + return CPUTYPE_NEHALEM; + } + break; } break; case 0x7: From 4376b6f7d22d23bdf816c39e40bd4e6912018602 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 7 May 2024 14:42:01 +0200 Subject: [PATCH 061/267] Restore Loongson LA64ARCH handling --- Makefile.system | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile.system b/Makefile.system index 8a7099808d..f452011ad2 100644 --- a/Makefile.system +++ b/Makefile.system @@ -962,12 +962,18 @@ endif ifeq ($(ARCH), loongarch64) LA64_ABI=$(shell $(CC) -mabi=lp64d -c $(TOPDIR)/cpuid_loongarch64.c -o /dev/null > /dev/null 2> /dev/null && echo lp64d) +LA64_ARCH=$(shell $(CC) -march=loongarch64 -c $(TOPDIR)/cpuid_loongarch64.c -o /dev/null > /dev/null 2> /dev/null && echo loongarch64) ifneq ($(LA64_ABI), lp64d) LA64_ABI=lp64 endif +ifneq ($(LA64_ARCH), loongarch64) +CCOMMON_OPT += -mabi=$(LA64_ABI) +FCOMMON_OPT += -mabi=$(LA64_ABI) +else CCOMMON_OPT += -march=loongarch64 -mabi=$(LA64_ABI) FCOMMON_OPT += -march=loongarch64 -mabi=$(LA64_ABI) endif +endif endif From edfe1aa4718684dac76c3d4dc576b50f80382109 Mon Sep 17 00:00:00 2001 From: Andrew Robbins Date: Tue, 7 May 2024 11:06:27 -0400 Subject: [PATCH 062/267] Expose whether locking is enabled in get_config --- driver/others/openblas_get_config.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/driver/others/openblas_get_config.c b/driver/others/openblas_get_config.c index 867d0e3614..ff52cfba8f 100644 --- a/driver/others/openblas_get_config.c +++ b/driver/others/openblas_get_config.c @@ -13,9 +13,9 @@ modification, are permitted provided that the following conditions are notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - 3. Neither the name of the OpenBLAS project nor the names of - its contributors may be used to endorse or promote products - derived from this software without specific prior written + 3. Neither the name of the OpenBLAS project nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" @@ -63,6 +63,9 @@ static char* openblas_config_str="" #ifdef USE_TLS "USE_TLS " #endif +#ifdef USE_LOCKING + "USE_LOCKING " +#endif #ifndef DYNAMIC_ARCH CHAR_CORENAME #endif @@ -83,7 +86,7 @@ char tmpstr[20]; #endif if (openblas_get_parallel() == 0) sprintf(tmpstr, " SINGLE_THREADED"); - else + else snprintf(tmpstr,19," MAX_THREADS=%d",MAX_CPU_NUMBER); strcat(tmp_config_str, tmpstr); return tmp_config_str; @@ -91,7 +94,7 @@ char tmpstr[20]; char* openblas_get_corename(void) { -#ifndef DYNAMIC_ARCH +#ifndef DYNAMIC_ARCH return CHAR_CORENAME; #else return gotoblas_corename(); From f0f1ff7820d81f6faac1ed7d7f77f8533a084ff7 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 8 May 2024 00:40:36 +0200 Subject: [PATCH 063/267] fix HUGETLB allocation for TLS mode as well --- driver/others/memory.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/driver/others/memory.c b/driver/others/memory.c index 94b2f1a6ad..784a1f0282 100644 --- a/driver/others/memory.c +++ b/driver/others/memory.c @@ -964,7 +964,9 @@ static void *alloc_shm(void *address){ return map_address; } -#if defined OS_LINUX || defined OS_AIX || defined __sun__ || defined OS_WINDOWS +#endif + +#if ((defined ALLOC_HUGETLB) && (defined OS_LINUX || defined OS_AIX || defined __sun__ || defined OS_WINDOWS)) static void alloc_hugetlb_free(struct alloc_t *alloc_info){ @@ -1066,7 +1068,8 @@ static void *alloc_hugetlb(void *address){ } #endif -#endif + + #ifdef ALLOC_HUGETLBFILE From 6017ad714642217a03ead099eb899c656995e24d Mon Sep 17 00:00:00 2001 From: gxw Date: Sun, 28 Apr 2024 16:36:17 +0800 Subject: [PATCH 064/267] loongarch64: Update dgemm_kernel_16x4 to dgemm_kernel_16x6 --- kernel/generic/laswp_ncopy_6.c | 276 + kernel/generic/symm_lcopy_6.c | 167 +- kernel/generic/symm_ucopy_6.c | 166 +- kernel/generic/trmm_lncopy_6.c | 904 +-- kernel/generic/trmm_ltcopy_6.c | 905 +-- kernel/generic/trmm_uncopy_6.c | 1231 ++-- kernel/generic/trmm_utcopy_6.c | 888 +-- kernel/generic/trsm_lncopy_6.c | 549 +- kernel/generic/trsm_ltcopy_6.c | 436 +- kernel/generic/trsm_uncopy_6.c | 820 ++- kernel/generic/trsm_utcopy_6.c | 420 +- kernel/loongarch64/KERNEL.LOONGSON3R5 | 14 +- kernel/loongarch64/dgemm_kernel_16x6.S | 6256 ++++++++++++++++++ kernel/loongarch64/dgemm_tcopy_6.S | 555 ++ kernel/loongarch64/gemm_ncopy_6.prefx.c | 299 + kernel/loongarch64/trsm_kernel_LN_UNROLLN6.c | 342 + kernel/loongarch64/trsm_kernel_LT_UNROLLN6.c | 327 + kernel/loongarch64/trsm_kernel_RN_UNROLLN6.c | 325 + kernel/loongarch64/trsm_kernel_RT_UNROLLN6.c | 351 + param.h | 3 +- 20 files changed, 12731 insertions(+), 2503 deletions(-) create mode 100644 kernel/generic/laswp_ncopy_6.c create mode 100644 kernel/loongarch64/dgemm_kernel_16x6.S create mode 100644 kernel/loongarch64/dgemm_tcopy_6.S create mode 100644 kernel/loongarch64/gemm_ncopy_6.prefx.c create mode 100644 kernel/loongarch64/trsm_kernel_LN_UNROLLN6.c create mode 100644 kernel/loongarch64/trsm_kernel_LT_UNROLLN6.c create mode 100644 kernel/loongarch64/trsm_kernel_RN_UNROLLN6.c create mode 100644 kernel/loongarch64/trsm_kernel_RT_UNROLLN6.c diff --git a/kernel/generic/laswp_ncopy_6.c b/kernel/generic/laswp_ncopy_6.c new file mode 100644 index 0000000000..85a17a092f --- /dev/null +++ b/kernel/generic/laswp_ncopy_6.c @@ -0,0 +1,276 @@ + +/*********************************************************************/ +/* Copyright 2009, 2010, 2024 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#include +#include "common.h" + +#define PREFETCHSIZE 4 + +int CNAME(BLASLONG n, BLASLONG k1, BLASLONG k2, FLOAT *a, BLASLONG lda, blasint *ipiv, FLOAT *buffer){ + + BLASLONG i, j, ip; + blasint *piv; + FLOAT *dx1, *dy1; + FLOAT *dx2, *dy2; + FLOAT *dx3, *dy3; + FLOAT *dx4, *dy4; + FLOAT *dx5, *dy5; + FLOAT *dx6, *dy6; + FLOAT atemp1, btemp1; + FLOAT atemp2, btemp2; + FLOAT atemp3, btemp3; + FLOAT atemp4, btemp4; + FLOAT atemp5, btemp5; + FLOAT atemp6, btemp6; + + a--; + ipiv += k1 - 1; + + if (n <= 0) return 0; + if (k1 > k2) return 0; + + j = (n / 6); + if (j > 0) { + do { + piv = ipiv; + i = k1; + + do { + ip = *piv; + piv ++; + + dx1 = a + i; + dy1 = a + ip; + dx2 = a + i + lda * 1; + dy2 = a + ip + lda * 1; + dx3 = a + i + lda * 2; + dy3 = a + ip + lda * 2; + dx4 = a + i + lda * 3; + dy4 = a + ip + lda * 3; + dx5 = a + i + lda * 4; + dy5 = a + ip + lda * 4; + dx6 = a + i + lda * 5; + dy6 = a + ip + lda * 5; + +#ifdef __GNUC__ + __builtin_prefetch(dx1 + PREFETCHSIZE, 0, 1); + __builtin_prefetch(dx2 + PREFETCHSIZE, 0, 1); + __builtin_prefetch(dx3 + PREFETCHSIZE, 0, 1); + __builtin_prefetch(dx4 + PREFETCHSIZE, 0, 1); + __builtin_prefetch(dx5 + PREFETCHSIZE, 0, 1); + __builtin_prefetch(dx6 + PREFETCHSIZE, 0, 1); +#endif + + atemp1 = *dx1; + btemp1 = *dy1; + atemp2 = *dx2; + btemp2 = *dy2; + atemp3 = *dx3; + btemp3 = *dy3; + atemp4 = *dx4; + btemp4 = *dy4; + + atemp5 = *dx5; + btemp5 = *dy5; + atemp6 = *dx6; + btemp6 = *dy6; + + if (ip != i) { + *dy1 = atemp1; + *dy2 = atemp2; + *dy3 = atemp3; + *dy4 = atemp4; + *dy5 = atemp5; + *dy6 = atemp6; + *(buffer + 0) = btemp1; + *(buffer + 1) = btemp2; + *(buffer + 2) = btemp3; + *(buffer + 3) = btemp4; + *(buffer + 4) = btemp5; + *(buffer + 5) = btemp6; + } else { + *(buffer + 0) = atemp1; + *(buffer + 1) = atemp2; + *(buffer + 2) = atemp3; + *(buffer + 3) = atemp4; + *(buffer + 4) = atemp5; + *(buffer + 5) = atemp6; + } + + buffer += 6; + + i++; + } while (i <= k2); + + a += 6 * lda; + j --; + } while (j > 0); + } + + if ((n % 6) & 4) { + piv = ipiv; + + ip = *piv; + piv ++; + + dx1 = a + k1; + dy1 = a + ip; + dx2 = a + k1 + lda * 1; + dy2 = a + ip + lda * 1; + dx3 = a + k1 + lda * 2; + dy3 = a + ip + lda * 2; + dx4 = a + k1 + lda * 3; + dy4 = a + ip + lda * 3; + + i = k1; + + do { + atemp1 = *dx1; + atemp2 = *dx2; + atemp3 = *dx3; + atemp4 = *dx4; + + btemp1 = *dy1; + btemp2 = *dy2; + btemp3 = *dy3; + btemp4 = *dy4; + + if (ip != i) { + *dy1 = atemp1; + *dy2 = atemp2; + *dy3 = atemp3; + *dy4 = atemp4; + *(buffer + 0) = btemp1; + *(buffer + 1) = btemp2; + *(buffer + 2) = btemp3; + *(buffer + 3) = btemp4; + } else { + *(buffer + 0) = atemp1; + *(buffer + 1) = atemp2; + *(buffer + 2) = atemp3; + *(buffer + 3) = atemp4; + } + + ip = *piv; + piv ++; + + i++; + dx1 = a + i; + dy1 = a + ip; + dx2 = a + i + lda * 1; + dy2 = a + ip + lda * 1; + dx3 = a + i + lda * 2; + dy3 = a + ip + lda * 2; + dx4 = a + i + lda * 3; + dy4 = a + ip + lda * 3; + + buffer += 4; + + } while (i <= k2); + + a += 4 * lda; + } + + if ((n % 6) & 2) { + piv = ipiv; + + i = k1; + do { + ip = *piv; + piv ++; + + dx1 = a + i; + dy1 = a + ip; + dx2 = a + i + lda; + dy2 = a + ip + lda; + + atemp1 = *dx1; + btemp1 = *dy1; + atemp2 = *dx2; + btemp2 = *dy2; + + if (ip != i) { + *dy1 = atemp1; + *dy2 = atemp2; + *(buffer + 0) = btemp1; + *(buffer + 1) = btemp2; + } else { + *(buffer + 0) = atemp1; + *(buffer + 1) = atemp2; + } + + buffer += 2; + + i++; + } while (i <= k2); + + a += 2 * lda; + } + + + if ((n % 6) & 1) { + piv = ipiv; + + i = k1; + do { + ip = *piv; + piv ++; + + dx1 = a + i; + dy1 = a + ip; + atemp1 = *dx1; + btemp1 = *dy1; + + if (ip != i) { + *dy1 = atemp1; + *buffer = btemp1; + } else { + *buffer = atemp1; + } + + buffer ++; + + i++; + } while (i <= k2); + + // a += lda; + } + + return 0; +} \ No newline at end of file diff --git a/kernel/generic/symm_lcopy_6.c b/kernel/generic/symm_lcopy_6.c index ca730e1eef..3a3e2d5b2f 100644 --- a/kernel/generic/symm_lcopy_6.c +++ b/kernel/generic/symm_lcopy_6.c @@ -41,98 +41,141 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, FLOAT *b){ - BLASLONG i, js, offset; - - FLOAT data01, data02, data03, data04; - FLOAT *ao1, *ao2, *ao3, *ao4; + BLASLONG i, js, offset; + + FLOAT data01, data02, data03, data04, data05, data06; + FLOAT *ao1, *ao2, *ao3, *ao4, *ao5, *ao6; + + js = (n / 6); + while (js > 0){ + + offset = posX - posY; + + if (offset > 0) ao1 = a + posX + 0 + posY * lda; else ao1 = a + posY + (posX + 0) * lda; + if (offset > -1) ao2 = a + posX + 1 + posY * lda; else ao2 = a + posY + (posX + 1) * lda; + if (offset > -2) ao3 = a + posX + 2 + posY * lda; else ao3 = a + posY + (posX + 2) * lda; + if (offset > -3) ao4 = a + posX + 3 + posY * lda; else ao4 = a + posY + (posX + 3) * lda; + if (offset > -4) ao5 = a + posX + 4 + posY * lda; else ao5 = a + posY + (posX + 4) * lda; + if (offset > -5) ao6 = a + posX + 5 + posY * lda; else ao6 = a + posY + (posX + 5) * lda; + + i = m; + + while (i > 0) { + data01 = *(ao1 + 0); + data02 = *(ao2 + 0); + data03 = *(ao3 + 0); + data04 = *(ao4 + 0); + data05 = *(ao5 + 0); + data06 = *(ao6 + 0); + + if (offset > 0) ao1 += lda; else ao1 ++; + if (offset > -1) ao2 += lda; else ao2 ++; + if (offset > -2) ao3 += lda; else ao3 ++; + if (offset > -3) ao4 += lda; else ao4 ++; + if (offset > -4) ao5 += lda; else ao5 ++; + if (offset > -5) ao6 += lda; else ao6 ++; + + b[ 0] = data01; + b[ 1] = data02; + b[ 2] = data03; + b[ 3] = data04; + b[ 4] = data05; + b[ 5] = data06; + + b += 6; + + offset --; + i --; + } + + posX += 6; + js --; + } - js = (n >> 2); - while (js > 0){ + if ((n - n/6) & 4) { + offset = posX - posY; - offset = posX - posY; + if (offset > 0) ao1 = a + posX + 0 + posY * lda; else ao1 = a + posY + (posX + 0) * lda; + if (offset > -1) ao2 = a + posX + 1 + posY * lda; else ao2 = a + posY + (posX + 1) * lda; + if (offset > -2) ao3 = a + posX + 2 + posY * lda; else ao3 = a + posY + (posX + 2) * lda; + if (offset > -3) ao4 = a + posX + 3 + posY * lda; else ao4 = a + posY + (posX + 3) * lda; - if (offset > 0) ao1 = a + posX + 0 + posY * lda; else ao1 = a + posY + (posX + 0) * lda; - if (offset > -1) ao2 = a + posX + 1 + posY * lda; else ao2 = a + posY + (posX + 1) * lda; - if (offset > -2) ao3 = a + posX + 2 + posY * lda; else ao3 = a + posY + (posX + 2) * lda; - if (offset > -3) ao4 = a + posX + 3 + posY * lda; else ao4 = a + posY + (posX + 3) * lda; + i = m; - i = m; + while (i > 0) { + data01 = *(ao1 + 0); + data02 = *(ao2 + 0); + data03 = *(ao3 + 0); + data04 = *(ao4 + 0); - while (i > 0) { - data01 = *(ao1 + 0); - data02 = *(ao2 + 0); - data03 = *(ao3 + 0); - data04 = *(ao4 + 0); + if (offset > 0) ao1 += lda; else ao1 ++; + if (offset > -1) ao2 += lda; else ao2 ++; + if (offset > -2) ao3 += lda; else ao3 ++; + if (offset > -3) ao4 += lda; else ao4 ++; - if (offset > 0) ao1 += lda; else ao1 ++; - if (offset > -1) ao2 += lda; else ao2 ++; - if (offset > -2) ao3 += lda; else ao3 ++; - if (offset > -3) ao4 += lda; else ao4 ++; + b[ 0] = data01; + b[ 1] = data02; + b[ 2] = data03; + b[ 3] = data04; - b[ 0] = data01; - b[ 1] = data02; - b[ 2] = data03; - b[ 3] = data04; + b += 4; - b += 4; + offset --; + i --; + } - offset --; - i --; + posX += 4; } - posX += 4; - js --; - } + if ((n - n/6) & 2) { - if (n & 2) { + offset = posX - posY; - offset = posX - posY; + if (offset > 0) ao1 = a + posX + 0 + posY * lda; else ao1 = a + posY + (posX + 0) * lda; + if (offset > -1) ao2 = a + posX + 1 + posY * lda; else ao2 = a + posY + (posX + 1) * lda; - if (offset > 0) ao1 = a + posX + 0 + posY * lda; else ao1 = a + posY + (posX + 0) * lda; - if (offset > -1) ao2 = a + posX + 1 + posY * lda; else ao2 = a + posY + (posX + 1) * lda; + i = m; - i = m; + while (i > 0) { + data01 = *(ao1 + 0); + data02 = *(ao2 + 0); - while (i > 0) { - data01 = *(ao1 + 0); - data02 = *(ao2 + 0); + if (offset > 0) ao1 += lda; else ao1 ++; + if (offset > -1) ao2 += lda; else ao2 ++; - if (offset > 0) ao1 += lda; else ao1 ++; - if (offset > -1) ao2 += lda; else ao2 ++; + b[ 0] = data01; + b[ 1] = data02; - b[ 0] = data01; - b[ 1] = data02; + b += 2; - b += 2; + offset --; + i --; + } - offset --; - i --; + posX += 2; } - posX += 2; - } - - if (n & 1) { + if ((n - n/6) & 1) { - offset = posX - posY; + offset = posX - posY; - if (offset > 0) ao1 = a + posX + 0 + posY * lda; else ao1 = a + posY + (posX + 0) * lda; + if (offset > 0) ao1 = a + posX + 0 + posY * lda; else ao1 = a + posY + (posX + 0) * lda; - i = m; + i = m; - while (i > 0) { - data01 = *(ao1 + 0); + while (i > 0) { + data01 = *(ao1 + 0); - if (offset > 0) ao1 += lda; else ao1 ++; + if (offset > 0) ao1 += lda; else ao1 ++; - b[ 0] = data01; + b[ 0] = data01; - b ++; + b ++; - offset --; - i --; + offset --; + i --; + } } - } - return 0; + return 0; } diff --git a/kernel/generic/symm_ucopy_6.c b/kernel/generic/symm_ucopy_6.c index 6dbb861e98..a83d937d0b 100644 --- a/kernel/generic/symm_ucopy_6.c +++ b/kernel/generic/symm_ucopy_6.c @@ -41,96 +41,140 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, FLOAT *b){ - BLASLONG i, js, offset; + BLASLONG i, js, offset; + + FLOAT data01, data02, data03, data04, data05, data06; + FLOAT *ao1, *ao2, *ao3, *ao4, *ao5, *ao6; + + js = (n / 6); + while (js > 0){ + + offset = posX - posY; + + if (offset > 0) ao1 = a + posY + (posX + 0) * lda; else ao1 = a + posX + 0 + posY * lda; + if (offset > -1) ao2 = a + posY + (posX + 1) * lda; else ao2 = a + posX + 1 + posY * lda; + if (offset > -2) ao3 = a + posY + (posX + 2) * lda; else ao3 = a + posX + 2 + posY * lda; + if (offset > -3) ao4 = a + posY + (posX + 3) * lda; else ao4 = a + posX + 3 + posY * lda; + if (offset > -4) ao5 = a + posY + (posX + 4) * lda; else ao5 = a + posX + 4 + posY * lda; + if (offset > -5) ao6 = a + posY + (posX + 5) * lda; else ao6 = a + posX + 5 + posY * lda; + + i = m; + + while (i > 0) { + data01 = *(ao1 + 0); + data02 = *(ao2 + 0); + data03 = *(ao3 + 0); + data04 = *(ao4 + 0); + data05 = *(ao5 + 0); + data06 = *(ao6 + 0); + + if (offset > 0) ao1 ++; else ao1 += lda; + if (offset > -1) ao2 ++; else ao2 += lda; + if (offset > -2) ao3 ++; else ao3 += lda; + if (offset > -3) ao4 ++; else ao4 += lda; + if (offset > -4) ao5 ++; else ao5 += lda; + if (offset > -5) ao6 ++; else ao6 += lda; + + b[ 0] = data01; + b[ 1] = data02; + b[ 2] = data03; + b[ 3] = data04; + b[ 4] = data05; + b[ 5] = data06; + + b += 6; + + offset --; + i --; + } + + posX += 6; + js --; + } - FLOAT data01, data02, data03, data04; - FLOAT *ao1, *ao2, *ao3, *ao4; + if ((n - n/6) & 4) { - js = (n >> 2); - while (js > 0){ + offset = posX - posY; - offset = posX - posY; + if (offset > 0) ao1 = a + posY + (posX + 0) * lda; else ao1 = a + posX + 0 + posY * lda; + if (offset > -1) ao2 = a + posY + (posX + 1) * lda; else ao2 = a + posX + 1 + posY * lda; + if (offset > -2) ao3 = a + posY + (posX + 2) * lda; else ao3 = a + posX + 2 + posY * lda; + if (offset > -3) ao4 = a + posY + (posX + 3) * lda; else ao4 = a + posX + 3 + posY * lda; - if (offset > 0) ao1 = a + posY + (posX + 0) * lda; else ao1 = a + posX + 0 + posY * lda; - if (offset > -1) ao2 = a + posY + (posX + 1) * lda; else ao2 = a + posX + 1 + posY * lda; - if (offset > -2) ao3 = a + posY + (posX + 2) * lda; else ao3 = a + posX + 2 + posY * lda; - if (offset > -3) ao4 = a + posY + (posX + 3) * lda; else ao4 = a + posX + 3 + posY * lda; + i = m; - i = m; + while (i > 0) { + data01 = *(ao1 + 0); + data02 = *(ao2 + 0); + data03 = *(ao3 + 0); + data04 = *(ao4 + 0); - while (i > 0) { - data01 = *(ao1 + 0); - data02 = *(ao2 + 0); - data03 = *(ao3 + 0); - data04 = *(ao4 + 0); + if (offset > 0) ao1 ++; else ao1 += lda; + if (offset > -1) ao2 ++; else ao2 += lda; + if (offset > -2) ao3 ++; else ao3 += lda; + if (offset > -3) ao4 ++; else ao4 += lda; - if (offset > 0) ao1 ++; else ao1 += lda; - if (offset > -1) ao2 ++; else ao2 += lda; - if (offset > -2) ao3 ++; else ao3 += lda; - if (offset > -3) ao4 ++; else ao4 += lda; + b[ 0] = data01; + b[ 1] = data02; + b[ 2] = data03; + b[ 3] = data04; - b[ 0] = data01; - b[ 1] = data02; - b[ 2] = data03; - b[ 3] = data04; + b += 4; - b += 4; + offset --; + i --; + } - offset --; - i --; + posX += 4; } - posX += 4; - js --; - } + if ((n - n/6) & 2) { + offset = posX - posY; - if (n & 2) { - offset = posX - posY; + if (offset > 0) ao1 = a + posY + (posX + 0) * lda; else ao1 = a + posX + 0 + posY * lda; + if (offset > -1) ao2 = a + posY + (posX + 1) * lda; else ao2 = a + posX + 1 + posY * lda; - if (offset > 0) ao1 = a + posY + (posX + 0) * lda; else ao1 = a + posX + 0 + posY * lda; - if (offset > -1) ao2 = a + posY + (posX + 1) * lda; else ao2 = a + posX + 1 + posY * lda; + i = m; - i = m; + while (i > 0) { + data01 = *(ao1 + 0); + data02 = *(ao2 + 0); - while (i > 0) { - data01 = *(ao1 + 0); - data02 = *(ao2 + 0); + if (offset > 0) ao1 ++; else ao1 += lda; + if (offset > -1) ao2 ++; else ao2 += lda; - if (offset > 0) ao1 ++; else ao1 += lda; - if (offset > -1) ao2 ++; else ao2 += lda; + b[ 0] = data01; + b[ 1] = data02; - b[ 0] = data01; - b[ 1] = data02; + b += 2; - b += 2; + offset --; + i --; + } - offset --; - i --; + posX += 2; } - posX += 2; - } - - if (n & 1) { - offset = posX - posY; + if ((n - n/6) & 1) { + offset = posX - posY; - if (offset > 0) ao1 = a + posY + (posX + 0) * lda; else ao1 = a + posX + 0 + posY * lda; + if (offset > 0) ao1 = a + posY + (posX + 0) * lda; else ao1 = a + posX + 0 + posY * lda; - i = m; + i = m; - while (i > 0) { - data01 = *(ao1 + 0); + while (i > 0) { + data01 = *(ao1 + 0); - if (offset > 0) ao1 ++; else ao1 += lda; + if (offset > 0) ao1 ++; else ao1 += lda; - b[ 0] = data01; + b[ 0] = data01; - b ++; + b ++; - offset --; - i --; + offset --; + i --; + } } - } - return 0; + return 0; } diff --git a/kernel/generic/trmm_lncopy_6.c b/kernel/generic/trmm_lncopy_6.c index 0dcfb965ac..999f0d367d 100644 --- a/kernel/generic/trmm_lncopy_6.c +++ b/kernel/generic/trmm_lncopy_6.c @@ -41,444 +41,510 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, FLOAT *b){ - BLASLONG i, js; - BLASLONG X; - - FLOAT data01, data02, data03, data04, data05, data06, data07, data08; - FLOAT data09, data10, data11, data12, data13, data14, data15, data16; - FLOAT *ao1, *ao2, *ao3, *ao4; - - js = (n >> 2); - - if (js > 0){ - do { - X = posX; - - if (posX <= posY) { - ao1 = a + posY + (posX + 0) * lda; - ao2 = a + posY + (posX + 1) * lda; - ao3 = a + posY + (posX + 2) * lda; - ao4 = a + posY + (posX + 3) * lda; - } else { - ao1 = a + posX + (posY + 0) * lda; - ao2 = a + posX + (posY + 1) * lda; - ao3 = a + posX + (posY + 2) * lda; - ao4 = a + posX + (posY + 3) * lda; - } - - i = (m >> 2); - if (i > 0) { - do { - if (X > posY) { - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); - data03 = *(ao1 + 2); - data04 = *(ao1 + 3); - - data05 = *(ao2 + 0); - data06 = *(ao2 + 1); - data07 = *(ao2 + 2); - data08 = *(ao2 + 3); - - data09 = *(ao3 + 0); - data10 = *(ao3 + 1); - data11 = *(ao3 + 2); - data12 = *(ao3 + 3); - - data13 = *(ao4 + 0); - data14 = *(ao4 + 1); - data15 = *(ao4 + 2); - data16 = *(ao4 + 3); - - b[ 0] = data01; - b[ 1] = data05; - b[ 2] = data09; - b[ 3] = data13; - b[ 4] = data02; - b[ 5] = data06; - b[ 6] = data10; - b[ 7] = data14; - - b[ 8] = data03; - b[ 9] = data07; - b[10] = data11; - b[11] = data15; - b[12] = data04; - b[13] = data08; - b[14] = data12; - b[15] = data16; - - ao1 += 4; - ao2 += 4; - ao3 += 4; - ao4 += 4; - b += 16; - - } else - if (X < posY) { - ao1 += 4 * lda; - ao2 += 4 * lda; - ao3 += 4 * lda; - ao4 += 4 * lda; - b += 16; - - } else { + BLASLONG i, js, ii; + BLASLONG X; + + FLOAT data01, data02, data05, data06; + FLOAT *ao1, *ao2, *ao3, *ao4, *ao5, *ao6; + + js = (n / 6); + + if (js > 0){ + do { + X = posX; + + if (posX <= posY) { + ao1 = a + posY + (posX + 0) * lda; + ao2 = a + posY + (posX + 1) * lda; + ao3 = a + posY + (posX + 2) * lda; + ao4 = a + posY + (posX + 3) * lda; + ao5 = a + posY + (posX + 4) * lda; + ao6 = a + posY + (posX + 5) * lda; + } else { + ao1 = a + posX + (posY + 0) * lda; + ao2 = a + posX + (posY + 1) * lda; + ao3 = a + posX + (posY + 2) * lda; + ao4 = a + posX + (posY + 3) * lda; + ao5 = a + posX + (posY + 4) * lda; + ao6 = a + posX + (posY + 5) * lda; + } + + i = (m / 6); + if (i > 0) { + do { + if (X > posY) { + for (ii = 0; ii < 6; ii++){ + + b[ 0] = *(ao1 + 0); + b[ 1] = *(ao2 + 0); + b[ 2] = *(ao3 + 0); + b[ 3] = *(ao4 + 0); + b[ 4] = *(ao5 + 0); + b[ 5] = *(ao6 + 0); + + ao1 ++; + ao2 ++; + ao3 ++; + ao4 ++; + ao5 ++; + ao6 ++; + b += 6; + } + + } else if (X < posY) { + ao1 += 6 * lda; + ao2 += 6 * lda; + ao3 += 6 * lda; + ao4 += 6 * lda; + ao5 += 6 * lda; + ao6 += 6 * lda; + b += 36; + + } else { #ifdef UNIT - data02 = *(ao1 + 1); - data03 = *(ao1 + 2); - data04 = *(ao1 + 3); - - data07 = *(ao2 + 2); - data08 = *(ao2 + 3); - - data12 = *(ao3 + 3); - - b[ 0] = ONE; - b[ 1] = ZERO; - b[ 2] = ZERO; - b[ 3] = ZERO; - b[ 4] = data02; - b[ 5] = ONE; - b[ 6] = ZERO; - b[ 7] = ZERO; - - b[ 8] = data03; - b[ 9] = data07; - b[10] = ONE; - b[11] = ZERO; - b[12] = data04; - b[13] = data08; - b[14] = data12; - b[15] = ONE; + b[ 0] = ONE; #else - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); - data03 = *(ao1 + 2); - data04 = *(ao1 + 3); - - data06 = *(ao2 + 1); - data07 = *(ao2 + 2); - data08 = *(ao2 + 3); - - data11 = *(ao3 + 2); - data12 = *(ao3 + 3); - - data16 = *(ao4 + 3); - - b[ 0] = data01; - b[ 1] = ZERO; - b[ 2] = ZERO; - b[ 3] = ZERO; - b[ 4] = data02; - b[ 5] = data06; - b[ 6] = ZERO; - b[ 7] = ZERO; - - b[ 8] = data03; - b[ 9] = data07; - b[10] = data11; - b[11] = ZERO; - b[12] = data04; - b[13] = data08; - b[14] = data12; - b[15] = data16; + b[ 0] = *(ao1 + 0); #endif - ao1 += 4; - ao2 += 4; - ao3 += 4; - ao4 += 4; - b += 16; - } - - X += 4; - i --; - } while (i > 0); - } - - i = (m & 3); - if (i) { - - if (X > posY) { - - if (m & 2) { - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); - data03 = *(ao2 + 0); - data04 = *(ao2 + 1); - data05 = *(ao3 + 0); - data06 = *(ao3 + 1); - data07 = *(ao4 + 0); - data08 = *(ao4 + 1); - - b[ 0] = data01; - b[ 1] = data03; - b[ 2] = data05; - b[ 3] = data07; - b[ 4] = data02; - b[ 5] = data04; - b[ 6] = data06; - b[ 7] = data08; - - ao1 += 2; - ao2 += 2; - ao3 += 2; - ao4 += 2; - b += 8; - } - - if (m & 1) { - data01 = *(ao1 + 0); - data02 = *(ao2 + 0); - data03 = *(ao3 + 0); - data04 = *(ao4 + 0); - - b[ 0] = data01; - b[ 1] = data02; - b[ 2] = data03; - b[ 3] = data04; - - ao1 += 1; - ao2 += 1; - ao3 += 1; - ao4 += 1; - b += 4; - } - - } else - if (X < posY) { - if (m & 2) { - ao1 += 2 * lda; - ao2 += 2 * lda; - - b += 8; - } - - if (m & 1) { - ao1 += lda; - b += 4; - } - - } else { + b[ 1] = ZERO; + b[ 2] = ZERO; + b[ 3] = ZERO; + b[ 4] = ZERO; + b[ 5] = ZERO; + + b[ 6] = *(ao1 + 1); +#ifdef UNIT + b[ 7] = ONE; +#else + b[ 7] = *(ao2 + 1); +#endif + b[ 8] = ZERO; + b[ 9] = ZERO; + b[10] = ZERO; + b[11] = ZERO; + + b[12] = *(ao1 + 2); + b[13] = *(ao2 + 2); +#ifdef UNIT + b[14] = ONE; +#else + b[14] = *(ao3 + 2); +#endif + b[15] = ZERO; + b[16] = ZERO; + b[17] = ZERO; + + b[18] = *(ao1 + 3); + b[19] = *(ao2 + 3); + b[20] = *(ao3 + 3); +#ifdef UNIT + b[21] = ONE; +#else + b[21] = *(ao4 + 3); +#endif + b[22] = ZERO; + b[23] = ZERO; + + b[24] = *(ao1 + 4); + b[25] = *(ao2 + 4); + b[26] = *(ao3 + 4); + b[27] = *(ao4 + 4); +#ifdef UNIT + b[28] = ONE; +#else + b[28] = *(ao5 + 4); +#endif + b[29] = ZERO; + + b[30] = *(ao1 + 5); + b[31] = *(ao2 + 5); + b[32] = *(ao3 + 5); + b[33] = *(ao4 + 5); + b[34] = *(ao5 + 5); +#ifdef UNIT + b[35] = ONE; +#else + b[35] = *(ao6 + 5); +#endif + ao1 += 6; + ao2 += 6; + ao3 += 6; + ao4 += 6; + ao5 += 6; + ao6 += 6; + b += 36; + } + + X += 6; + i --; + } while (i > 0); + } + + i = (m % 6); + if (i) { + + if (X > posY) { + for (ii = 0; ii < i; ii++){ + b[ 0] = *(ao1 + 0); + b[ 1] = *(ao2 + 0); + b[ 2] = *(ao3 + 0); + b[ 3] = *(ao4 + 0); + b[ 4] = *(ao5 + 0); + b[ 5] = *(ao6 + 0); + + ao1 ++; + ao2 ++; + ao3 ++; + ao4 ++; + ao5 ++; + ao6 ++; + b += 6; + } + + } else if (X < posY) { + + b += 6 * i; + + } else { +#ifdef UNIT + b[ 0] = ONE; +#else + b[ 0] = *(ao1 + 0); +#endif + b[ 1] = ZERO; + b[ 2] = ZERO; + b[ 3] = ZERO; + b[ 4] = ZERO; + b[ 5] = ZERO; + b += 6; + + if (i >= 2) { + b[ 0] = *(ao1 + 1); +#ifdef UNIT + b[ 1] = ONE; +#else + b[ 1] = *(ao2 + 1); +#endif + b[ 2] = ZERO; + b[ 3] = ZERO; + b[ 4] = ZERO; + b[ 5] = ZERO; + b += 6; + } + + if (i >= 3) { + b[ 0] = *(ao1 + 2); + b[ 1] = *(ao2 + 2); +#ifdef UNIT + b[ 2] = ONE; +#else + b[ 2] = *(ao3 + 2); +#endif + b[ 3] = ZERO; + b[ 4] = ZERO; + b[ 5] = ZERO; + b += 6; + } + + if (i >= 4) { + b[ 0] = *(ao1 + 3); + b[ 1] = *(ao2 + 3); + b[ 2] = *(ao3 + 3); +#ifdef UNIT + b[ 3] = ONE; +#else + b[ 3] = *(ao4 + 3); +#endif + b[ 4] = ZERO; + b[ 5] = ZERO; + b += 6; + } + + if (i >= 5) { + b[ 0] = *(ao1 + 4); + b[ 1] = *(ao2 + 4); + b[ 2] = *(ao3 + 4); + b[ 3] = *(ao4 + 4); +#ifdef UNIT + b[ 4] = ONE; +#else + b[ 4] = *(ao5 + 4); +#endif + b[ 5] = ZERO; + b += 6; + } + } + } + + posY += 6; + js --; + } while (js > 0); + } /* End of main loop */ + + if ((n % 6) & 4){ + X = posX; + + if (posX <= posY) { + ao1 = a + posY + (posX + 0) * lda; + ao2 = a + posY + (posX + 1) * lda; + ao3 = a + posY + (posX + 2) * lda; + ao4 = a + posY + (posX + 3) * lda; + } else { + ao1 = a + posX + (posY + 0) * lda; + ao2 = a + posX + (posY + 1) * lda; + ao3 = a + posX + (posY + 2) * lda; + ao4 = a + posX + (posY + 3) * lda; + } + + i = (m >> 1); + if (i > 0) { + do { + if (X > posY) { + for (ii = 0; ii < 2; ii++){ + + b[ 0] = *(ao1 + 0); + b[ 1] = *(ao2 + 0); + b[ 2] = *(ao3 + 0); + b[ 3] = *(ao4 + 0); + + ao1 ++; + ao2 ++; + ao3 ++; + ao4 ++; + b += 4; + } + } else if (X < posY) { + ao1 += 2 * lda; + ao2 += 2 * lda; + ao3 += 2 * lda; + ao4 += 2 * lda; + b += 8; + } else { +#ifdef UNIT + b[ 0] = ONE; +#else + b[ 0] = *(ao1 + 0); +#endif + b[ 1] = ZERO; + b[ 2] = ZERO; + b[ 3] = ZERO; + + b[ 4] = *(ao1 + 1); +#ifdef UNIT + b[ 5] = ONE; +#else + b[ 5] = *(ao2 + 1); +#endif + b[ 6] = ZERO; + b[ 7] = ZERO; + + b[ 8] = *(ao1 + 2); + b[ 9] = *(ao2 + 2); +#ifdef UNIT + b[ 10] = ONE; +#else + b[ 10] = *(ao3 + 2); +#endif + b[ 11] = ZERO; + + b[ 12] = *(ao1 + 3); + b[ 13] = *(ao2 + 3); + b[ 14] = *(ao3 + 3); +#ifdef UNIT + b[ 15] = ONE; +#else + b[ 15] = *(ao4 + 3); +#endif + + ao1 += 4; + ao2 += 4; + ao3 += 4; + ao4 += 4; + b += 16; + X += 4; + i -= 2; + continue; + } + + X += 2; + i --; + } while (i > 0); + } + + i = (m & 1); + if (i) { + + if (X > posY) { + for (ii = 0; ii < i; ii++){ + + b[ 0] = *(ao1 + 0); + b[ 1] = *(ao2 + 0); + b[ 2] = *(ao3 + 0); + b[ 3] = *(ao4 + 0); + + ao1 ++; + ao2 ++; + ao3 ++; + ao4 ++; + b += 4; + } + } else if (X < posY) { + /* ao1 += i * lda; + ao2 += i * lda; + ao3 += i * lda; + ao4 += i * lda; */ + b += 4 * i; + } else { #ifdef UNIT - data05 = *(ao2 + 0); - data09 = *(ao3 + 0); - data13 = *(ao4 + 0); - - if (i >= 2) { - data10 = *(ao3 + 1); - data14 = *(ao4 + 1); - } - - if (i >= 3) { - data15 = *(ao4 + 2); - } - - b[ 0] = ONE; - b[ 1] = data05; - b[ 2] = data09; - b[ 3] = data13; - b += 4; - - if(i >= 2) { - b[ 0] = ZERO; - b[ 1] = ONE; - b[ 2] = data10; - b[ 3] = data14; - b += 4; - } - - if (i >= 3) { - b[ 0] = ZERO; - b[ 1] = ZERO; - b[ 2] = ONE; - b[ 3] = data15; - b += 4; - } + b[ 0] = ONE; #else - data01 = *(ao1 + 0); - data05 = *(ao2 + 0); - data09 = *(ao3 + 0); - data13 = *(ao4 + 0); - - if (i >= 2) { - data06 = *(ao2 + 1); - data10 = *(ao3 + 1); - data14 = *(ao4 + 1); - } - - if (i >= 3) { - data11 = *(ao3 + 2); - data15 = *(ao4 + 2); - } - - b[ 0] = data01; - b[ 1] = data05; - b[ 2] = data09; - b[ 3] = data13; - b += 4; - - if(i >= 2) { - b[ 0] = ZERO; - b[ 1] = data06; - b[ 2] = data10; - b[ 3] = data14; - b += 4; - } - - if (i >= 3) { - b[ 0] = ZERO; - b[ 1] = ZERO; - b[ 2] = data11; - b[ 3] = data15; - b += 4; - } + b[ 0] = *(ao1 + 0); #endif - } - } - - posY += 4; - js --; - } while (js > 0); - } /* End of main loop */ - - - if (n & 2){ - X = posX; - - if (posX <= posY) { - ao1 = a + posY + (posX + 0) * lda; - ao2 = a + posY + (posX + 1) * lda; - } else { - ao1 = a + posX + (posY + 0) * lda; - ao2 = a + posX + (posY + 1) * lda; - } - - i = (m >> 1); - if (i > 0) { - do { - if (X > posY) { - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); - data05 = *(ao2 + 0); - data06 = *(ao2 + 1); - - b[ 0] = data01; - b[ 1] = data05; - b[ 2] = data02; - b[ 3] = data06; - - ao1 += 2; - ao2 += 2; - b += 4; - - } else - if (X < posY) { - ao1 += 2 * lda; - ao2 += 2 * lda; - b += 4; - } else { + b[ 1] = ZERO; + b[ 2] = ZERO; + b[ 3] = ZERO; + b += 4; + } + } + + posY += 4; + } + + + if ((n % 6) & 2){ + X = posX; + + if (posX <= posY) { + ao1 = a + posY + (posX + 0) * lda; + ao2 = a + posY + (posX + 1) * lda; + } else { + ao1 = a + posX + (posY + 0) * lda; + ao2 = a + posX + (posY + 1) * lda; + } + + i = (m >> 1); + if (i > 0) { + do { + if (X > posY) { + data01 = *(ao1 + 0); + data02 = *(ao1 + 1); + data05 = *(ao2 + 0); + data06 = *(ao2 + 1); + + b[ 0] = data01; + b[ 1] = data05; + b[ 2] = data02; + b[ 3] = data06; + + ao1 += 2; + ao2 += 2; + b += 4; + + } else if (X < posY) { + ao1 += 2 * lda; + ao2 += 2 * lda; + b += 4; + } else { #ifdef UNIT - data02 = *(ao1 + 1); + data02 = *(ao1 + 1); - b[ 0] = ONE; - b[ 1] = ZERO; - b[ 2] = data02; - b[ 3] = ONE; + b[ 0] = ONE; + b[ 1] = ZERO; + b[ 2] = data02; + b[ 3] = ONE; #else - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); - data06 = *(ao2 + 1); - - b[ 0] = data01; - b[ 1] = ZERO; - b[ 2] = data02; - b[ 3] = data06; + data01 = *(ao1 + 0); + data02 = *(ao1 + 1); + data06 = *(ao2 + 1); + + b[ 0] = data01; + b[ 1] = ZERO; + b[ 2] = data02; + b[ 3] = data06; #endif - ao1 += 2; - ao2 += 2; - - b += 4; - } - - X += 2; - i --; - } while (i > 0); - } - - i = (m & 1); - if (i) { - - if (X > posY) { - data01 = *(ao1 + 0); - data02 = *(ao2 + 0); - b[ 0] = data01; - b[ 1] = data02; - - ao1 += 1; - ao2 += 1; - b += 2; - } else - if (X < posY) { - ao1 += lda; - b += 2; - } else { + ao1 += 2; + ao2 += 2; + + b += 4; + } + + X += 2; + i --; + } while (i > 0); + } + + i = (m & 1); + if (i) { + + if (X > posY) { + data01 = *(ao1 + 0); + data02 = *(ao2 + 0); + b[ 0] = data01; + b[ 1] = data02; + + ao1 += 1; + ao2 += 1; + b += 2; + } else if (X < posY) { + ao1 += lda; + b += 2; + } else { #ifdef UNIT - data05 = *(ao2 + 0); + data05 = *(ao2 + 0); - b[ 0] = ONE; - b[ 1] = data05; + b[ 0] = ONE; + b[ 1] = data05; #else - data01 = *(ao1 + 0); - data05 = *(ao2 + 0); + data01 = *(ao1 + 0); + data05 = *(ao2 + 0); - b[ 0] = data01; - b[ 1] = data05; + b[ 0] = data01; + b[ 1] = data05; #endif - b += 2; - } - } - posY += 2; - } - - if (n & 1){ - X = posX; - - if (posX <= posY) { - ao1 = a + posY + (posX + 0) * lda; - } else { - ao1 = a + posX + (posY + 0) * lda; - } - - i = m; - if (i > 0) { - do { - if (X > posY) { - data01 = *(ao1 + 0); - b[ 0] = data01; - b += 1; - ao1 += 1; - } else - if (X < posY) { - b += 1; - ao1 += lda; - } else { + b += 2; + } + } + posY += 2; + } + + if ((n % 6) & 1){ + X = posX; + + if (posX <= posY) { + ao1 = a + posY + (posX + 0) * lda; + } else { + ao1 = a + posX + (posY + 0) * lda; + } + + i = m; + if (i > 0) { + do { + if (X > posY) { + data01 = *(ao1 + 0); + b[ 0] = data01; + b += 1; + ao1 += 1; + } else if (X < posY) { + b += 1; + ao1 += lda; + } else { #ifdef UNIT - b[ 0] = ONE; + b[ 0] = ONE; #else - data01 = *(ao1 + 0); - b[ 0] = data01; + data01 = *(ao1 + 0); + b[ 0] = data01; #endif - b += 1; - ao1 += 1; - } + b += 1; + ao1 += 1; + } - X ++; - i --; - } while (i > 0); - } + X ++; + i --; + } while (i > 0); + } - posY += 1; - } + posY += 1; + } - return 0; + return 0; } diff --git a/kernel/generic/trmm_ltcopy_6.c b/kernel/generic/trmm_ltcopy_6.c index 66a7325bb1..7c22450369 100644 --- a/kernel/generic/trmm_ltcopy_6.c +++ b/kernel/generic/trmm_ltcopy_6.c @@ -41,448 +41,511 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, FLOAT *b){ - BLASLONG i, js; - BLASLONG X; - - FLOAT data01, data02, data03, data04, data05, data06, data07, data08; - FLOAT data09, data10, data11, data12, data13, data14, data15, data16; - FLOAT *ao1, *ao2, *ao3, *ao4; - - js = (n >> 2); - - if (js > 0){ - do { - X = posX; - - if (posX <= posY) { - ao1 = a + posY + (posX + 0) * lda; - ao2 = a + posY + (posX + 1) * lda; - ao3 = a + posY + (posX + 2) * lda; - ao4 = a + posY + (posX + 3) * lda; - } else { - ao1 = a + posX + (posY + 0) * lda; - ao2 = a + posX + (posY + 1) * lda; - ao3 = a + posX + (posY + 2) * lda; - ao4 = a + posX + (posY + 3) * lda; - } - - i = (m >> 2); - if (i > 0) { - do { - if (X > posY) { - ao1 += 4; - ao2 += 4; - ao3 += 4; - ao4 += 4; - b += 16; - - } else - if (X < posY) { - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); - data03 = *(ao1 + 2); - data04 = *(ao1 + 3); - - data05 = *(ao2 + 0); - data06 = *(ao2 + 1); - data07 = *(ao2 + 2); - data08 = *(ao2 + 3); - - data09 = *(ao3 + 0); - data10 = *(ao3 + 1); - data11 = *(ao3 + 2); - data12 = *(ao3 + 3); - - data13 = *(ao4 + 0); - data14 = *(ao4 + 1); - data15 = *(ao4 + 2); - data16 = *(ao4 + 3); - - b[ 0] = data01; - b[ 1] = data02; - b[ 2] = data03; - b[ 3] = data04; - b[ 4] = data05; - b[ 5] = data06; - b[ 6] = data07; - b[ 7] = data08; - - b[ 8] = data09; - b[ 9] = data10; - b[10] = data11; - b[11] = data12; - b[12] = data13; - b[13] = data14; - b[14] = data15; - b[15] = data16; - - ao1 += 4 * lda; - ao2 += 4 * lda; - ao3 += 4 * lda; - ao4 += 4 * lda; - b += 16; - - } else { + BLASLONG i, js, ii; + BLASLONG X; + + FLOAT data01, data02, data05, data06; + FLOAT *ao1, *ao2, *ao3, *ao4, *ao5, *ao6; + + js = (n / 6); + + if (js > 0){ + do { + X = posX; + + if (posX <= posY) { + ao1 = a + posY + (posX + 0) * lda; + ao2 = a + posY + (posX + 1) * lda; + ao3 = a + posY + (posX + 2) * lda; + ao4 = a + posY + (posX + 3) * lda; + ao5 = a + posY + (posX + 4) * lda; + ao6 = a + posY + (posX + 5) * lda; + } else { + ao1 = a + posX + (posY + 0) * lda; + ao2 = a + posX + (posY + 1) * lda; + ao3 = a + posX + (posY + 2) * lda; + ao4 = a + posX + (posY + 3) * lda; + ao5 = a + posX + (posY + 4) * lda; + ao6 = a + posX + (posY + 5) * lda; + } + + i = (m / 6); + if (i > 0) { + do { + if (X > posY) { + ao1 += 6; + ao2 += 6; + ao3 += 6; + ao4 += 6; + ao5 += 6; + ao6 += 6; + b += 36; + + } else if (X < posY) { + for (ii = 0; ii < 6; ii++){ + + b[ 0] = *(ao1 + 0); + b[ 1] = *(ao1 + 1); + b[ 2] = *(ao1 + 2); + b[ 3] = *(ao1 + 3); + b[ 4] = *(ao1 + 4); + b[ 5] = *(ao1 + 5); + + ao1 += lda; + b += 6; + } + + ao2 += 6 * lda; + ao3 += 6 * lda; + ao4 += 6 * lda; + ao5 += 6 * lda; + ao6 += 6 * lda; + + } else { +#ifdef UNIT + b[ 0] = ONE; +#else + b[ 0] = *(ao1 + 0); +#endif + b[ 1] = *(ao1 + 1); + b[ 2] = *(ao1 + 2); + b[ 3] = *(ao1 + 3); + b[ 4] = *(ao1 + 4); + b[ 5] = *(ao1 + 5); + + b[ 6] = ZERO; +#ifdef UNIT + b[ 7] = ONE; +#else + b[ 7] = *(ao2 + 1); +#endif + b[ 8] = *(ao2 + 2); + b[ 9] = *(ao2 + 3); + b[10] = *(ao2 + 4); + b[11] = *(ao2 + 5); + + b[12] = ZERO; + b[13] = ZERO; +#ifdef UNIT + b[14] = ONE; +#else + b[14] = *(ao3 + 2); +#endif + b[15] = *(ao3 + 3); + b[16] = *(ao3 + 4); + b[17] = *(ao3 + 5); + + b[18] = ZERO; + b[19] = ZERO; + b[20] = ZERO; +#ifdef UNIT + b[21] = ONE; +#else + b[21] = *(ao4 + 3); +#endif + b[22] = *(ao4 + 4); + b[23] = *(ao4 + 5); + + b[24] = ZERO; + b[25] = ZERO; + b[26] = ZERO; + b[27] = ZERO; +#ifdef UNIT + b[28] = ONE; +#else + b[28] = *(ao5 + 4); +#endif + b[29] = *(ao5 + 5); + + b[30] = ZERO; + b[31] = ZERO; + b[32] = ZERO; + b[33] = ZERO; + b[34] = ZERO; +#ifdef UNIT + b[35] = ONE; +#else + b[35] = *(ao6 + 5); +#endif + + ao1 += 6; + ao2 += 6; + ao3 += 6; + ao4 += 6; + ao5 += 6; + ao6 += 6; + b += 36; + } + + X += 6; + i --; + } while (i > 0); + } + + i = (m % 6); + if (i) { + + if (X > posY) { + + b += 6 * i; + + } else if (X < posY) { + for (ii = 0; ii < i; ii++){ + + b[ 0] = *(ao1 + 0); + b[ 1] = *(ao1 + 1); + b[ 2] = *(ao1 + 2); + b[ 3] = *(ao1 + 3); + b[ 4] = *(ao1 + 4); + b[ 5] = *(ao1 + 5); + + ao1 += lda; + ao2 += lda; + ao3 += lda; + ao4 += lda; + ao5 += lda; + ao6 += lda; + b += 6; + } + + } else { +#ifdef UNIT + b[ 0] = ONE; +#else + b[ 0] = *(ao1 + 0); +#endif + b[ 1] = *(ao1 + 1); + b[ 2] = *(ao1 + 2); + b[ 3] = *(ao1 + 3); + b[ 4] = *(ao1 + 4); + b[ 5] = *(ao1 + 5); + b += 6; + + if (i >= 2) { + b[ 0] = ZERO; +#ifdef UNIT + b[ 1] = ONE; +#else + b[ 1] = *(ao2 + 1); +#endif + b[ 2] = *(ao2 + 2); + b[ 3] = *(ao2 + 3); + b[ 4] = *(ao2 + 4); + b[ 5] = *(ao2 + 5); + b += 6; + } + + if (i >= 3) { + b[ 0] = ZERO; + b[ 1] = ZERO; +#ifdef UNIT + b[ 2] = ONE; +#else + b[ 2] = *(ao3 + 2); +#endif + b[ 3] = *(ao3 + 3); + b[ 4] = *(ao3 + 4); + b[ 5] = *(ao3 + 5); + b += 6; + } + + if (i >= 4) { + b[ 0] = ZERO; + b[ 1] = ZERO; + b[ 2] = ZERO; +#ifdef UNIT + b[ 3] = ONE; +#else + b[ 3] = *(ao4 + 3); +#endif + b[ 4] = *(ao4 + 4); + b[ 5] = *(ao4 + 5); + b += 6; + } + + if (i >= 5) { + b[ 0] = ZERO; + b[ 1] = ZERO; + b[ 2] = ZERO; + b[ 3] = ZERO; +#ifdef UNIT + b[ 4] = ONE; +#else + b[ 4] = *(ao5 + 4); +#endif + b[ 5] = *(ao5 + 5); + b += 6; + } + } + } + + posY += 6; + js --; + } while (js > 0); + } /* End of main loop */ + + if ((n % 6) & 4){ + X = posX; + + if (posX <= posY) { + ao1 = a + posY + (posX + 0) * lda; + ao2 = a + posY + (posX + 1) * lda; + ao3 = a + posY + (posX + 2) * lda; + ao4 = a + posY + (posX + 3) * lda; + } else { + ao1 = a + posX + (posY + 0) * lda; + ao2 = a + posX + (posY + 1) * lda; + ao3 = a + posX + (posY + 2) * lda; + ao4 = a + posX + (posY + 3) * lda; + } + + i = (m >> 1); + if (i > 0) { + do { + if (X > posY) { + ao1 += 2; + ao2 += 2; + ao3 += 2; + ao4 += 2; + b += 8; + } else if (X < posY) { + + for (ii = 0; ii < 2; ii++){ + b[ 0] = *(ao1 + 0); + b[ 1] = *(ao1 + 1); + b[ 2] = *(ao1 + 2); + b[ 3] = *(ao1 + 3); + ao1 += lda; + b += 4; + } + + ao2 += 2 * lda; + ao3 += 2 * lda; + ao4 += 2 * lda; + } else { + +#ifdef UNIT + b[ 0] = ONE; +#else + b[ 0] = *(ao1 + 0); +#endif + b[ 1] = *(ao1 + 1); + b[ 2] = *(ao1 + 2); + b[ 3] = *(ao1 + 3); + + b[ 4] = ZERO; +#ifdef UNIT + b[ 5] = ONE; +#else + b[ 5] = *(ao2 + 1); +#endif + b[ 6] = *(ao2 + 2); + b[ 7] = *(ao2 + 3); + b[ 8] = ZERO; + b[ 9] = ZERO; #ifdef UNIT - data02 = *(ao1 + 1); - data03 = *(ao1 + 2); - data04 = *(ao1 + 3); - - data07 = *(ao2 + 2); - data08 = *(ao2 + 3); - - data12 = *(ao3 + 3); - - b[ 0] = ONE; - b[ 1] = data02; - b[ 2] = data03; - b[ 3] = data04; - - b[ 4] = ZERO; - b[ 5] = ONE; - b[ 6] = data07; - b[ 7] = data08; - - b[ 8] = ZERO; - b[ 9] = ZERO; - b[10] = ONE; - b[11] = data12; - - b[12] = ZERO; - b[13] = ZERO; - b[14] = ZERO; - b[15] = ONE; + b[ 10] = ONE; #else - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); - data03 = *(ao1 + 2); - data04 = *(ao1 + 3); - - data06 = *(ao2 + 1); - data07 = *(ao2 + 2); - data08 = *(ao2 + 3); - - data11 = *(ao3 + 2); - data12 = *(ao3 + 3); - - data16 = *(ao4 + 3); - - b[ 0] = data01; - b[ 1] = data02; - b[ 2] = data03; - b[ 3] = data04; - b[ 4] = ZERO; - b[ 5] = data06; - b[ 6] = data07; - b[ 7] = data08; - - b[ 8] = ZERO; - b[ 9] = ZERO; - b[10] = data11; - b[11] = data12; - b[12] = ZERO; - b[13] = ZERO; - b[14] = ZERO; - b[15] = data16; + b[ 10] = *(ao3 + 2); #endif - ao1 += 4; - ao2 += 4; - ao3 += 4; - ao4 += 4; - b += 16; - } - - X += 4; - i --; - } while (i > 0); - } - - i = (m & 3); - if (i) { - - if (X > posY) { - - if (m & 2) { - ao1 += 2; - ao2 += 2; - ao3 += 2; - ao4 += 2; - b += 8; - } - - if (m & 1) { - ao1 += 1; - ao2 += 1; - ao3 += 1; - ao4 += 1; - b += 4; - } - - } else - if (X < posY) { - if (m & 2) { - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); - data03 = *(ao1 + 2); - data04 = *(ao1 + 3); - data05 = *(ao2 + 0); - data06 = *(ao2 + 1); - data07 = *(ao2 + 2); - data08 = *(ao2 + 3); - - b[ 0] = data01; - b[ 1] = data02; - b[ 2] = data03; - b[ 3] = data04; - b[ 4] = data05; - b[ 5] = data06; - b[ 6] = data07; - b[ 7] = data08; - - ao1 += 2 * lda; - ao2 += 2 * lda; - - b += 8; - } - - if (m & 1) { - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); - data03 = *(ao1 + 2); - data04 = *(ao1 + 3); - - b[ 0] = data01; - b[ 1] = data02; - b[ 2] = data03; - b[ 3] = data04; - - ao1 += lda; - b += 4; - } - - } else { + b[ 11] = *(ao3 + 3); + b[ 12] = ZERO; + b[ 13] = ZERO; + b[ 14] = ZERO; +#ifdef UNIT + b[ 15] = ONE; +#else + b[ 15] = *(ao4 + 3); +#endif + ao1 += 4; + ao2 += 4; + ao3 += 4; + ao4 += 4; + b += 16; + X += 4; + i -= 2; + continue; + } + X += 2; + i --; + } while (i > 0); + } + + i = (m & 1); + if (i > 0) { + if (X > posY) { + /* ao1 += i; + ao2 += i; + ao3 += i; + ao4 += i; */ + b += 4 * i; + } else if (X < posY) { + + for (ii = 0; ii < i; ii++){ + + b[ 0] = *(ao1 + 0); + b[ 1] = *(ao1 + 1); + b[ 2] = *(ao1 + 2); + b[ 3] = *(ao1 + 3); + + // ao1 += lda; + // ao2 += lda; + // ao3 += lda; + // ao4 += lda; + b += 4; + } + } else { #ifdef UNIT - data02 = *(ao1 + 1); - data03 = *(ao1 + 2); - data04 = *(ao1 + 3); - - if (i >= 2) { - data07 = *(ao2 + 2); - data08 = *(ao2 + 3); - } - - if (i >= 3) { - data12 = *(ao3 + 3); - } - - b[ 0] = ONE; - b[ 1] = data02; - b[ 2] = data03; - b[ 3] = data04; - b += 4; - - if(i >= 2) { - b[ 0] = ZERO; - b[ 1] = ONE; - b[ 2] = data07; - b[ 3] = data08; - b += 4; - } - - if (i >= 3) { - b[ 0] = ZERO; - b[ 1] = ZERO; - b[ 2] = ONE; - b[ 3] = data12; - b += 4; - } + b[ 0] = ONE; #else - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); - data03 = *(ao1 + 2); - data04 = *(ao1 + 3); - - if (i >= 2) { - data06 = *(ao2 + 1); - data07 = *(ao2 + 2); - data08 = *(ao2 + 3); - } - - if (i >= 3) { - data11 = *(ao3 + 2); - data12 = *(ao3 + 3); - } - - b[ 0] = data01; - b[ 1] = data02; - b[ 2] = data03; - b[ 3] = data04; - b += 4; - - if(i >= 2) { - b[ 0] = ZERO; - b[ 1] = data06; - b[ 2] = data07; - b[ 3] = data08; - b += 4; - } - - if (i >= 3) { - b[ 0] = ZERO; - b[ 1] = ZERO; - b[ 2] = data11; - b[ 3] = data12; - b += 4; - } + b[ 0] = *(ao1 + 0); #endif - } - } - - posY += 4; - js --; - } while (js > 0); - } /* End of main loop */ - - - if (n & 2){ - X = posX; - - if (posX <= posY) { - ao1 = a + posY + (posX + 0) * lda; - ao2 = a + posY + (posX + 1) * lda; - } else { - ao1 = a + posX + (posY + 0) * lda; - ao2 = a + posX + (posY + 1) * lda; - } - - i = (m >> 1); - if (i > 0) { - do { - if (X > posY) { - ao1 += 2; - ao2 += 2; - b += 4; - - } else - if (X < posY) { - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); - data05 = *(ao2 + 0); - data06 = *(ao2 + 1); - - b[ 0] = data01; - b[ 1] = data02; - b[ 2] = data05; - b[ 3] = data06; - - ao1 += 2 * lda; - ao2 += 2 * lda; - b += 4; - } else { + b[ 1] = *(ao1 + 1); + b[ 2] = *(ao1 + 2); + b[ 3] = *(ao1 + 3); + b += 4; + } + } + posY += 4; + } + + + if ((n % 6) & 2){ + X = posX; + + if (posX <= posY) { + ao1 = a + posY + (posX + 0) * lda; + ao2 = a + posY + (posX + 1) * lda; + } else { + ao1 = a + posX + (posY + 0) * lda; + ao2 = a + posX + (posY + 1) * lda; + } + + i = (m >> 1); + if (i > 0) { + do { + if (X > posY) { + ao1 += 2; + ao2 += 2; + b += 4; + + } else if (X < posY) { + data01 = *(ao1 + 0); + data02 = *(ao1 + 1); + data05 = *(ao2 + 0); + data06 = *(ao2 + 1); + + b[ 0] = data01; + b[ 1] = data02; + b[ 2] = data05; + b[ 3] = data06; + + ao1 += 2 * lda; + ao2 += 2 * lda; + b += 4; + } else { #ifdef UNIT - data02 = *(ao1 + 1); + data02 = *(ao1 + 1); - b[ 0] = ONE; - b[ 1] = data02; - b[ 2] = ZERO; - b[ 3] = ONE; + b[ 0] = ONE; + b[ 1] = data02; + b[ 2] = ZERO; + b[ 3] = ONE; #else - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); - data06 = *(ao2 + 1); - - b[ 0] = data01; - b[ 1] = data02; - b[ 2] = ZERO; - b[ 3] = data06; + data01 = *(ao1 + 0); + data02 = *(ao1 + 1); + data06 = *(ao2 + 1); + + b[ 0] = data01; + b[ 1] = data02; + b[ 2] = ZERO; + b[ 3] = data06; #endif - ao1 += 2; - ao2 += 2; - b += 4; - } - - X += 2; - i --; - } while (i > 0); - } - - i = (m & 1); - if (i) { - - if (X > posY) { - ao1 += 1; - ao2 += 1; - - b += 2; - } else - if (X < posY) { - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); - - b[ 0] = data01; - b[ 1] = data02; - ao1 += lda; - b += 2; - } else { + ao1 += 2; + ao2 += 2; + b += 4; + } + + X += 2; + i --; + } while (i > 0); + } + + i = (m & 1); + if (i) { + + if (X > posY) { + ao1 += 1; + ao2 += 1; + + b += 2; + } else if (X < posY) { + data01 = *(ao1 + 0); + data02 = *(ao1 + 1); + + b[ 0] = data01; + b[ 1] = data02; + ao1 += lda; + b += 2; + } else { #ifdef UNIT - data02 = *(ao1 + 1); + data02 = *(ao1 + 1); - b[ 0] = ONE; - b[ 1] = data02; + b[ 0] = ONE; + b[ 1] = data02; #else - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); + data01 = *(ao1 + 0); + data02 = *(ao1 + 1); - b[ 0] = data01; - b[ 1] = data02; + b[ 0] = data01; + b[ 1] = data02; #endif - b += 2; - } - } - posY += 2; - } - - if (n & 1){ - X = posX; - - if (posX <= posY) { - ao1 = a + posY + (posX + 0) * lda; - } else { - ao1 = a + posX + (posY + 0) * lda; - } - - i = m; - if (i > 0) { - do { - if (X > posY) { - b += 1; - ao1 += 1; - } else - if (X < posY) { - data01 = *(ao1 + 0); - b[ 0] = data01; - ao1 += lda; - b += 1; - } else { + b += 2; + } + } + posY += 2; + } + + if ((n % 6) & 1){ + X = posX; + + if (posX <= posY) { + ao1 = a + posY + (posX + 0) * lda; + } else { + ao1 = a + posX + (posY + 0) * lda; + } + + i = m; + if (i > 0) { + do { + if (X > posY) { + b += 1; + ao1 += 1; + } else if (X < posY) { + data01 = *(ao1 + 0); + b[ 0] = data01; + ao1 += lda; + b += 1; + } else { #ifdef UNIT - b[ 0] = ONE; + b[ 0] = ONE; #else - data01 = *(ao1 + 0); - b[ 0] = data01; + data01 = *(ao1 + 0); + b[ 0] = data01; #endif - ao1 += 1; - b += 1; - } + ao1 += 1; + b += 1; + } - X ++; - i --; - } while (i > 0); - } + X ++; + i --; + } while (i > 0); + } - posY += 1; - } + posY += 1; + } - return 0; + return 0; } diff --git a/kernel/generic/trmm_uncopy_6.c b/kernel/generic/trmm_uncopy_6.c index 4878f3f530..9521cc7243 100644 --- a/kernel/generic/trmm_uncopy_6.c +++ b/kernel/generic/trmm_uncopy_6.c @@ -41,745 +41,544 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, FLOAT *b){ - BLASLONG i, js; - BLASLONG X, mm; - - FLOAT data01, data02, data03, data04, data05, data06; - FLOAT data07, data08, data09, data10, data11, data12; - FLOAT data13, data14, data15, data16, data17, data18; - FLOAT data19, data20, data21, data22, data23, data24; - FLOAT data25, data26, data27, data28, data29, data30; - FLOAT data31, data32, data33, data34, data35, data36; - - FLOAT *ao1, *ao2, *ao3, *ao4, *ao5, *ao6; - - //js = (n >> 2); - js = n/6; - if (js > 0){ - do { - X = posX; - - if (posX <= posY) { - ao1 = a + posX + (posY + 0) * lda; - ao2 = a + posX + (posY + 1) * lda; - ao3 = a + posX + (posY + 2) * lda; - ao4 = a + posX + (posY + 3) * lda; - ao5 = a + posX + (posY + 4) * lda; - ao6 = a + posX + (posY + 5) * lda; - } else { - ao1 = a + posY + (posX + 0) * lda; - ao2 = a + posY + (posX + 1) * lda; - ao3 = a + posY + (posX + 2) * lda; - ao4 = a + posY + (posX + 3) * lda; - ao5 = a + posY + (posX + 4) * lda; - ao6 = a + posY + (posX + 5) * lda; - } - - i = m/6; - if (i > 0) { - do { - if (X < posY) { - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); - data03 = *(ao1 + 2); - data04 = *(ao1 + 3); - data05 = *(ao1 + 4); - data06 = *(ao1 + 5); - - data07 = *(ao2 + 0); - data08 = *(ao2 + 1); - data09 = *(ao2 + 2); - data10 = *(ao2 + 3); - data11 = *(ao2 + 4); - data12 = *(ao2 + 5); - - data13 = *(ao3 + 0); - data14 = *(ao3 + 1); - data15 = *(ao3 + 2); - data16 = *(ao3 + 3); - data17 = *(ao3 + 4); - data18 = *(ao3 + 5); - - data19 = *(ao4 + 0); - data20 = *(ao4 + 1); - data21 = *(ao4 + 2); - data22 = *(ao4 + 3); - data23 = *(ao4 + 4); - data24 = *(ao4 + 5); - - data25 = *(ao5 + 0); - data26 = *(ao5 + 1); - data27 = *(ao5 + 2); - data28 = *(ao5 + 3); - data29 = *(ao5 + 4); - data30 = *(ao5 + 5); - - data31 = *(ao6 + 0); - data32 = *(ao6 + 1); - data33 = *(ao6 + 2); - data34 = *(ao6 + 3); - data35 = *(ao6 + 4); - data36 = *(ao6 + 5); - - b[ 0] = data01; - b[ 1] = data07; - b[ 2] = data13; - b[ 3] = data19; - b[ 4] = data25; - b[ 5] = data31; - - b[ 6] = data02; - b[ 7] = data08; - b[ 8] = data14; - b[ 9] = data20; - b[10] = data26; - b[11] = data32; - - b[12] = data03; - b[13] = data09; - b[14] = data15; - b[15] = data21; - b[16] = data27; - b[17] = data33; - - b[18] = data04; - b[19] = data10; - b[20] = data16; - b[21] = data22; - b[22] = data28; - b[23] = data34; - - b[24] = data05; - b[25] = data11; - b[26] = data17; - b[27] = data23; - b[28] = data29; - b[29] = data35; - - b[30] = data06; - b[31] = data12; - b[32] = data18; - b[33] = data24; - b[34] = data30; - b[35] = data36; - - ao1 += 6; - ao2 += 6; - ao3 += 6; - ao4 += 6; - ao5 += 6; - ao6 += 6; - b += 36; - } else - if (X > posY) { - b[ 0] = ZERO; - b[ 1] = ZERO; - b[ 2] = ZERO; - b[ 3] = ZERO; - b[ 4] = ZERO; - b[ 5] = ZERO; - b[ 6] = ZERO; - b[ 7] = ZERO; - b[ 8] = ZERO; - b[ 9] = ZERO; - b[10] = ZERO; - b[11] = ZERO; - b[12] = ZERO; - b[13] = ZERO; - b[14] = ZERO; - b[15] = ZERO; - b[16] = ZERO; - b[17] = ZERO; - b[18] = ZERO; - b[19] = ZERO; - b[20] = ZERO; - b[21] = ZERO; - b[22] = ZERO; - b[23] = ZERO; - b[24] = ZERO; - b[25] = ZERO; - b[26] = ZERO; - b[27] = ZERO; - b[28] = ZERO; - b[29] = ZERO; - b[30] = ZERO; - b[31] = ZERO; - b[32] = ZERO; - b[33] = ZERO; - b[34] = ZERO; - b[35] = ZERO; - - ao1 += 6 * lda; - ao2 += 6 * lda; - ao3 += 6 * lda; - ao4 += 6 * lda; - ao5 += 6 * lda; - ao6 += 6 * lda; - - b += 36; - } else { - data01 = *(ao1 + 0); - data07 = *(ao2 + 0); - data13 = *(ao3 + 0); - data19 = *(ao4 + 0); - data25 = *(ao5 + 0); - data31 = *(ao6 + 0); - - data08 = *(ao2 + 1); - data14 = *(ao3 + 1); - data20 = *(ao4 + 1); - data26 = *(ao5 + 1); - data32 = *(ao6 + 1); - - data15 = *(ao3 + 2); - data21 = *(ao4 + 2); - data27 = *(ao5 + 2); - data33 = *(ao6 + 2); - - data22 = *(ao4 + 3); - data28 = *(ao5 + 3); - data34 = *(ao6 + 3); - - data29 = *(ao5 + 4); - data35 = *(ao6 + 4); - - data36 = *(ao6 + 5); + BLASLONG i, js, ii; + BLASLONG X; + + FLOAT data01, data02, data05, data06; + + FLOAT *ao1, *ao2, *ao3, *ao4, *ao5, *ao6; + + js = n/6; + if (js > 0){ + do { + X = posX; + + if (posX <= posY) { + ao1 = a + posX + (posY + 0) * lda; + ao2 = a + posX + (posY + 1) * lda; + ao3 = a + posX + (posY + 2) * lda; + ao4 = a + posX + (posY + 3) * lda; + ao5 = a + posX + (posY + 4) * lda; + ao6 = a + posX + (posY + 5) * lda; + } else { + ao1 = a + posY + (posX + 0) * lda; + ao2 = a + posY + (posX + 1) * lda; + ao3 = a + posY + (posX + 2) * lda; + ao4 = a + posY + (posX + 3) * lda; + ao5 = a + posY + (posX + 4) * lda; + ao6 = a + posY + (posX + 5) * lda; + } + + i = m/6; + if (i > 0) { + do { + if (X < posY) { + for (ii = 0; ii < 6; ii++){ + + b[ 0] = *(ao1 + 0); + b[ 1] = *(ao2 + 0); + b[ 2] = *(ao3 + 0); + b[ 3] = *(ao4 + 0); + b[ 4] = *(ao5 + 0); + b[ 5] = *(ao6 + 0); + + ao1 ++; + ao2 ++; + ao3 ++; + ao4 ++; + ao5 ++; + ao6 ++; + b += 6; + } + } else if (X > posY) { + // b[ 0] = ZERO; + // b[ 1] = ZERO; + // b[ 2] = ZERO; + // b[ 3] = ZERO; + // b[ 4] = ZERO; + // b[ 5] = ZERO; + // b[ 6] = ZERO; + // b[ 7] = ZERO; + // b[ 8] = ZERO; + // b[ 9] = ZERO; + // b[10] = ZERO; + // b[11] = ZERO; + // b[12] = ZERO; + // b[13] = ZERO; + // b[14] = ZERO; + // b[15] = ZERO; + // b[16] = ZERO; + // b[17] = ZERO; + // b[18] = ZERO; + // b[19] = ZERO; + // b[20] = ZERO; + // b[21] = ZERO; + // b[22] = ZERO; + // b[23] = ZERO; + // b[24] = ZERO; + // b[25] = ZERO; + // b[26] = ZERO; + // b[27] = ZERO; + // b[28] = ZERO; + // b[29] = ZERO; + // b[30] = ZERO; + // b[31] = ZERO; + // b[32] = ZERO; + // b[33] = ZERO; + // b[34] = ZERO; + // b[35] = ZERO; + + ao1 += 6 * lda; + ao2 += 6 * lda; + ao3 += 6 * lda; + ao4 += 6 * lda; + ao5 += 6 * lda; + ao6 += 6 * lda; + + b += 36; + } else { +#ifdef UNIT + b[ 0] = ONE; +#else + b[ 0] = *(ao1 + 0); +#endif + b[ 1] = *(ao2 + 0); + b[ 2] = *(ao3 + 0); + b[ 3] = *(ao4 + 0); + b[ 4] = *(ao5 + 0); + b[ 5] = *(ao6 + 0); + + b[ 6] = ZERO; +#ifdef UNIT + b[ 7] = ONE; +#else + b[ 7] = *(ao2 + 1); +#endif + b[ 8] = *(ao3 + 1); + b[ 9] = *(ao4 + 1); + b[10] = *(ao5 + 1); + b[11] = *(ao6 + 1); + + b[ 12] = ZERO; + b[ 13] = ZERO; +#ifdef UNIT + b[ 14] = ONE; +#else + b[ 14] = *(ao3 + 2); +#endif + b[ 15] = *(ao4 + 2); + b[ 16] = *(ao5 + 2); + b[ 17] = *(ao6 + 2); + + b[ 18] = ZERO; + b[ 19] = ZERO; + b[ 20] = ZERO; +#ifdef UNIT + b[ 21] = ONE; +#else + b[ 21] = *(ao4 + 3); +#endif + b[ 22] = *(ao5 + 3); + b[ 23] = *(ao6 + 3); + + b[ 24] = ZERO; + b[ 25] = ZERO; + b[ 26] = ZERO; + b[ 27] = ZERO; +#ifdef UNIT + b[ 28] = ONE; +#else + b[ 28] = *(ao5 + 4); +#endif + b[ 29] = *(ao6 + 4); + + b[ 30] = ZERO; + b[ 31] = ZERO; + b[ 32] = ZERO; + b[ 33] = ZERO; + b[ 34] = ZERO; +#ifdef UNIT + b[ 35] = ONE; +#else + b[ 35] = *(ao6 + 5); +#endif + ao1 += 6 * lda; + ao2 += 6 * lda; + ao3 += 6 * lda; + ao4 += 6 * lda; + ao5 += 6 * lda; + ao6 += 6 * lda; + + b += 36; + } + X += 6; + i --; + } while (i > 0); + } + i = m % 6; + if (i) { + if (X < posY) { + for (ii = 0; ii < i; ii++){ + + b[ 0] = *(ao1 + 0); + b[ 1] = *(ao2 + 0); + b[ 2] = *(ao3 + 0); + b[ 3] = *(ao4 + 0); + b[ 4] = *(ao5 + 0); + b[ 5] = *(ao6 + 0); + + ao1 ++; + ao2 ++; + ao3 ++; + ao4 ++; + ao5 ++; + ao6 ++; + b += 6; + } + } else if (X > posY) { + b += 6 * i; + } else { +#ifdef UNIT + b[ 0] = ONE; +#else + b[ 0] = *(ao1 + 0); +#endif + b[ 1] = *(ao2 + 0); + b[ 2] = *(ao3 + 0); + b[ 3] = *(ao4 + 0); + b[ 4] = *(ao5 + 0); + b[ 5] = *(ao6 + 0); + b += 6; + + if (i >= 2) { + b[ 0] = ZERO; +#ifdef UNIT + b[ 1] = ONE; +#else + b[ 1] = *(ao2 + 1); +#endif + b[ 2] = *(ao3 + 1); + b[ 3] = *(ao4 + 1); + b[ 4] = *(ao5 + 1); + b[ 5] = *(ao6 + 1); + b += 6; + } + + if (i >= 3) { + b[ 0] = ZERO; + b[ 1] = ZERO; +#ifdef UNIT + b[ 2] = ONE; +#else + b[ 2] = *(ao3 + 2); +#endif + b[ 3] = *(ao4 + 2); + b[ 4] = *(ao5 + 2); + b[ 5] = *(ao6 + 2); + b += 6; + } + + if (i >= 4) { + b[ 0] = ZERO; + b[ 1] = ZERO; + b[ 2] = ZERO; +#ifdef UNIT + b[ 3] = ONE; +#else + b[ 3] = *(ao4 + 3); +#endif + b[ 4] = *(ao5 + 3); + b[ 5] = *(ao6 + 3); + b += 6; + } + + if (i >= 5) { + b[ 0] = ZERO; + b[ 1] = ZERO; + b[ 2] = ZERO; + b[ 3] = ZERO; +#ifdef UNIT + b[ 4] = ONE; +#else + b[ 4] = *(ao5 + 4); +#endif + b[ 5] = *(ao6 + 4); + b += 6; + } + } + } + posY += 6; + js --; + } while (js > 0); + } /* End of main loop */ + + + if ((n % 6) & 4){ + X = posX; + + if (posX <= posY) { + ao1 = a + posX + (posY + 0) * lda; + ao2 = a + posX + (posY + 1) * lda; + ao3 = a + posX + (posY + 2) * lda; + ao4 = a + posX + (posY + 3) * lda; + } else { + ao1 = a + posY + (posX + 0) * lda; + ao2 = a + posY + (posX + 1) * lda; + ao3 = a + posY + (posX + 2) * lda; + ao4 = a + posY + (posX + 3) * lda; + } + + i = (m >> 1); + if (i > 0) { + do { + if (X < posY) { + for (ii = 0; ii < 2; ii++){ + + b[ 0] = *(ao1 + 0); + b[ 1] = *(ao2 + 0); + b[ 2] = *(ao3 + 0); + b[ 3] = *(ao4 + 0); + + ao1 ++; + ao2 ++; + ao3 ++; + ao4 ++; + b += 4; + } + } else if (X > posY) { + ao1 += 2 * lda; + ao2 += 2 * lda; + ao3 += 2 * lda; + ao4 += 2 * lda; + b += 8; + } else { +#ifdef UNIT + b[ 0] = ONE; +#else + b[ 0] = *(ao1 + 0); +#endif + b[ 1] = *(ao2 + 0); + b[ 2] = *(ao3 + 0); + b[ 3] = *(ao4 + 0); + b[ 4] = ZERO; #ifdef UNIT - b[ 0] = ONE; - b[ 1] = data07; - b[ 2] = data13; - b[ 3] = data19; - b[ 4] = data25; - b[ 5] = data31; - - b[ 6] = ZERO; - b[ 7] = ONE; - b[ 8] = data14; - b[ 9] = data20; - b[10] = data26; - b[11] = data32; - - b[12] = ZERO; - b[13] = ZERO; - b[14] = ONE; - b[15] = data21; - b[16] = data27; - b[17] = data33; - - b[18] = ZERO; - b[19] = ZERO; - b[20] = ZERO; - b[21] = ONE; - b[22] = data28; - b[23] = data34; - - b[24] = ZERO; - b[25] = ZERO; - b[26] = ZERO; - b[27] = ZERO; - b[28] = ONE; - b[29] = data35; - - b[30] = ZERO; - b[31] = ZERO; - b[32] = ZERO; - b[33] = ZERO; - b[34] = ZERO; - b[35] = ONE; + b[ 5] = ONE; #else - b[ 0] = data01; - b[ 1] = data07; - b[ 2] = data13; - b[ 3] = data19; - b[ 4] = data25; - b[ 5] = data31; - - b[ 6] = ZERO; - b[ 7] = data08; - b[ 8] = data14; - b[ 9] = data20; - b[10] = data26; - b[11] = data32; - - b[12] = ZERO; - b[13] = ZERO; - b[14] = data15; - b[15] = data21; - b[16] = data27; - b[17] = data33; - - b[18] = ZERO; - b[19] = ZERO; - b[20] = ZERO; - b[21] = data22; - b[22] = data28; - b[23] = data34; - - b[24] = ZERO; - b[25] = ZERO; - b[26] = ZERO; - b[27] = ZERO; - b[28] = data29; - b[29] = data35; - - b[30] = ZERO; - b[31] = ZERO; - b[32] = ZERO; - b[33] = ZERO; - b[34] = ZERO; - b[35] = data36; + b[ 5] = *(ao2 + 1); #endif + b[ 6] = *(ao3 + 1); + b[ 7] = *(ao4 + 1); - ao1 += 6; - ao2 += 6; - ao3 += 6; - ao4 += 6; - ao5 += 6; - ao6 += 7; - - b += 36; - } - X += 6; - i --; - } while (i > 0); - } - mm = m - m/6; - if (mm & 4) { - if (X < posY) { - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); - data03 = *(ao1 + 2); - data04 = *(ao1 + 3); - - data05 = *(ao2 + 0); - data06 = *(ao2 + 1); - data07 = *(ao2 + 2); - data08 = *(ao2 + 3); - - data09 = *(ao3 + 0); - data10 = *(ao3 + 1); - data11 = *(ao3 + 2); - data12 = *(ao3 + 3); - - data13 = *(ao4 + 0); - data14 = *(ao4 + 1); - data15 = *(ao4 + 2); - data16 = *(ao4 + 3); - - b[ 0] = data01; - b[ 1] = data05; - b[ 2] = data09; - b[ 3] = data13; - b[ 4] = data02; - b[ 5] = data06; - b[ 6] = data10; - b[ 7] = data14; - - b[ 8] = data03; - b[ 9] = data07; - b[10] = data11; - b[11] = data15; - b[12] = data04; - b[13] = data08; - b[14] = data12; - b[15] = data16; - - ao1 += 4; - ao2 += 4; - ao3 += 4; - ao4 += 4; - b += 16; - } else - if (X > posY) { - b[ 0] = ZERO; - b[ 1] = ZERO; - b[ 2] = ZERO; - b[ 3] = ZERO; - b[ 4] = ZERO; - b[ 5] = ZERO; - b[ 6] = ZERO; - b[ 7] = ZERO; - b[ 8] = ZERO; - b[ 9] = ZERO; - b[10] = ZERO; - b[11] = ZERO; - b[12] = ZERO; - b[13] = ZERO; - b[14] = ZERO; - b[15] = ZERO; - b[16] = ZERO; - b[17] = ZERO; - b[18] = ZERO; - b[19] = ZERO; - b[20] = ZERO; - b[21] = ZERO; - b[22] = ZERO; - b[23] = ZERO; - - ao1 += 4 * lda; - ao2 += 4 * lda; - ao3 += 4 * lda; - ao4 += 4 * lda; - - b += 16; - } else { + b[ 8] = ZERO; + b[ 9] = ZERO; #ifdef UNIT - data05 = *(ao2 + 0); - - data09 = *(ao3 + 0); - data10 = *(ao3 + 1); - - data13 = *(ao4 + 0); - data14 = *(ao4 + 1); - data15 = *(ao4 + 2); - - b[ 0] = ONE; - b[ 1] = data05; - b[ 2] = data09; - b[ 3] = data13; - - b[ 4] = ZERO; - b[ 5] = ONE; - b[ 6] = data10; - b[ 7] = data14; - - b[ 8] = ZERO; - b[ 9] = ZERO; - b[10] = ONE; - b[11] = data15; - - b[12] = ZERO; - b[13] = ZERO; - b[14] = ZERO; - b[15] = ONE; + b[ 10] = ONE; #else - data01 = *(ao1 + 0); - - data05 = *(ao2 + 0); - data06 = *(ao2 + 1); - - data09 = *(ao3 + 0); - data10 = *(ao3 + 1); - data11 = *(ao3 + 2); - - data13 = *(ao4 + 0); - data14 = *(ao4 + 1); - data15 = *(ao4 + 2); - data16 = *(ao4 + 3); - - b[ 0] = data01; - b[ 1] = data05; - b[ 2] = data09; - b[ 3] = data13; - - b[ 4] = ZERO; - b[ 5] = data06; - b[ 6] = data10; - b[ 7] = data14; - - b[ 8] = ZERO; - b[ 9] = ZERO; - b[10] = data11; - b[11] = data15; - - b[12] = ZERO; - b[13] = ZERO; - b[14] = ZERO; - b[15] = data16; + b[ 10] = *(ao3 + 2); #endif - ao1 += 4; - ao2 += 4; - ao3 += 4; - ao4 += 4; - - b += 16; - } - X += 4; - } - - if (mm & 3) { - if (X < posY) { - if (mm & 2) { - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); - data03 = *(ao2 + 0); - data04 = *(ao2 + 1); - data05 = *(ao3 + 0); - data06 = *(ao3 + 1); - data07 = *(ao4 + 0); - data08 = *(ao4 + 1); - - b[ 0] = data01; - b[ 1] = data03; - b[ 2] = data05; - b[ 3] = data07; - b[ 4] = data02; - b[ 5] = data04; - b[ 6] = data06; - b[ 7] = data08; - - ao1 += 2; - ao2 += 2; - ao3 += 2; - ao4 += 2; - b += 8; - } - - if (mm & 1) { - data01 = *(ao1 + 0); - data03 = *(ao2 + 0); - data05 = *(ao3 + 0); - data07 = *(ao4 + 0); - - b[ 0] = data01; - b[ 1] = data03; - b[ 2] = data05; - b[ 3] = data07; - - ao1 += 1; - ao2 += 1; - ao3 += 1; - ao4 += 1; - b += 4; - } - - } else - if (X > posY) { - if (m & 2) { - ao1 += 2 * lda; - ao2 += 2 * lda; - b += 8; - } - - if (m & 1) { - ao1 += lda; - b += 4; - } - - } else { + b[ 11] = *(ao4 + 2); + + b[ 12] = ZERO; + b[ 13] = ZERO; + b[ 14] = ZERO; +#ifdef UNIT + b[ 15] = ONE; +#else + b[ 15] = *(ao4 + 3); +#endif + + ao1 += 4 * lda; + ao2 += 4 * lda; + ao3 += 4 * lda; + ao4 += 4 * lda; + b += 16; + X += 4; + i -= 2; + continue; + } + + X += 2; + i --; + } while (i > 0); + } + + i = (m & 1); + if (i) { + + if (X < posY) { + for (ii = 0; ii < i; ii++){ + + b[ 0] = *(ao1 + 0); + b[ 1] = *(ao2 + 0); + b[ 2] = *(ao3 + 0); + b[ 3] = *(ao4 + 0); + + ao1 ++; + ao2 ++; + ao3 ++; + ao4 ++; + b += 4; + } + } else if (X > posY) { + /* ao1 += i * lda; + ao2 += i * lda; + ao3 += i * lda; + ao4 += i * lda; */ + b += 4 * i; + } else { #ifdef UNIT - data05 = *(ao2 + 0); - data09 = *(ao3 + 0); - data13 = *(ao4 + 0); - - if (i >= 2) { - data10 = *(ao3 + 1); - data14 = *(ao4 + 1); - } - - if (i >= 3) { - data15 = *(ao4 + 2); - } - - b[ 0] = ONE; - b[ 1] = data05; - b[ 2] = data09; - b[ 3] = data13; - b += 4; - - if(i >= 2) { - b[ 0] = ZERO; - b[ 1] = ONE; - b[ 2] = data10; - b[ 3] = data14; - b += 4; - } - - if (i >= 3) { - b[ 0] = ZERO; - b[ 1] = ZERO; - b[ 2] = ONE; - b[ 3] = data15; - b += 4; - } + b[ 0] = ONE; #else - data01 = *(ao1 + 0); - data05 = *(ao2 + 0); - data09 = *(ao3 + 0); - data13 = *(ao4 + 0); - - if (i >= 2) { - data06 = *(ao2 + 1); - data10 = *(ao3 + 1); - data14 = *(ao4 + 1); - } - - if (i >= 3) { - data11 = *(ao3 + 2); - data15 = *(ao4 + 2); - } - - b[ 0] = data01; - b[ 1] = data05; - b[ 2] = data09; - b[ 3] = data13; - b += 4; - - if(i >= 2) { - b[ 0] = ZERO; - b[ 1] = data06; - b[ 2] = data10; - b[ 3] = data14; - b += 4; - } - - if (i >= 3) { - b[ 0] = ZERO; - b[ 1] = ZERO; - b[ 2] = data11; - b[ 3] = data15; - b += 4; - } + b[ 0] = *(ao1 + 0); #endif - } - } - - posY += 4; - js --; - } while (js > 0); - } /* End of main loop */ - - if (n & 2){ - X = posX; - - if (posX <= posY) { - ao1 = a + posX + (posY + 0) * lda; - ao2 = a + posX + (posY + 1) * lda; - } else { - ao1 = a + posY + (posX + 0) * lda; - ao2 = a + posY + (posX + 1) * lda; - } - - i = (m >> 1); - if (i > 0) { - do { - if (X < posY) { - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); - data05 = *(ao2 + 0); - data06 = *(ao2 + 1); - - b[ 0] = data01; - b[ 1] = data05; - b[ 2] = data02; - b[ 3] = data06; - - ao1 += 2; - ao2 += 2; - b += 4; - - } else - if (X > posY) { - ao1 += 2 * lda; - ao2 += 2 * lda; - b += 4; - - } else { + b[ 1] = *(ao2 + 0); + b[ 2] = *(ao3 + 0); + b[ 3] = *(ao4 + 0); + b += 4; + } + } + + posY += 4; + } + + if ((n % 6) & 2){ + X = posX; + + if (posX <= posY) { + ao1 = a + posX + (posY + 0) * lda; + ao2 = a + posX + (posY + 1) * lda; + } else { + ao1 = a + posY + (posX + 0) * lda; + ao2 = a + posY + (posX + 1) * lda; + } + + i = (m >> 1); + if (i > 0) { + do { + if (X < posY) { + data01 = *(ao1 + 0); + data02 = *(ao1 + 1); + data05 = *(ao2 + 0); + data06 = *(ao2 + 1); + + b[ 0] = data01; + b[ 1] = data05; + b[ 2] = data02; + b[ 3] = data06; + + ao1 += 2; + ao2 += 2; + b += 4; + + } else if (X > posY) { + ao1 += 2 * lda; + ao2 += 2 * lda; + b += 4; + + } else { #ifdef UNIT - data05 = *(ao2 + 0); + data05 = *(ao2 + 0); - b[ 0] = ONE; - b[ 1] = data05; - b[ 2] = ZERO; - b[ 3] = ONE; + b[ 0] = ONE; + b[ 1] = data05; + b[ 2] = ZERO; + b[ 3] = ONE; #else - data01 = *(ao1 + 0); - data05 = *(ao2 + 0); - data06 = *(ao2 + 1); - - b[ 0] = data01; - b[ 1] = data05; - b[ 2] = ZERO; - b[ 3] = data06; + data01 = *(ao1 + 0); + data05 = *(ao2 + 0); + data06 = *(ao2 + 1); + + b[ 0] = data01; + b[ 1] = data05; + b[ 2] = ZERO; + b[ 3] = data06; #endif - ao1 += 2 * lda; - ao2 += 2 * lda; - - b += 4; - } - - X += 2; - i --; - } while (i > 0); - } - - i = (m & 1); - if (i) { - - if (X < posY) { - data01 = *(ao1 + 0); - data05 = *(ao2 + 0); - - b[ 0] = data01; - b[ 1] = data05; - ao1 += 1; - ao2 += 1; - b += 2; - } else - if (X > posY) { - ao1 += lda; - ao2 += lda; - b += 2; - } else { + ao1 += 2 * lda; + ao2 += 2 * lda; + + b += 4; + } + + X += 2; + i --; + } while (i > 0); + } + + i = (m & 1); + if (i) { + + if (X < posY) { + data01 = *(ao1 + 0); + data05 = *(ao2 + 0); + + b[ 0] = data01; + b[ 1] = data05; + ao1 += 1; + ao2 += 1; + b += 2; + } else if (X > posY) { + ao1 += lda; + ao2 += lda; + b += 2; + } else { #ifdef UNIT - data05 = *(ao2 + 0); - b[ 0] = ONE; - b[ 1] = data05; + data05 = *(ao2 + 0); + b[ 0] = ONE; + b[ 1] = data05; #else - data01 = *(ao1 + 0); - data05 = *(ao2 + 0); + data01 = *(ao1 + 0); + data05 = *(ao2 + 0); - b[ 0] = data01; - b[ 1] = data05; + b[ 0] = data01; + b[ 1] = data05; #endif - ao1 += lda; - ao2 += lda; - b += 2; - } - } - - posY += 2; - } - - if (n & 1){ - X = posX; - - if (posX <= posY) { - ao1 = a + posX + (posY + 0) * lda; - } else { - ao1 = a + posY + (posX + 0) * lda; - } - - i = m; - if (m > 0) { - do { - if (X < posY) { - data01 = *(ao1 + 0); - b[ 0] = data01; - ao1 += 1; - b += 1; - } else - if (X > posY) { - ao1 += lda; - b += 1; - } else { + ao1 += lda; + ao2 += lda; + b += 2; + } + } + + posY += 2; + } + + if ((n % 6) & 1){ + X = posX; + + if (posX <= posY) { + ao1 = a + posX + (posY + 0) * lda; + } else { + ao1 = a + posY + (posX + 0) * lda; + } + + i = m; + if (m > 0) { + do { + if (X < posY) { + data01 = *(ao1 + 0); + b[ 0] = data01; + ao1 += 1; + b += 1; + } else if (X > posY) { + ao1 += lda; + b += 1; + } else { #ifdef UNIT - b[ 0] = ONE; + b[ 0] = ONE; #else - data01 = *(ao1 + 0); - b[ 0] = data01; + data01 = *(ao1 + 0); + b[ 0] = data01; #endif - ao1 += lda; - b += 1; - } + ao1 += lda; + b += 1; + } - X += 1; - i --; - } while (i > 0); - } - } + X += 1; + i --; + } while (i > 0); + } + } - return 0; + return 0; } diff --git a/kernel/generic/trmm_utcopy_6.c b/kernel/generic/trmm_utcopy_6.c index 441f7338b5..e7ec4999f3 100644 --- a/kernel/generic/trmm_utcopy_6.c +++ b/kernel/generic/trmm_utcopy_6.c @@ -41,432 +41,510 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, FLOAT *b){ - BLASLONG i, js; - BLASLONG X; - - FLOAT data01, data02, data03, data04, data05, data06, data07, data08; - FLOAT data09, data10, data11, data12, data13, data14, data15, data16; - FLOAT *ao1, *ao2, *ao3, *ao4; - - js = (n >> 2); - - if (js > 0){ - do { - X = posX; - - if (posX <= posY) { - ao1 = a + posX + (posY + 0) * lda; - ao2 = a + posX + (posY + 1) * lda; - ao3 = a + posX + (posY + 2) * lda; - ao4 = a + posX + (posY + 3) * lda; - } else { - ao1 = a + posY + (posX + 0) * lda; - ao2 = a + posY + (posX + 1) * lda; - ao3 = a + posY + (posX + 2) * lda; - ao4 = a + posY + (posX + 3) * lda; - } - - i = (m >> 2); - if (i > 0) { - do { - if (X < posY) { - ao1 += 4; - ao2 += 4; - ao3 += 4; - ao4 += 4; - b += 16; - } else - if (X > posY) { - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); - data03 = *(ao1 + 2); - data04 = *(ao1 + 3); - - data05 = *(ao2 + 0); - data06 = *(ao2 + 1); - data07 = *(ao2 + 2); - data08 = *(ao2 + 3); - - data09 = *(ao3 + 0); - data10 = *(ao3 + 1); - data11 = *(ao3 + 2); - data12 = *(ao3 + 3); - - data13 = *(ao4 + 0); - data14 = *(ao4 + 1); - data15 = *(ao4 + 2); - data16 = *(ao4 + 3); - - b[ 0] = data01; - b[ 1] = data02; - b[ 2] = data03; - b[ 3] = data04; - b[ 4] = data05; - b[ 5] = data06; - b[ 6] = data07; - b[ 7] = data08; - - b[ 8] = data09; - b[ 9] = data10; - b[10] = data11; - b[11] = data12; - b[12] = data13; - b[13] = data14; - b[14] = data15; - b[15] = data16; - - ao1 += 4 * lda; - ao2 += 4 * lda; - ao3 += 4 * lda; - ao4 += 4 * lda; - b += 16; - - } else { + BLASLONG i, js, ii; + BLASLONG X; + + FLOAT data01, data02, data05, data06; + FLOAT *ao1, *ao2, *ao3, *ao4, *ao5, *ao6; + + js = (n / 6); + + if (js > 0){ + do { + X = posX; + + if (posX <= posY) { + ao1 = a + posX + (posY + 0) * lda; + ao2 = a + posX + (posY + 1) * lda; + ao3 = a + posX + (posY + 2) * lda; + ao4 = a + posX + (posY + 3) * lda; + ao5 = a + posX + (posY + 4) * lda; + ao6 = a + posX + (posY + 5) * lda; + } else { + ao1 = a + posY + (posX + 0) * lda; + ao2 = a + posY + (posX + 1) * lda; + ao3 = a + posY + (posX + 2) * lda; + ao4 = a + posY + (posX + 3) * lda; + ao5 = a + posY + (posX + 4) * lda; + ao6 = a + posY + (posX + 5) * lda; + } + + i = (m / 6); + if (i > 0) { + do { + if (X < posY) { + ao1 += 6; + ao2 += 6; + ao3 += 6; + ao4 += 6; + ao5 += 6; + ao6 += 6; + + b += 36; + } else if (X > posY) { + for (ii = 0; ii < 6; ii++){ + + b[ 0] = *(ao1 + 0); + b[ 1] = *(ao1 + 1); + b[ 2] = *(ao1 + 2); + b[ 3] = *(ao1 + 3); + b[ 4] = *(ao1 + 4); + b[ 5] = *(ao1 + 5); + + ao1 += lda; + b += 6; + } + ao2 += 6 * lda; + ao3 += 6 * lda; + ao4 += 6 * lda; + ao5 += 6 * lda; + ao6 += 6 * lda; + + } else { #ifdef UNIT - data05 = *(ao2 + 0); - data09 = *(ao3 + 0); - data10 = *(ao3 + 1); - data13 = *(ao4 + 0); - data14 = *(ao4 + 1); - data15 = *(ao4 + 2); - - b[ 0] = ONE; - b[ 1] = ZERO; - b[ 2] = ZERO; - b[ 3] = ZERO; - - b[ 4] = data05; - b[ 5] = ONE; - b[ 6] = ZERO; - b[ 7] = ZERO; - - b[ 8] = data09; - b[ 9] = data10; - b[10] = ONE; - b[11] = ZERO; - - b[12] = data13; - b[13] = data14; - b[14] = data15; - b[15] = ONE; + b[ 0] = ONE; #else - data01 = *(ao1 + 0); - data05 = *(ao2 + 0); - data06 = *(ao2 + 1); - data09 = *(ao3 + 0); - data10 = *(ao3 + 1); - data11 = *(ao3 + 2); - data13 = *(ao4 + 0); - data14 = *(ao4 + 1); - data15 = *(ao4 + 2); - data16 = *(ao4 + 3); - - b[ 0] = data01; - b[ 1] = ZERO; - b[ 2] = ZERO; - b[ 3] = ZERO; - - b[ 4] = data05; - b[ 5] = data06; - b[ 6] = ZERO; - b[ 7] = ZERO; - - b[ 8] = data09; - b[ 9] = data10; - b[10] = data11; - b[11] = ZERO; - - b[12] = data13; - b[13] = data14; - b[14] = data15; - b[15] = data16; + b[ 0] = *(ao1 + 0); #endif + b[ 1] = ZERO; + b[ 2] = ZERO; + b[ 3] = ZERO; + b[ 4] = ZERO; + b[ 5] = ZERO; - ao1 += 4 * lda; - ao2 += 4 * lda; - ao3 += 4 * lda; - ao4 += 4 * lda; - - b += 16; - } - - X += 4; - i --; - } while (i > 0); - } - - i = (m & 3); - if (i) { - - if (X < posY) { - - if (m & 2) { - ao1 += 2; - ao2 += 2; - ao3 += 2; - ao4 += 2; - b += 8; - } - - if (m & 1) { - ao1 += 1; - ao2 += 1; - ao3 += 1; - ao4 += 1; - b += 4; - } - - } else - if (X > posY) { - if (m & 2) { - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); - data03 = *(ao1 + 2); - data04 = *(ao1 + 3); - data05 = *(ao2 + 0); - data06 = *(ao2 + 1); - data07 = *(ao2 + 2); - data08 = *(ao2 + 3); - - b[ 0] = data01; - b[ 1] = data02; - b[ 2] = data03; - b[ 3] = data04; - b[ 4] = data05; - b[ 5] = data06; - b[ 6] = data07; - b[ 7] = data08; - - ao1 += 2 * lda; - ao2 += 2 * lda; - b += 8; - } - - if (m & 1) { - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); - data03 = *(ao1 + 2); - data04 = *(ao1 + 3); - - b[ 0] = data01; - b[ 1] = data02; - b[ 2] = data03; - b[ 3] = data04; - - ao1 += lda; - b += 4; - } - - } else { + b[ 6] = *(ao2 + 0); +#ifdef UNIT + b[ 7] = ONE; +#else + b[ 7] = *(ao2 + 1); +#endif + b[ 8] = ZERO; + b[ 9] = ZERO; + b[10] = ZERO; + b[11] = ZERO; + + b[12] = *(ao3 + 0); + b[13] = *(ao3 + 1); +#ifdef UNIT + b[14] = ONE; +#else + b[14] = *(ao3 + 2); +#endif + b[15] = ZERO; + b[16] = ZERO; + b[17] = ZERO; + + b[18] = *(ao4 + 0); + b[19] = *(ao4 + 1); + b[20] = *(ao4 + 2); +#ifdef UNIT + b[21] = ONE; +#else + b[21] = *(ao4 + 3); +#endif + b[22] = ZERO; + b[23] = ZERO; + + b[24] = *(ao5 + 0); + b[25] = *(ao5 + 1); + b[26] = *(ao5 + 2); + b[27] = *(ao5 + 3); +#ifdef UNIT + b[28] = ONE; +#else + b[28] = *(ao5 + 4); +#endif + b[29] = ZERO; + + b[30] = *(ao6 + 0); + b[31] = *(ao6 + 1); + b[32] = *(ao6 + 2); + b[33] = *(ao6 + 3); + b[34] = *(ao6 + 4); +#ifdef UNIT + b[35] = ONE; +#else + b[35] = *(ao6 + 5); +#endif + + ao1 += 6 * lda; + ao2 += 6 * lda; + ao3 += 6 * lda; + ao4 += 6 * lda; + ao5 += 6 * lda; + ao6 += 6 * lda; + + b += 36; + } + + X += 6; + i --; + } while (i > 0); + } + + i = m % 6; + if (i > 0) { + if (X < posY) { + + ao1 += i; + ao2 += i; + ao3 += i; + ao4 += i; + ao5 += i; + ao6 += i; + b += 6 * i; + + } else if (X > posY) { + for (ii = 0; ii < i; ii++){ + + b[ 0] = *(ao1 + 0); + b[ 1] = *(ao1 + 1); + b[ 2] = *(ao1 + 2); + b[ 3] = *(ao1 + 3); + b[ 4] = *(ao1 + 4); + b[ 5] = *(ao1 + 5); + + ao1 += lda; + ao2 += lda; + ao3 += lda; + ao4 += lda; + ao5 += lda; + ao6 += lda; + b += 6; + } + } else { + +#ifdef UNIT + b[ 0] = ONE; +#else + b[ 0] = *(ao1 + 0); +#endif + b[ 1] = ZERO; + b[ 2] = ZERO; + b[ 3] = ZERO; + b[ 4] = ZERO; + b[ 5] = ZERO; + + if (i >= 2) { + b[ 0] = *(ao2 + 0); +#ifdef UNIT + b[ 1] = ONE; +#else + b[ 1] = *(ao2 + 1); +#endif + b[ 2] = ZERO; + b[ 3] = ZERO; + b[ 4] = ZERO; + b[ 5] = ZERO; + b += 6; + } + + if (i >= 3) { + b[ 0] = *(ao3 + 0); + b[ 1] = *(ao3 + 1); +#ifdef UNIT + b[ 2] = ONE; +#else + b[ 2] = *(ao3 + 2); +#endif + b[ 3] = ZERO; + b[ 4] = ZERO; + b[ 5] = ZERO; + b += 6; + } + + if (i >= 4) { + b[ 0] = *(ao4 + 0); + b[ 1] = *(ao4 + 1); + b[ 2] = *(ao4 + 2); +#ifdef UNIT + b[ 3] = ONE; +#else + b[ 3] = *(ao4 + 3); +#endif + b[ 4] = ZERO; + b[ 5] = ZERO; + b += 6; + } + + if (i >= 5) { + b[ 0] = *(ao5 + 0); + b[ 1] = *(ao5 + 1); + b[ 2] = *(ao5 + 2); + b[ 3] = *(ao5 + 3); +#ifdef UNIT + b[ 4] = ONE; +#else + b[ 4] = *(ao5 + 4); +#endif + b[ 5] = ZERO; + b += 6; + } + } + } + + posY += 6; + js --; + } while (js > 0); + } /* End of main loop */ + + if ((n % 6) & 4){ + X = posX; + + if (posX <= posY) { + ao1 = a + posX + (posY + 0) * lda; + ao2 = a + posX + (posY + 1) * lda; + ao3 = a + posX + (posY + 2) * lda; + ao4 = a + posX + (posY + 3) * lda; + } else { + ao1 = a + posY + (posX + 0) * lda; + ao2 = a + posY + (posX + 1) * lda; + ao3 = a + posY + (posX + 2) * lda; + ao4 = a + posY + (posX + 3) * lda; + } + + i = (m >> 1); + if (i > 0) { + do { + if (X < posY) { + ao1 += 2; + ao2 += 2; + ao3 += 2; + ao4 += 2; + b += 8; + } else if (X > posY) { + for (ii = 0; ii < 2; ii++){ + + b[ 0] = *(ao1 + 0); + b[ 1] = *(ao1 + 1); + b[ 2] = *(ao1 + 2); + b[ 3] = *(ao1 + 3); + ao1 += lda; + b += 4; + } + + ao2 += 2 * lda; + ao3 += 2 * lda; + ao4 += 2 * lda; + + } else { +#ifdef UNIT + b[ 0] = ONE; +#else + b[ 0] = *(ao1 + 0); +#endif + b[ 1] = ZERO; + b[ 2] = ZERO; + b[ 3] = ZERO; + + b[ 4] = *(ao2 + 0); +#ifdef UNIT + b[ 5] = ONE; +#else + b[ 5] = *(ao2 + 1); +#endif + b[ 6] = ZERO; + b[ 7] = ZERO; + + b[ 8] = *(ao3 + 0); + b[ 9] = *(ao3 + 1); +#ifdef UNIT + b[ 10] = ONE; +#else + b[ 10] = *(ao3 + 2); +#endif + b[ 11] = ZERO; + + b[ 12] = *(ao4 + 0); + b[ 13] = *(ao4 + 1); + b[ 14] = *(ao4 + 2); +#ifdef UNIT + b[ 15] = ONE; +#else + b[ 15] = *(ao4 + 3); +#endif + + ao1 += 4 * lda; + ao2 += 4 * lda; + ao3 += 4 * lda; + ao4 += 4 * lda; + b += 16; + X += 4; + i -= 2; + continue; + } + + X += 2; + i --; + } while (i > 0); + } + + i = (m & 1); + if (i > 0) { + if (X < posY) { + ao1 += i; + ao2 += i; + ao3 += i; + ao4 += i; + b += 4 * i; + } else if (X > posY) { + for (ii = 0; ii < i; ii++){ + + b[ 0] = *(ao1 + 0); + b[ 1] = *(ao1 + 1); + b[ 2] = *(ao1 + 2); + b[ 3] = *(ao1 + 3); + ao1 += lda; + b += 4; + } + ao2 += lda; + ao3 += lda; + ao4 += lda; + } else { #ifdef UNIT - if (i >= 2) { - data05 = *(ao2 + 0); - } - - if (i >= 3) { - data09 = *(ao3 + 0); - data10 = *(ao3 + 1); - } - - b[ 0] = ONE; - b[ 1] = ZERO; - b[ 2] = ZERO; - b[ 3] = ZERO; - b += 4; - - if(i >= 2) { - b[ 0] = data05; - b[ 1] = ONE; - b[ 2] = ZERO; - b[ 3] = ZERO; - b += 4; - } - - if (i >= 3) { - b[ 0] = data09; - b[ 1] = data10; - b[ 2] = ONE; - b[ 3] = ZERO; - b += 4; - } + b[ 0] = ONE; #else - data01 = *(ao1 + 0); - - if (i >= 2) { - data05 = *(ao2 + 0); - data06 = *(ao2 + 1); - } - - if (i >= 3) { - data09 = *(ao3 + 0); - data10 = *(ao3 + 1); - data11 = *(ao3 + 2); - } - - b[ 0] = data01; - b[ 1] = ZERO; - b[ 2] = ZERO; - b[ 3] = ZERO; - b += 4; - - if(i >= 2) { - b[ 0] = data05; - b[ 1] = data06; - b[ 2] = ZERO; - b[ 3] = ZERO; - b += 4; - } - - if (i >= 3) { - b[ 0] = data09; - b[ 1] = data10; - b[ 2] = data11; - b[ 3] = ZERO; - b += 4; - } + b[ 0] = *(ao1 + 0); #endif - } - } - - posY += 4; - js --; - } while (js > 0); - } /* End of main loop */ - - if (n & 2){ - X = posX; - - if (posX <= posY) { - ao1 = a + posX + (posY + 0) * lda; - ao2 = a + posX + (posY + 1) * lda; - } else { - ao1 = a + posY + (posX + 0) * lda; - ao2 = a + posY + (posX + 1) * lda; - } - - i = (m >> 1); - if (i > 0) { - do { - if (X < posY) { - ao1 += 2; - ao2 += 2; - b += 4; - - } else - if (X > posY) { - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); - data05 = *(ao2 + 0); - data06 = *(ao2 + 1); - - b[ 0] = data01; - b[ 1] = data02; - b[ 2] = data05; - b[ 3] = data06; - - ao1 += 2 * lda; - ao2 += 2 * lda; - b += 4; - } else { + b[ 1] = ZERO; + b[ 2] = ZERO; + b[ 3] = ZERO; + b += 4; + } + } + posY += 4; + } + + if ((n % 6) & 2){ + X = posX; + + if (posX <= posY) { + ao1 = a + posX + (posY + 0) * lda; + ao2 = a + posX + (posY + 1) * lda; + } else { + ao1 = a + posY + (posX + 0) * lda; + ao2 = a + posY + (posX + 1) * lda; + } + + i = (m >> 1); + if (i > 0) { + do { + if (X < posY) { + ao1 += 2; + ao2 += 2; + b += 4; + + } else if (X > posY) { + data01 = *(ao1 + 0); + data02 = *(ao1 + 1); + data05 = *(ao2 + 0); + data06 = *(ao2 + 1); + + b[ 0] = data01; + b[ 1] = data02; + b[ 2] = data05; + b[ 3] = data06; + + ao1 += 2 * lda; + ao2 += 2 * lda; + b += 4; + } else { #ifdef UNIT - data05 = *(ao2 + 0); + data05 = *(ao2 + 0); - b[ 0] = ONE; - b[ 1] = ZERO; - b[ 2] = data05; - b[ 3] = ONE; + b[ 0] = ONE; + b[ 1] = ZERO; + b[ 2] = data05; + b[ 3] = ONE; #else - data01 = *(ao1 + 0); - data05 = *(ao2 + 0); - data06 = *(ao2 + 1); + data01 = *(ao1 + 0); + data05 = *(ao2 + 0); + data06 = *(ao2 + 1); - b[ 0] = data01; - b[ 1] = ZERO; - b[ 2] = data05; - b[ 3] = data06; + b[ 0] = data01; + b[ 1] = ZERO; + b[ 2] = data05; + b[ 3] = data06; #endif - ao1 += 2 * lda; - ao2 += 2 * lda; - b += 4; - } - - X += 2; - i --; - } while (i > 0); - } - - i = (m & 1); - if (i) { - - if (X < posY) { - ao1 += 2; - b += 2; - } else - if (X > posY) { - data01 = *(ao1 + 0); - data02 = *(ao1 + 1); - - b[ 0] = data01; - b[ 1] = data02; - - ao1 += lda; - b += 2; - } else { + ao1 += 2 * lda; + ao2 += 2 * lda; + b += 4; + } + + X += 2; + i --; + } while (i > 0); + } + + i = (m & 1); + if (i) { + + if (X < posY) { + ao1 += 2; + b += 2; + } else if (X > posY) { + data01 = *(ao1 + 0); + data02 = *(ao1 + 1); + + b[ 0] = data01; + b[ 1] = data02; + + ao1 += lda; + b += 2; + } else { #ifdef UNIT - b[ 0] = ONE; - b[ 1] = ZERO; + b[ 0] = ONE; + b[ 1] = ZERO; #else - data01 = *(ao1 + 0); + data01 = *(ao1 + 0); - b[ 0] = data01; - b[ 1] = ZERO; + b[ 0] = data01; + b[ 1] = ZERO; #endif - b += 2; - } - } - posY += 2; - } - - if (n & 1){ - X = posX; - - if (posX <= posY) { - ao1 = a + posX + (posY + 0) * lda; - } else { - ao1 = a + posY + (posX + 0) * lda; - } - - i = m; - if (m > 0) { - do { - - if (X < posY) { - b += 1; - ao1 += 1; - } else - if (X > posY) { - data01 = *(ao1 + 0); - b[ 0] = data01; - ao1 += lda; - b += 1; - } else { + b += 2; + } + } + posY += 2; + } + + if ((n % 6) & 1){ + X = posX; + + if (posX <= posY) { + ao1 = a + posX + (posY + 0) * lda; + } else { + ao1 = a + posY + (posX + 0) * lda; + } + + i = m; + if (m > 0) { + do { + if (X < posY) { + b += 1; + ao1 += 1; + } else if (X > posY) { + data01 = *(ao1 + 0); + b[ 0] = data01; + ao1 += lda; + b += 1; + } else { #ifdef UNIT - b[ 0] = ONE; + b[ 0] = ONE; #else - data01 = *(ao1 + 0); - b[ 0] = data01; + data01 = *(ao1 + 0); + b[ 0] = data01; #endif - ao1 += lda; - b += 1; - } + ao1 += lda; + b += 1; + } - X += 1; - i --; - } while (i > 0); - } - } + X += 1; + i --; + } while (i > 0); + } + } - return 0; + return 0; } diff --git a/kernel/generic/trsm_lncopy_6.c b/kernel/generic/trsm_lncopy_6.c index a37c50d1f8..b0cc7ba40c 100644 --- a/kernel/generic/trsm_lncopy_6.c +++ b/kernel/generic/trsm_lncopy_6.c @@ -49,22 +49,35 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT BLASLONG i, ii, j, jj; - FLOAT data01, data02, data03, data04, data05, data06, data07, data08; - FLOAT data09, data10, data11, data12, data13, data14, data15, data16; - FLOAT *a1, *a2, *a3, *a4; + FLOAT data01, data02, data03, data04, data05, data06; + FLOAT data09, data10, data11, data12, data13, data14; + FLOAT data17, data18, data19, data20, data21, data22; + FLOAT data25, data26, data27, data28, data29, data30; + FLOAT data33, data34, data35, data36, data37, data38; + FLOAT data41, data42, data43, data44, data45, data46; + + FLOAT *a1, *a2, *a3, *a4, *a5, *a6, *a7, *a8; jj = offset; - j = (n >> 2); + BLASLONG mmod6, nmod6; + mmod6 = m - (m/6)*6 ; + nmod6 = n - (n/6)*6 ; + + // j = (n >> 3); + j = (n / 6); while (j > 0){ a1 = a + 0 * lda; a2 = a + 1 * lda; a3 = a + 2 * lda; a4 = a + 3 * lda; + a5 = a + 4 * lda; + a6 = a + 5 * lda; - i = (m >> 2); ii = 0; + // i = (m >> 3); + i = (m / 6); while (i > 0) { if (ii == jj) { @@ -74,233 +87,562 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT data02 = *(a1 + 1); data03 = *(a1 + 2); data04 = *(a1 + 3); + data05 = *(a1 + 4); + data06 = *(a1 + 5); + +#ifndef UNIT + data10 = *(a2 + 1); +#endif + data11 = *(a2 + 2); + data12 = *(a2 + 3); + data13 = *(a2 + 4); + data14 = *(a2 + 5); #ifndef UNIT - data06 = *(a2 + 1); + data19 = *(a3 + 2); #endif - data07 = *(a2 + 2); - data08 = *(a2 + 3); + data20 = *(a3 + 3); + data21 = *(a3 + 4); + data22 = *(a3 + 5); #ifndef UNIT - data11 = *(a3 + 2); + data28 = *(a4 + 3); #endif - data12 = *(a3 + 3); + data29 = *(a4 + 4); + data30 = *(a4 + 5); #ifndef UNIT - data16 = *(a4 + 3); + data37 = *(a5 + 4); +#endif + data38 = *(a5 + 5); + +#ifndef UNIT + data46 = *(a6 + 5); #endif *(b + 0) = INV(data01); - *(b + 4) = data02; - *(b + 5) = INV(data06); + *(b + 6) = data02; + *(b + 7) = INV(data10); - *(b + 8) = data03; - *(b + 9) = data07; - *(b + 10) = INV(data11); + *(b + 12) = data03; + *(b + 13) = data11; + *(b + 14) = INV(data19); + + *(b + 18) = data04; + *(b + 19) = data12; + *(b + 20) = data20; + *(b + 21) = INV(data28); + + *(b + 24) = data05; + *(b + 25) = data13; + *(b + 26) = data21; + *(b + 27) = data29; + *(b + 28) = INV(data37); + + *(b + 30) = data06; + *(b + 31) = data14; + *(b + 32) = data22; + *(b + 33) = data30; + *(b + 34) = data38; + *(b + 35) = INV(data46); - *(b + 12) = data04; - *(b + 13) = data08; - *(b + 14) = data12; - *(b + 15) = INV(data16); } if (ii > jj) { + data01 = *(a1 + 0); + data02 = *(a1 + 1); + data03 = *(a1 + 2); + data04 = *(a1 + 3); + data05 = *(a1 + 4); + data06 = *(a1 + 5); + + data09 = *(a2 + 0); + data10 = *(a2 + 1); + data11 = *(a2 + 2); + data12 = *(a2 + 3); + data13 = *(a2 + 4); + data14 = *(a2 + 5); + + data17 = *(a3 + 0); + data18 = *(a3 + 1); + data19 = *(a3 + 2); + data20 = *(a3 + 3); + data21 = *(a3 + 4); + data22 = *(a3 + 5); + + data25 = *(a4 + 0); + data26 = *(a4 + 1); + data27 = *(a4 + 2); + data28 = *(a4 + 3); + data29 = *(a4 + 4); + data30 = *(a4 + 5); + + data33 = *(a5 + 0); + data34 = *(a5 + 1); + data35 = *(a5 + 2); + data36 = *(a5 + 3); + data37 = *(a5 + 4); + data38 = *(a5 + 5); + + data41 = *(a6 + 0); + data42 = *(a6 + 1); + data43 = *(a6 + 2); + data44 = *(a6 + 3); + data45 = *(a6 + 4); + data46 = *(a6 + 5); + + *(b + 0) = data01; + *(b + 1) = data09; + *(b + 2) = data17; + *(b + 3) = data25; + *(b + 4) = data33; + *(b + 5) = data41; + + *(b + 6) = data02; + *(b + 7) = data10; + *(b + 8) = data18; + *(b + 9) = data26; + *(b + 10) = data34; + *(b + 11) = data42; + + *(b + 12) = data03; + *(b + 13) = data11; + *(b + 14) = data19; + *(b + 15) = data27; + *(b + 16) = data35; + *(b + 17) = data43; + + *(b + 18) = data04; + *(b + 19) = data12; + *(b + 20) = data20; + *(b + 21) = data28; + *(b + 22) = data36; + *(b + 23) = data44; + + *(b + 24) = data05; + *(b + 25) = data13; + *(b + 26) = data21; + *(b + 27) = data29; + *(b + 28) = data37; + *(b + 29) = data45; + + *(b + 30) = data06; + *(b + 31) = data14; + *(b + 32) = data22; + *(b + 33) = data30; + *(b + 34) = data38; + *(b + 35) = data46; + } + + a1 += 6; + a2 += 6; + a3 += 6; + a4 += 6; + a5 += 6; + a6 += 6; + a7 += 6; + a8 += 6; + b += 36; + + i --; + ii += 6; + } + + if (mmod6 & 4) { + if (ii == jj) { +#ifndef UNIT data01 = *(a1 + 0); +#endif data02 = *(a1 + 1); data03 = *(a1 + 2); data04 = *(a1 + 3); - data05 = *(a2 + 0); - data06 = *(a2 + 1); - data07 = *(a2 + 2); - data08 = *(a2 + 3); +#ifndef UNIT + data10 = *(a2 + 1); +#endif + data11 = *(a2 + 2); + data12 = *(a2 + 3); + + +#ifndef UNIT + data19 = *(a3 + 2); +#endif + data20 = *(a3 + 3); - data09 = *(a3 + 0); - data10 = *(a3 + 1); - data11 = *(a3 + 2); - data12 = *(a3 + 3); +#ifndef UNIT + data28 = *(a4 + 3); +#endif + + *(b + 0) = INV(data01); - data13 = *(a4 + 0); - data14 = *(a4 + 1); - data15 = *(a4 + 2); - data16 = *(a4 + 3); + *(b + 6) = data02; + *(b + 7) = INV(data10); + + *(b + 12) = data03; + *(b + 13) = data11; + *(b + 14) = INV(data19); + + *(b + 18) = data04; + *(b + 19) = data12; + *(b + 20) = data20; + *(b + 21) = INV(data28); + } + + if (ii > jj) { + data01 = *(a1 + 0); + data02 = *(a1 + 1); + data03 = *(a1 + 2); + data04 = *(a1 + 3); + data09 = *(a2 + 0); + data10 = *(a2 + 1); + data11 = *(a2 + 2); + data12 = *(a2 + 3); + + data17 = *(a3 + 0); + data18 = *(a3 + 1); + data19 = *(a3 + 2); + data20 = *(a3 + 3); + data25 = *(a4 + 0); + data26 = *(a4 + 1); + data27 = *(a4 + 2); + data28 = *(a4 + 3); + + data33 = *(a5 + 0); + data34 = *(a5 + 1); + data35 = *(a5 + 2); + data36 = *(a5 + 3); + data41 = *(a6 + 0); + data42 = *(a6 + 1); + data43 = *(a6 + 2); + data44 = *(a6 + 3); *(b + 0) = data01; - *(b + 1) = data05; - *(b + 2) = data09; - *(b + 3) = data13; - *(b + 4) = data02; - *(b + 5) = data06; - *(b + 6) = data10; - *(b + 7) = data14; + *(b + 1) = data09; + *(b + 2) = data17; + *(b + 3) = data25; + *(b + 4) = data33; + *(b + 5) = data41; + + *(b + 6) = data02; + *(b + 7) = data10; + *(b + 8) = data18; + *(b + 9) = data26; + *(b + 10) = data34; + *(b + 11) = data42; + + *(b + 12) = data03; + *(b + 13) = data11; + *(b + 14) = data19; + *(b + 15) = data27; + *(b + 16) = data35; + *(b + 17) = data43; + + *(b + 18) = data04; + *(b + 19) = data12; + *(b + 20) = data20; + *(b + 21) = data28; + *(b + 22) = data36; + *(b + 23) = data44; - *(b + 8) = data03; - *(b + 9) = data07; - *(b + 10) = data11; - *(b + 11) = data15; - *(b + 12) = data04; - *(b + 13) = data08; - *(b + 14) = data12; - *(b + 15) = data16; } a1 += 4; a2 += 4; a3 += 4; a4 += 4; - b += 16; - - i --; + a5 += 4; + a6 += 4; + a7 += 4; + a8 += 4; + b += 24; ii += 4; } - if ((m & 2) != 0) { - - if (ii== jj) { + if (mmod6 & 2) { + if (ii == jj) { #ifndef UNIT data01 = *(a1 + 0); #endif data02 = *(a1 + 1); #ifndef UNIT - data06 = *(a2 + 1); + data10 = *(a2 + 1); #endif *(b + 0) = INV(data01); - *(b + 4) = data02; - *(b + 5) = INV(data06); + *(b + 6) = data02; + *(b + 7) = INV(data10); } if (ii > jj) { data01 = *(a1 + 0); data02 = *(a1 + 1); - data03 = *(a2 + 0); - data04 = *(a2 + 1); - data05 = *(a3 + 0); - data06 = *(a3 + 1); - data07 = *(a4 + 0); - data08 = *(a4 + 1); + data09 = *(a2 + 0); + data10 = *(a2 + 1); + data17 = *(a3 + 0); + data18 = *(a3 + 1); + data25 = *(a4 + 0); + data26 = *(a4 + 1); + + data33 = *(a5 + 0); + data34 = *(a5 + 1); + data41 = *(a6 + 0); + data42 = *(a6 + 1); *(b + 0) = data01; - *(b + 1) = data03; - *(b + 2) = data05; - *(b + 3) = data07; - *(b + 4) = data02; - *(b + 5) = data04; - *(b + 6) = data06; - *(b + 7) = data08; + *(b + 1) = data09; + *(b + 2) = data17; + *(b + 3) = data25; + *(b + 4) = data33; + *(b + 5) = data41; + + *(b + 6) = data02; + *(b + 7) = data10; + *(b + 8) = data18; + *(b + 9) = data26; + *(b + 10) = data34; + *(b + 11) = data42; } a1 += 2; a2 += 2; a3 += 2; a4 += 2; - b += 8; - + a5 += 2; + a6 += 2; + a7 += 2; + a8 += 2; + b += 12; ii += 2; } - if ((m & 1) != 0) { + if (mmod6 & 1) { + if (ii == jj) { +#ifndef UNIT + data01 = *(a1 + 0); +#endif + + *(b + 0) = INV(data01); + } + + if (ii > jj) { + data01 = *(a1 + 0); + data09 = *(a2 + 0); + data17 = *(a3 + 0); + data25 = *(a4 + 0); + data33 = *(a5 + 0); + data41 = *(a6 + 0); + + *(b + 0) = data01; + *(b + 1) = data09; + *(b + 2) = data17; + *(b + 3) = data25; + *(b + 4) = data33; + *(b + 5) = data41; + } + b += 6; + } + + a += 6 * lda; + jj += 6; + j --; + } + + if (nmod6 & 4) { + a1 = a + 0 * lda; + a2 = a + 1 * lda; + a3 = a + 2 * lda; + a4 = a + 3 * lda; + + ii = 0; + i = (m >> 1); + while (i > 0) { + + if (ii == jj) { +#ifndef UNIT + data01 = *(a1 + 0); +#endif + data02 = *(a1 + 1); + data03 = *(a1 + 2); + data04 = *(a1 + 3); + +#ifndef UNIT + data10 = *(a2 + 1); +#endif + data11 = *(a2 + 2); + data12 = *(a2 + 3); + +#ifndef UNIT + data19 = *(a3 + 2); +#endif + data20 = *(a3 + 3); + +#ifndef UNIT + data28 = *(a4 + 3); +#endif + + *(b + 0) = INV(data01); + + *(b + 4) = data02; + *(b + 5) = INV(data10); + + *(b + 8) = data03; + *(b + 9) = data11; + *(b + 10) = INV(data19); + + *(b + 12) = data04; + *(b + 13) = data12; + *(b + 14) = data20; + *(b + 15) = INV(data28); + + a1 += 4; + a2 += 4; + a3 += 4; + a4 += 4; + b += 16; + + i -= 2; + ii += 4; + } + + else if (ii > jj) { + data01 = *(a1 + 0); + data02 = *(a1 + 1); + data09 = *(a2 + 0); + data10 = *(a2 + 1); + data17 = *(a3 + 0); + data18 = *(a3 + 1); + data25 = *(a4 + 0); + data26 = *(a4 + 1); + + *(b + 0) = data01; + *(b + 1) = data09; + *(b + 2) = data17; + *(b + 3) = data25; + *(b + 4) = data02; + *(b + 5) = data10; + *(b + 6) = data18; + *(b + 7) = data26; + + a1 += 2; + a2 += 2; + a3 += 2; + a4 += 2; + b += 8; + i -- ; + ii += 2; + } + + else { + a1 += 2; + a2 += 2; + a3 += 2; + a4 += 2; + b += 8; + i -- ; + ii += 2; + } + } - if (ii== jj) { + if (m & 1) { + if (ii == jj) { #ifndef UNIT data01 = *(a1 + 0); #endif *(b + 0) = INV(data01); } - if (ii > jj) { + if (ii > jj) { data01 = *(a1 + 0); - data02 = *(a2 + 0); - data03 = *(a3 + 0); - data04 = *(a4 + 0); + data09 = *(a2 + 0); + data17 = *(a3 + 0); + data25 = *(a4 + 0); *(b + 0) = data01; - *(b + 1) = data02; - *(b + 2) = data03; - *(b + 3) = data04; + *(b + 1) = data09; + *(b + 2) = data17; + *(b + 3) = data25; } - b += 4; + b += 4; } a += 4 * lda; jj += 4; - j --; } - if (n & 2) { + if (nmod6 & 2) { a1 = a + 0 * lda; a2 = a + 1 * lda; - i = (m >> 1); ii = 0; + i = (m >> 1); while (i > 0) { if (ii == jj) { - #ifndef UNIT data01 = *(a1 + 0); #endif data02 = *(a1 + 1); #ifndef UNIT - data04 = *(a2 + 1); + data10 = *(a2 + 1); #endif *(b + 0) = INV(data01); *(b + 2) = data02; - *(b + 3) = INV(data04); + *(b + 3) = INV(data10); } if (ii > jj) { data01 = *(a1 + 0); data02 = *(a1 + 1); - data03 = *(a2 + 0); - data04 = *(a2 + 1); + data09 = *(a2 + 0); + data10 = *(a2 + 1); *(b + 0) = data01; - *(b + 1) = data03; + *(b + 1) = data09; *(b + 2) = data02; - *(b + 3) = data04; + *(b + 3) = data10; } a1 += 2; a2 += 2; - b += 4; + b += 4; i --; ii += 2; } - if ((m & 1) != 0) { - - if (ii== jj) { + if (m & 1) { + if (ii == jj) { #ifndef UNIT data01 = *(a1 + 0); #endif *(b + 0) = INV(data01); } - if (ii > jj) { + if (ii > jj) { data01 = *(a1 + 0); - data02 = *(a2 + 0); + data09 = *(a2 + 0); + *(b + 0) = data01; - *(b + 1) = data02; + *(b + 1) = data09; } - b += 2; + b += 2; } + a += 2 * lda; jj += 2; } - if (n & 1) { + if (nmod6 & 1) { a1 = a + 0 * lda; - i = m; ii = 0; + i = m; while (i > 0) { if (ii == jj) { @@ -315,8 +657,9 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT *(b + 0) = data01; } - a1+= 1; - b += 1; + a1 += 1; + b += 1; + i --; ii += 1; } diff --git a/kernel/generic/trsm_ltcopy_6.c b/kernel/generic/trsm_ltcopy_6.c index 12043eb335..9cda3d72ff 100644 --- a/kernel/generic/trsm_ltcopy_6.c +++ b/kernel/generic/trsm_ltcopy_6.c @@ -49,22 +49,35 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT BLASLONG i, ii, j, jj; - FLOAT data01, data02, data03, data04, data05, data06, data07, data08; - FLOAT data09, data10, data11, data12, data13, data14, data15, data16; - FLOAT *a1, *a2, *a3, *a4; + FLOAT data01, data02, data03, data04, data05, data06; + FLOAT data09, data10, data11, data12, data13, data14; + FLOAT data17, data18, data19, data20, data21, data22; + FLOAT data25, data26, data27, data28, data29, data30; + FLOAT data33, data34, data35, data36, data37, data38; + FLOAT data41, data42, data43, data44, data45, data46; + + FLOAT *a1, *a2, *a3, *a4, *a5, *a6, *a7, *a8; jj = offset; - j = (n >> 2); + BLASLONG mmod6, nmod6, k; + mmod6 = m - (m/6)*6 ; + nmod6 = n - (n/6)*6 ; + + // j = (n >> 3); + j = (n / 6); while (j > 0){ a1 = a + 0 * lda; a2 = a + 1 * lda; a3 = a + 2 * lda; a4 = a + 3 * lda; + a5 = a + 4 * lda; + a6 = a + 5 * lda; - i = (m >> 2); ii = 0; + // i = (m >> 3); + i = (m / 6); while (i > 0) { if (ii == jj) { @@ -75,35 +88,65 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT data02 = *(a1 + 1); data03 = *(a1 + 2); data04 = *(a1 + 3); + data05 = *(a1 + 4); + data06 = *(a1 + 5); #ifndef UNIT - data06 = *(a2 + 1); + data10 = *(a2 + 1); #endif - data07 = *(a2 + 2); - data08 = *(a2 + 3); + data11 = *(a2 + 2); + data12 = *(a2 + 3); + data13 = *(a2 + 4); + data14 = *(a2 + 5); #ifndef UNIT - data11 = *(a3 + 2); + data19 = *(a3 + 2); #endif - data12 = *(a3 + 3); + data20 = *(a3 + 3); + data21 = *(a3 + 4); + data22 = *(a3 + 5); #ifndef UNIT - data16 = *(a4 + 3); + data28 = *(a4 + 3); +#endif + data29 = *(a4 + 4); + data30 = *(a4 + 5); + +#ifndef UNIT + data37 = *(a5 + 4); +#endif + data38 = *(a5 + 5); + +#ifndef UNIT + data46 = *(a6 + 5); #endif *(b + 0) = INV(data01); *(b + 1) = data02; *(b + 2) = data03; *(b + 3) = data04; + *(b + 4) = data05; + *(b + 5) = data06; - *(b + 5) = INV(data06); - *(b + 6) = data07; - *(b + 7) = data08; + *(b + 7) = INV(data10); + *(b + 8) = data11; + *(b + 9) = data12; + *(b + 10) = data13; + *(b + 11) = data14; - *(b + 10) = INV(data11); - *(b + 11) = data12; + *(b + 14) = INV(data19); + *(b + 15) = data20; + *(b + 16) = data21; + *(b + 17) = data22; - *(b + 15) = INV(data16); + *(b + 21) = INV(data28); + *(b + 22) = data29; + *(b + 23) = data30; + + *(b + 28) = INV(data37); + *(b + 29) = data38; + + *(b + 35) = INV(data46); } if (ii < jj) { @@ -111,21 +154,182 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT data02 = *(a1 + 1); data03 = *(a1 + 2); data04 = *(a1 + 3); + data05 = *(a1 + 4); + data06 = *(a1 + 5); + + data09 = *(a2 + 0); + data10 = *(a2 + 1); + data11 = *(a2 + 2); + data12 = *(a2 + 3); + data13 = *(a2 + 4); + data14 = *(a2 + 5); + + data17 = *(a3 + 0); + data18 = *(a3 + 1); + data19 = *(a3 + 2); + data20 = *(a3 + 3); + data21 = *(a3 + 4); + data22 = *(a3 + 5); + + data25 = *(a4 + 0); + data26 = *(a4 + 1); + data27 = *(a4 + 2); + data28 = *(a4 + 3); + data29 = *(a4 + 4); + data30 = *(a4 + 5); + + data33 = *(a5 + 0); + data34 = *(a5 + 1); + data35 = *(a5 + 2); + data36 = *(a5 + 3); + data37 = *(a5 + 4); + data38 = *(a5 + 5); + + data41 = *(a6 + 0); + data42 = *(a6 + 1); + data43 = *(a6 + 2); + data44 = *(a6 + 3); + data45 = *(a6 + 4); + data46 = *(a6 + 5); + + *(b + 0) = data01; + *(b + 1) = data02; + *(b + 2) = data03; + *(b + 3) = data04; + *(b + 4) = data05; + *(b + 5) = data06; + *(b + 6) = data09; + *(b + 7) = data10; + *(b + 8) = data11; + *(b + 9) = data12; + *(b + 10) = data13; + *(b + 11) = data14; + + *(b + 12) = data17; + *(b + 13) = data18; + *(b + 14) = data19; + *(b + 15) = data20; + *(b + 16) = data21; + *(b + 17) = data22; + *(b + 18) = data25; + *(b + 19) = data26; + *(b + 20) = data27; + *(b + 21) = data28; + *(b + 22) = data29; + *(b + 23) = data30; + + *(b + 24) = data33; + *(b + 25) = data34; + *(b + 26) = data35; + *(b + 27) = data36; + *(b + 28) = data37; + *(b + 29) = data38; + *(b + 30) = data41; + *(b + 31) = data42; + *(b + 32) = data43; + *(b + 33) = data44; + *(b + 34) = data45; + *(b + 35) = data46; + } - data05 = *(a2 + 0); - data06 = *(a2 + 1); - data07 = *(a2 + 2); - data08 = *(a2 + 3); + a1 += 6 * lda; + a2 += 6 * lda; + a3 += 6 * lda; + a4 += 6 * lda; + a5 += 6 * lda; + a6 += 6 * lda; + a7 += 6 * lda; + a8 += 6 * lda; + b += 36; - data09 = *(a3 + 0); - data10 = *(a3 + 1); - data11 = *(a3 + 2); - data12 = *(a3 + 3); + i --; + ii += 6; + } - data13 = *(a4 + 0); - data14 = *(a4 + 1); - data15 = *(a4 + 2); - data16 = *(a4 + 3); + if (mmod6 & 4) { + if (ii == jj) { + +#ifndef UNIT + data01 = *(a1 + 0); +#endif + data02 = *(a1 + 1); + data03 = *(a1 + 2); + data04 = *(a1 + 3); + data05 = *(a1 + 4); + data06 = *(a1 + 5); + +#ifndef UNIT + data10 = *(a2 + 1); +#endif + data11 = *(a2 + 2); + data12 = *(a2 + 3); + data13 = *(a2 + 4); + data14 = *(a2 + 5); + +#ifndef UNIT + data19 = *(a3 + 2); +#endif + data20 = *(a3 + 3); + data21 = *(a3 + 4); + data22 = *(a3 + 5); + +#ifndef UNIT + data28 = *(a4 + 3); +#endif + data29 = *(a4 + 4); + data30 = *(a4 + 5); + + *(b + 0) = INV(data01); + *(b + 1) = data02; + *(b + 2) = data03; + *(b + 3) = data04; + *(b + 4) = data05; + *(b + 5) = data06; + + *(b + 7) = INV(data10); + *(b + 8) = data11; + *(b + 9) = data12; + *(b + 10) = data13; + *(b + 11) = data14; + + *(b + 14) = INV(data19); + *(b + 15) = data20; + *(b + 16) = data21; + *(b + 17) = data22; + + *(b + 21) = INV(data28); + *(b + 22) = data29; + *(b + 23) = data30; + } + + if (ii < jj) { + data01 = *(a1 + 0); + data02 = *(a1 + 1); + data03 = *(a1 + 2); + data04 = *(a1 + 3); + data05 = *(a1 + 4); + data06 = *(a1 + 5); + + data09 = *(a2 + 0); + data10 = *(a2 + 1); + data11 = *(a2 + 2); + data12 = *(a2 + 3); + data13 = *(a2 + 4); + data14 = *(a2 + 5); + + data17 = *(a3 + 0); + data18 = *(a3 + 1); + data19 = *(a3 + 2); + data20 = *(a3 + 3); + data21 = *(a3 + 4); + data22 = *(a3 + 5); + + data25 = *(a4 + 0); + data26 = *(a4 + 1); + data27 = *(a4 + 2); + data28 = *(a4 + 3); + data29 = *(a4 + 4); + data30 = *(a4 + 5); *(b + 0) = data01; *(b + 1) = data02; @@ -133,32 +337,38 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT *(b + 3) = data04; *(b + 4) = data05; *(b + 5) = data06; - *(b + 6) = data07; - *(b + 7) = data08; - - *(b + 8) = data09; - *(b + 9) = data10; - *(b + 10) = data11; - *(b + 11) = data12; - *(b + 12) = data13; - *(b + 13) = data14; - *(b + 14) = data15; - *(b + 15) = data16; + *(b + 6) = data09; + *(b + 7) = data10; + *(b + 8) = data11; + *(b + 9) = data12; + *(b + 10) = data13; + *(b + 11) = data14; + + *(b + 12) = data17; + *(b + 13) = data18; + *(b + 14) = data19; + *(b + 15) = data20; + *(b + 16) = data21; + *(b + 17) = data22; + *(b + 18) = data25; + *(b + 19) = data26; + *(b + 20) = data27; + *(b + 21) = data28; + *(b + 22) = data29; + *(b + 23) = data30; } a1 += 4 * lda; a2 += 4 * lda; - a3 += 4 * lda; - a4 += 4 * lda; - b += 16; + /* a3 += 4 * lda; + a4 += 4 * lda; */ + b += 24; - i --; ii += 4; } - if ((m & 2) != 0) { - - if (ii== jj) { + if (mmod6 & 2) { + if (ii == jj) { #ifndef UNIT data01 = *(a1 + 0); @@ -166,22 +376,29 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT data02 = *(a1 + 1); data03 = *(a1 + 2); data04 = *(a1 + 3); + data05 = *(a1 + 4); + data06 = *(a1 + 5); #ifndef UNIT - data06 = *(a2 + 1); + data10 = *(a2 + 1); #endif - data07 = *(a2 + 2); - data08 = *(a2 + 3); + data11 = *(a2 + 2); + data12 = *(a2 + 3); + data13 = *(a2 + 4); + data14 = *(a2 + 5); *(b + 0) = INV(data01); *(b + 1) = data02; *(b + 2) = data03; *(b + 3) = data04; + *(b + 4) = data05; + *(b + 5) = data06; - *(b + 5) = INV(data06); - *(b + 6) = data07; - *(b + 7) = data08; - + *(b + 7) = INV(data10); + *(b + 8) = data11; + *(b + 9) = data12; + *(b + 10) = data13; + *(b + 11) = data14; } if (ii < jj) { @@ -189,11 +406,15 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT data02 = *(a1 + 1); data03 = *(a1 + 2); data04 = *(a1 + 3); + data05 = *(a1 + 4); + data06 = *(a1 + 5); - data05 = *(a2 + 0); - data06 = *(a2 + 1); - data07 = *(a2 + 2); - data08 = *(a2 + 3); + data09 = *(a2 + 0); + data10 = *(a2 + 1); + data11 = *(a2 + 2); + data12 = *(a2 + 3); + data13 = *(a2 + 4); + data14 = *(a2 + 5); *(b + 0) = data01; *(b + 1) = data02; @@ -201,20 +422,23 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT *(b + 3) = data04; *(b + 4) = data05; *(b + 5) = data06; - *(b + 6) = data07; - *(b + 7) = data08; + *(b + 6) = data09; + *(b + 7) = data10; + *(b + 8) = data11; + *(b + 9) = data12; + *(b + 10) = data13; + *(b + 11) = data14; } a1 += 2 * lda; - a2 += 2 * lda; - b += 8; + // a2 += 2 * lda; + b += 12; ii += 2; } - if ((m & 1) != 0) { - - if (ii== jj) { + if (mmod6 & 1) { + if (ii == jj) { #ifndef UNIT data01 = *(a1 + 0); @@ -222,38 +446,78 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT data02 = *(a1 + 1); data03 = *(a1 + 2); data04 = *(a1 + 3); + data05 = *(a1 + 4); + data06 = *(a1 + 5); *(b + 0) = INV(data01); *(b + 1) = data02; *(b + 2) = data03; *(b + 3) = data04; + *(b + 4) = data05; + *(b + 5) = data06; } - if (ii < jj) { + if (ii < jj) { data01 = *(a1 + 0); data02 = *(a1 + 1); data03 = *(a1 + 2); data04 = *(a1 + 3); + data05 = *(a1 + 4); + data06 = *(a1 + 5); *(b + 0) = data01; *(b + 1) = data02; *(b + 2) = data03; *(b + 3) = data04; + *(b + 4) = data05; + *(b + 5) = data06; } - b += 4; + b += 6; } + a += 6; + jj += 6; + j --; + } + if (nmod6 & 4) { + + a1 = a; a += 4; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 4)) { + *(b + ii - jj) = INV(*(a1 + ii - jj)); + + for (k = ii - jj + 1; k < 4; k ++) { + *(b + k) = *(a1 + k); + } + + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a1 + 1); + *(b + 2) = *(a1 + 2); + *(b + 3) = *(a1 + 3); + } + + b += 4; + a1 += lda; + ii ++; + } + jj += 4; - j --; } - if (n & 2) { + if (nmod6 & 2) { + a1 = a + 0 * lda; a2 = a + 1 * lda; - i = (m >> 1); ii = 0; + i = (m >> 1); while (i > 0) { if (ii == jj) { @@ -264,25 +528,24 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT data02 = *(a1 + 1); #ifndef UNIT - data04 = *(a2 + 1); + data10 = *(a2 + 1); #endif *(b + 0) = INV(data01); *(b + 1) = data02; - - *(b + 3) = INV(data04); + *(b + 3) = INV(data10); } if (ii < jj) { data01 = *(a1 + 0); data02 = *(a1 + 1); - data03 = *(a2 + 0); - data04 = *(a2 + 1); + data09 = *(a2 + 0); + data10 = *(a2 + 1); *(b + 0) = data01; *(b + 1) = data02; - *(b + 2) = data03; - *(b + 3) = data04; + *(b + 2) = data09; + *(b + 3) = data10; } a1 += 2 * lda; @@ -293,19 +556,22 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT ii += 2; } - if ((m & 1) != 0) { - - if (ii== jj) { + if (m & 1) { + if (ii == jj) { #ifndef UNIT data01 = *(a1 + 0); #endif + // data02 = *(a1 + 1); + *(b + 0) = INV(data01); + // *(b + 1) = data02; } - if (ii < jj) { + if (ii < jj) { data01 = *(a1 + 0); data02 = *(a1 + 1); + *(b + 0) = data01; *(b + 1) = data02; } @@ -315,11 +581,12 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT jj += 2; } - if (n & 1) { + if (nmod6 & 1) { + a1 = a + 0 * lda; - i = m; ii = 0; + i = m; while (i > 0) { if (ii == jj) { @@ -334,12 +601,13 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT *(b + 0) = data01; } - a1 += 1 * lda; + a1 += lda; b += 1; i --; ii += 1; } + } return 0; diff --git a/kernel/generic/trsm_uncopy_6.c b/kernel/generic/trsm_uncopy_6.c index a1bb1e2034..e20773da47 100644 --- a/kernel/generic/trsm_uncopy_6.c +++ b/kernel/generic/trsm_uncopy_6.c @@ -36,7 +36,6 @@ /* or implied, of The University of Texas at Austin. */ /*********************************************************************/ -#include #include "common.h" #ifndef UNIT @@ -49,22 +48,38 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT BLASLONG i, ii, j, jj; - FLOAT data01, data02, data03, data04, data05, data06, data07, data08; - FLOAT data09, data10, data11, data12, data13, data14, data15, data16; - FLOAT *a1, *a2, *a3, *a4; + FLOAT data01, data02, data03, data04, data05, data06; + FLOAT data09, data10, data11, data12, data13, data14; + FLOAT data17, data18, data19, data20, data21, data22; + FLOAT data25, data26, data27, data28, data29, data30; + FLOAT data33, data34, data35, data36, data37, data38; + FLOAT data41, data42, data43, data44, data45, data46; + + FLOAT *a1, *a2, *a3, *a4, *a5, *a6, *a7, *a8; jj = offset; - j = (n >> 2); + BLASLONG mmod6, nmod6; + mmod6 = m - (m/6)*6 ; + nmod6 = n - (n/6)*6 ; + + // j = (n >> 3); + j = (n / 6); while (j > 0){ a1 = a + 0 * lda; a2 = a + 1 * lda; a3 = a + 2 * lda; a4 = a + 3 * lda; + a5 = a + 4 * lda; + a6 = a + 5 * lda; + // a7 = a + 6 * lda; + // a8 = a + 7 * lda; - i = (m >> 2); ii = 0; + + // i = (m >> 3); + i = (m / 6); while (i > 0) { if (ii == jj) { @@ -73,188 +88,729 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT data01 = *(a1 + 0); #endif - data05 = *(a2 + 0); + data09 = *(a2 + 0); #ifndef UNIT - data06 = *(a2 + 1); + data10 = *(a2 + 1); #endif - data09 = *(a3 + 0); - data10 = *(a3 + 1); + data17 = *(a3 + 0); + data18 = *(a3 + 1); #ifndef UNIT - data11 = *(a3 + 2); + data19 = *(a3 + 2); #endif - data13 = *(a4 + 0); - data14 = *(a4 + 1); - data15 = *(a4 + 2); + data25 = *(a4 + 0); + data26 = *(a4 + 1); + data27 = *(a4 + 2); #ifndef UNIT - data16 = *(a4 + 3); + data28 = *(a4 + 3); #endif - *(b + 0) = INV(data01); - *(b + 1) = data05; - *(b + 2) = data09; - *(b + 3) = data13; + data33 = *(a5 + 0); + data34 = *(a5 + 1); + data35 = *(a5 + 2); + data36 = *(a5 + 3); +#ifndef UNIT + data37 = *(a5 + 4); +#endif - *(b + 5) = INV(data06); - *(b + 6) = data10; - *(b + 7) = data14; + data41 = *(a6 + 0); + data42 = *(a6 + 1); + data43 = *(a6 + 2); + data44 = *(a6 + 3); + data45 = *(a6 + 4); +#ifndef UNIT + data46 = *(a6 + 5); +#endif - *(b + 10) = INV(data11); - *(b + 11) = data15; +// data49 = *(a7 + 0); +// data50 = *(a7 + 1); +// data51 = *(a7 + 2); +// data52 = *(a7 + 3); +// data53 = *(a7 + 4); +// data54 = *(a7 + 5); +// #ifndef UNIT +// data55 = *(a7 + 6); +// #endif +// +// data57 = *(a8 + 0); +// data58 = *(a8 + 1); +// data59 = *(a8 + 2); +// data60 = *(a8 + 3); +// data61 = *(a8 + 4); +// data62 = *(a8 + 5); +// data63 = *(a8 + 6); +// #ifndef UNIT +// data64 = *(a8 + 7); +// #endif - *(b + 15) = INV(data16); + *(b + 0) = INV(data01); + *(b + 1) = data09; + *(b + 2) = data17; + *(b + 3) = data25; + *(b + 4) = data33; + *(b + 5) = data41; + // *(b + 6) = data49; + // *(b + 7) = data57; + + *(b + 7) = INV(data10); + *(b + 8) = data18; + *(b + 9) = data26; + *(b + 10) = data34; + *(b + 11) = data42; + // *(b + 14) = data50; + // *(b + 15) = data58; + + *(b + 14) = INV(data19); + *(b + 15) = data27; + *(b + 16) = data35; + *(b + 17) = data43; + // *(b + 22) = data51; + // *(b + 23) = data59; + + *(b + 21) = INV(data28); + *(b + 22) = data36; + *(b + 23) = data44; + // *(b + 30) = data52; + // *(b + 31) = data60; + + *(b + 28) = INV(data37); + *(b + 29) = data45; + // *(b + 38) = data53; + // *(b + 39) = data61; + + *(b + 35) = INV(data46); + // *(b + 46) = data54; + // *(b + 47) = data62; + + // *(b + 54) = INV(data55); + // *(b + 55) = data63; + + // *(b + 63) = INV(data64); } if (ii < jj) { - data01 = *(a1 + 0); data02 = *(a1 + 1); data03 = *(a1 + 2); data04 = *(a1 + 3); + data05 = *(a1 + 4); + data06 = *(a1 + 5); + // data07 = *(a1 + 6); + // data08 = *(a1 + 7); + + data09 = *(a2 + 0); + data10 = *(a2 + 1); + data11 = *(a2 + 2); + data12 = *(a2 + 3); + data13 = *(a2 + 4); + data14 = *(a2 + 5); + // data15 = *(a2 + 6); + // data16 = *(a2 + 7); + + data17 = *(a3 + 0); + data18 = *(a3 + 1); + data19 = *(a3 + 2); + data20 = *(a3 + 3); + data21 = *(a3 + 4); + data22 = *(a3 + 5); + // data23 = *(a3 + 6); + // data24 = *(a3 + 7); + + data25 = *(a4 + 0); + data26 = *(a4 + 1); + data27 = *(a4 + 2); + data28 = *(a4 + 3); + data29 = *(a4 + 4); + data30 = *(a4 + 5); + // data31 = *(a4 + 6); + // data32 = *(a4 + 7); + + data33 = *(a5 + 0); + data34 = *(a5 + 1); + data35 = *(a5 + 2); + data36 = *(a5 + 3); + data37 = *(a5 + 4); + data38 = *(a5 + 5); + // data39 = *(a5 + 6); + // data40 = *(a5 + 7); + + data41 = *(a6 + 0); + data42 = *(a6 + 1); + data43 = *(a6 + 2); + data44 = *(a6 + 3); + data45 = *(a6 + 4); + data46 = *(a6 + 5); + // data47 = *(a6 + 6); + // data48 = *(a6 + 7); + + // data49 = *(a7 + 0); + // data50 = *(a7 + 1); + // data51 = *(a7 + 2); + // data52 = *(a7 + 3); + // data53 = *(a7 + 4); + // data54 = *(a7 + 5); + // data55 = *(a7 + 6); + // data56 = *(a7 + 7); + + // data57 = *(a8 + 0); + // data58 = *(a8 + 1); + // data59 = *(a8 + 2); + // data60 = *(a8 + 3); + // data61 = *(a8 + 4); + // data62 = *(a8 + 5); + // data63 = *(a8 + 6); + // data64 = *(a8 + 7); - data05 = *(a2 + 0); - data06 = *(a2 + 1); - data07 = *(a2 + 2); - data08 = *(a2 + 3); + *(b + 0) = data01; + *(b + 1) = data09; + *(b + 2) = data17; + *(b + 3) = data25; + *(b + 4) = data33; + *(b + 5) = data41; + // *(b + 6) = data49; + // *(b + 7) = data57; + + *(b + 6) = data02; + *(b + 7) = data10; + *(b + 8) = data18; + *(b + 9) = data26; + *(b + 10) = data34; + *(b + 11) = data42; + // *(b + 14) = data50; + // *(b + 15) = data58; + + *(b + 12) = data03; + *(b + 13) = data11; + *(b + 14) = data19; + *(b + 15) = data27; + *(b + 16) = data35; + *(b + 17) = data43; + // *(b + 22) = data51; + // *(b + 23) = data59; + + *(b + 18) = data04; + *(b + 19) = data12; + *(b + 20) = data20; + *(b + 21) = data28; + *(b + 22) = data36; + *(b + 23) = data44; + // *(b + 30) = data52; + // *(b + 31) = data60; + + *(b + 24) = data05; + *(b + 25) = data13; + *(b + 26) = data21; + *(b + 27) = data29; + *(b + 28) = data37; + *(b + 29) = data45; + // *(b + 38) = data53; + // *(b + 39) = data61; + + *(b + 30) = data06; + *(b + 31) = data14; + *(b + 32) = data22; + *(b + 33) = data30; + *(b + 34) = data38; + *(b + 35) = data46; + // *(b + 46) = data54; + // *(b + 47) = data62; + + // *(b + 48) = data07; + // *(b + 49) = data15; + // *(b + 50) = data23; + // *(b + 51) = data31; + // *(b + 52) = data39; + // *(b + 53) = data47; + // *(b + 54) = data55; + // *(b + 55) = data63; + + // *(b + 56) = data08; + // *(b + 57) = data16; + // *(b + 58) = data24; + // *(b + 59) = data32; + // *(b + 60) = data40; + // *(b + 61) = data48; + // *(b + 62) = data56; + // *(b + 63) = data64; + } + + a1 += 6; + a2 += 6; + a3 += 6; + a4 += 6; + a5 += 6; + a6 += 6; + // a7 += 6; + // a8 += 6; + b += 36; - data09 = *(a3 + 0); - data10 = *(a3 + 1); - data11 = *(a3 + 2); - data12 = *(a3 + 3); + i --; + ii += 6; + } - data13 = *(a4 + 0); - data14 = *(a4 + 1); - data15 = *(a4 + 2); - data16 = *(a4 + 3); + if (mmod6 & 4) { + if (ii == jj) { + +#ifndef UNIT + data01 = *(a1 + 0); +#endif + + data09 = *(a2 + 0); +#ifndef UNIT + data10 = *(a2 + 1); +#endif + + data17 = *(a3 + 0); + data18 = *(a3 + 1); +#ifndef UNIT + data19 = *(a3 + 2); +#endif + + data25 = *(a4 + 0); + data26 = *(a4 + 1); + data27 = *(a4 + 2); +#ifndef UNIT + data28 = *(a4 + 3); +#endif + + data33 = *(a5 + 0); + data34 = *(a5 + 1); + data35 = *(a5 + 2); + data36 = *(a5 + 3); + + data41 = *(a6 + 0); + data42 = *(a6 + 1); + data43 = *(a6 + 2); + data44 = *(a6 + 3); + + // data49 = *(a7 + 0); + // data50 = *(a7 + 1); + // data51 = *(a7 + 2); + // data52 = *(a7 + 3); + + // data57 = *(a8 + 0); + // data58 = *(a8 + 1); + // data59 = *(a8 + 2); + // data60 = *(a8 + 3); + + *(b + 0) = INV(data01); + *(b + 1) = data09; + *(b + 2) = data17; + *(b + 3) = data25; + *(b + 4) = data33; + *(b + 5) = data41; + // *(b + 6) = data49; + // *(b + 7) = data57; + + *(b + 7) = INV(data10); + *(b + 8) = data18; + *(b + 9) = data26; + *(b + 10) = data34; + *(b + 11) = data42; + // *(b + 14) = data50; + // *(b + 15) = data58; + + *(b + 14) = INV(data19); + *(b + 15) = data27; + *(b + 16) = data35; + *(b + 17) = data43; + // *(b + 22) = data51; + // *(b + 23) = data59; + + *(b + 21) = INV(data28); + *(b + 22) = data36; + *(b + 23) = data44; + // *(b + 30) = data52; + // *(b + 31) = data60; + + } + + if (ii < jj) { + data01 = *(a1 + 0); + data02 = *(a1 + 1); + data03 = *(a1 + 2); + data04 = *(a1 + 3); + data09 = *(a2 + 0); + data10 = *(a2 + 1); + data11 = *(a2 + 2); + data12 = *(a2 + 3); + + data17 = *(a3 + 0); + data18 = *(a3 + 1); + data19 = *(a3 + 2); + data20 = *(a3 + 3); + data25 = *(a4 + 0); + data26 = *(a4 + 1); + data27 = *(a4 + 2); + data28 = *(a4 + 3); + + data33 = *(a5 + 0); + data34 = *(a5 + 1); + data35 = *(a5 + 2); + data36 = *(a5 + 3); + data41 = *(a6 + 0); + data42 = *(a6 + 1); + data43 = *(a6 + 2); + data44 = *(a6 + 3); + + // data49 = *(a7 + 0); + // data50 = *(a7 + 1); + // data51 = *(a7 + 2); + // data52 = *(a7 + 3); + // data57 = *(a8 + 0); + // data58 = *(a8 + 1); + // data59 = *(a8 + 2); + // data60 = *(a8 + 3); *(b + 0) = data01; - *(b + 1) = data05; - *(b + 2) = data09; - *(b + 3) = data13; - *(b + 4) = data02; - *(b + 5) = data06; - *(b + 6) = data10; - *(b + 7) = data14; - - *(b + 8) = data03; - *(b + 9) = data07; - *(b + 10) = data11; - *(b + 11) = data15; - *(b + 12) = data04; - *(b + 13) = data08; - *(b + 14) = data12; - *(b + 15) = data16; + *(b + 1) = data09; + *(b + 2) = data17; + *(b + 3) = data25; + *(b + 4) = data33; + *(b + 5) = data41; + // *(b + 6) = data49; + // *(b + 7) = data57; + + *(b + 6) = data02; + *(b + 7) = data10; + *(b + 8) = data18; + *(b + 9) = data26; + *(b + 10) = data34; + *(b + 11) = data42; + // *(b + 14) = data50; + // *(b + 15) = data58; + + *(b + 12) = data03; + *(b + 13) = data11; + *(b + 14) = data19; + *(b + 15) = data27; + *(b + 16) = data35; + *(b + 17) = data43; + // *(b + 22) = data51; + // *(b + 23) = data59; + + *(b + 18) = data04; + *(b + 19) = data12; + *(b + 20) = data20; + *(b + 21) = data28; + *(b + 22) = data36; + *(b + 23) = data44; + // *(b + 30) = data52; + // *(b + 31) = data60; } a1 += 4; a2 += 4; a3 += 4; a4 += 4; - b += 16; - - i --; + a5 += 4; + a6 += 4; + // a7 += 4; + // a8 += 4; + b += 24; ii += 4; } - if ((m & 2) != 0) { + if (mmod6 & 2) { + if (ii == jj) { + +#ifndef UNIT + data01 = *(a1 + 0); +#endif + data09 = *(a2 + 0); +#ifndef UNIT + data10 = *(a2 + 1); +#endif + + data17 = *(a3 + 0); + data18 = *(a3 + 1); + data25 = *(a4 + 0); + data26 = *(a4 + 1); - if (ii== jj) { + data33 = *(a5 + 0); + data34 = *(a5 + 1); + data41 = *(a6 + 0); + data42 = *(a6 + 1); + + // data49 = *(a7 + 0); + // data50 = *(a7 + 1); + // data57 = *(a8 + 0); + // data58 = *(a8 + 1); + + *(b + 0) = INV(data01); + *(b + 1) = data09; + *(b + 2) = data17; + *(b + 3) = data25; + *(b + 4) = data33; + *(b + 5) = data41; + // *(b + 6) = data49; + // *(b + 7) = data57; + + *(b + 7) = INV(data10); + *(b + 8) = data18; + *(b + 9) = data26; + *(b + 10) = data34; + *(b + 11) = data42; + // *(b + 14) = data50; + // *(b + 15) = data58; + } + + if (ii < jj) { + data01 = *(a1 + 0); + data02 = *(a1 + 1); + data09 = *(a2 + 0); + data10 = *(a2 + 1); + data17 = *(a3 + 0); + data18 = *(a3 + 1); + data25 = *(a4 + 0); + data26 = *(a4 + 1); + + data33 = *(a5 + 0); + data34 = *(a5 + 1); + data41 = *(a6 + 0); + data42 = *(a6 + 1); + // data49 = *(a7 + 0); + // data50 = *(a7 + 1); + // data57 = *(a8 + 0); + // data58 = *(a8 + 1); + + *(b + 0) = data01; + *(b + 1) = data09; + *(b + 2) = data17; + *(b + 3) = data25; + *(b + 4) = data33; + *(b + 5) = data41; + // *(b + 6) = data49; + // *(b + 7) = data57; + + *(b + 6) = data02; + *(b + 7) = data10; + *(b + 8) = data18; + *(b + 9) = data26; + *(b + 10) = data34; + *(b + 11) = data42; + // *(b + 14) = data50; + // *(b + 15) = data58; + } + + a1 += 2; + a2 += 2; + a3 += 2; + a4 += 2; + a5 += 2; + a6 += 2; + a7 += 2; + a8 += 2; + b += 12; + ii += 2; + } + + if (mmod6 & 1) { + if (ii == jj) { #ifndef UNIT data01 = *(a1 + 0); #endif + data09 = *(a2 + 0); + data17 = *(a3 + 0); + data25 = *(a4 + 0); + data33 = *(a5 + 0); + data41 = *(a6 + 0); + // data49 = *(a7 + 0); + // data57 = *(a8 + 0); - data05 = *(a2 + 0); + *(b + 0) = INV(data01); + *(b + 1) = data09; + *(b + 2) = data17; + *(b + 3) = data25; + *(b + 4) = data33; + *(b + 5) = data41; + // *(b + 6) = data49; + // *(b + 7) = data57; + } + + if (ii < jj) { + data01 = *(a1 + 0); + // data02 = *(a1 + 1); + data09 = *(a2 + 0); + // data10 = *(a2 + 1); + data17 = *(a3 + 0); + // data18 = *(a3 + 1); + data25 = *(a4 + 0); + // data26 = *(a4 + 1); + + // // data33 = *(a5 + 0); + // data34 = *(a5 + 1); + // // data41 = *(a6 + 0); + // data42 = *(a6 + 1); + // data49 = *(a7 + 0); + // data50 = *(a7 + 1); + // data57 = *(a8 + 0); + // data58 = *(a8 + 1); + + *(b + 0) = data01; + *(b + 1) = data09; + *(b + 2) = data17; + *(b + 3) = data25; + *(b + 4) = data33; + *(b + 5) = data41; + // *(b + 6) = data49; + // *(b + 7) = data57; + } + b += 6; + // ii += 1; + } + + a += 6 * lda; + jj += 6; + j --; + } + + + if (nmod6 & 4) { + a1 = a + 0 * lda; + a2 = a + 1 * lda; + a3 = a + 2 * lda; + a4 = a + 3 * lda; + + ii = 0; + + i = (m >> 1); + while (i > 0) { + + if (ii == jj) { + +#ifndef UNIT + data01 = *(a1 + 0); +#endif + + data09 = *(a2 + 0); #ifndef UNIT - data06 = *(a2 + 1); + data10 = *(a2 + 1); #endif - data09 = *(a3 + 0); - data10 = *(a3 + 1); + data17 = *(a3 + 0); + data18 = *(a3 + 1); +#ifndef UNIT + data19 = *(a3 + 2); +#endif - data13 = *(a4 + 0); - data14 = *(a4 + 1); + data25 = *(a4 + 0); + data26 = *(a4 + 1); + data27 = *(a4 + 2); +#ifndef UNIT + data28 = *(a4 + 3); +#endif *(b + 0) = INV(data01); - *(b + 1) = data05; - *(b + 2) = data09; - *(b + 3) = data13; + *(b + 1) = data09; + *(b + 2) = data17; + *(b + 3) = data25; + + *(b + 5) = INV(data10); + *(b + 6) = data18; + *(b + 7) = data26; + + *(b + 10) = INV(data19); + *(b + 11) = data27; - *(b + 5) = INV(data06); - *(b + 6) = data10; - *(b + 7) = data14; + *(b + 15) = INV(data28); + + a1 += 4; + a2 += 4; + a3 += 4; + a4 += 4; + b += 16; + + i -= 2; + ii += 4; } - if (ii < jj) { + else if (ii < jj) { data01 = *(a1 + 0); data02 = *(a1 + 1); - data03 = *(a2 + 0); - data04 = *(a2 + 1); - data05 = *(a3 + 0); - data06 = *(a3 + 1); - data07 = *(a4 + 0); - data08 = *(a4 + 1); + data09 = *(a2 + 0); + data10 = *(a2 + 1); + data17 = *(a3 + 0); + data18 = *(a3 + 1); + data25 = *(a4 + 0); + data26 = *(a4 + 1); *(b + 0) = data01; - *(b + 1) = data02; - *(b + 2) = data03; - *(b + 3) = data04; - *(b + 4) = data05; - *(b + 5) = data06; - *(b + 6) = data07; - *(b + 7) = data08; + *(b + 1) = data09; + *(b + 2) = data17; + *(b + 3) = data25; + *(b + 4) = data02; + *(b + 5) = data10; + *(b + 6) = data18; + *(b + 7) = data26; + + a1 += 2; + a2 += 2; + a3 += 2; + a4 += 2; + b += 8; + + i -- ; + ii += 2; } + else{ + a1 += 2; a2 += 2; - b += 8; + a3 += 2; + a4 += 2; + b += 8; + i -- ; ii += 2; } + } - if ((m & 1) != 0) { + if (m & 1) { + if (ii == jj) { - if (ii== jj) { #ifndef UNIT data01 = *(a1 + 0); #endif - - data05 = *(a2 + 0); - data09 = *(a3 + 0); - data13 = *(a4 + 0); + data09 = *(a2 + 0); + data17 = *(a3 + 0); + data25 = *(a4 + 0); *(b + 0) = INV(data01); - *(b + 1) = data05; - *(b + 2) = data09; - *(b + 3) = data13; + *(b + 1) = data09; + *(b + 2) = data17; + *(b + 3) = data25; } - if (ii < jj) { + if (ii < jj) { data01 = *(a1 + 0); - data02 = *(a2 + 0); - data03 = *(a3 + 0); - data04 = *(a4 + 0); + data09 = *(a2 + 0); + data17 = *(a3 + 0); + data25 = *(a4 + 0); *(b + 0) = data01; - *(b + 1) = data02; - *(b + 2) = data03; - *(b + 3) = data04; + *(b + 1) = data09; + *(b + 2) = data17; + *(b + 3) = data25; } b += 4; + // ii += 1; } - a += 4 * lda; + a += 4 * lda; jj += 4; - j --; } - if (n & 2) { + if (nmod6 & 2) { a1 = a + 0 * lda; a2 = a + 1 * lda; - i = (m >> 1); ii = 0; + + i = (m >> 1); while (i > 0) { if (ii == jj) { @@ -263,68 +819,70 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT data01 = *(a1 + 0); #endif - data03 = *(a2 + 0); + data09 = *(a2 + 0); #ifndef UNIT - data04 = *(a2 + 1); + data10 = *(a2 + 1); #endif *(b + 0) = INV(data01); - *(b + 1) = data03; - *(b + 3) = INV(data04); + *(b + 1) = data09; + + *(b + 3) = INV(data10); } if (ii < jj) { data01 = *(a1 + 0); data02 = *(a1 + 1); - data03 = *(a2 + 0); - data04 = *(a2 + 1); + data09 = *(a2 + 0); + data10 = *(a2 + 1); *(b + 0) = data01; - *(b + 1) = data03; + *(b + 1) = data09; *(b + 2) = data02; - *(b + 3) = data04; + *(b + 3) = data10; } a1 += 2; a2 += 2; - b += 4; + b += 4; i --; ii += 2; } - if ((m & 1) != 0) { - - if (ii== jj) { - + if (m & 1) { + if (ii == jj) { #ifndef UNIT data01 = *(a1 + 0); #endif - - data03 = *(a2 + 0); + data09 = *(a2 + 0); *(b + 0) = INV(data01); - *(b + 1) = data03; + *(b + 1) = data09; } - if (ii < jj) { + if (ii < jj) { data01 = *(a1 + 0); - data02 = *(a2 + 0); + data09 = *(a2 + 0); + *(b + 0) = data01; - *(b + 1) = data02; + *(b + 1) = data09; } b += 2; + // ii += 1; } - a += 2 * lda; + + a += 2 * lda; jj += 2; } - if (n & 1) { + if (nmod6 & 1) { a1 = a + 0 * lda; - i = m; ii = 0; + + i = m; while (i > 0) { if (ii == jj) { @@ -339,10 +897,10 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT *(b + 0) = data01; } - a1+= 1; - b += 1; + a1 += 1; + b += 1; i --; - ii += 1; + ii ++; } } diff --git a/kernel/generic/trsm_utcopy_6.c b/kernel/generic/trsm_utcopy_6.c index f83617224f..6afc005470 100644 --- a/kernel/generic/trsm_utcopy_6.c +++ b/kernel/generic/trsm_utcopy_6.c @@ -49,21 +49,34 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT BLASLONG i, ii, j, jj; - FLOAT data01, data02, data03, data04, data05, data06, data07, data08; - FLOAT data09, data10, data11, data12, data13, data14, data15, data16; - FLOAT *a1, *a2, *a3, *a4; + FLOAT data01, data02, data03, data04, data05, data06; + FLOAT data09, data10, data11, data12, data13, data14; + FLOAT data17, data18, data19, data20, data21, data22; + FLOAT data25, data26, data27, data28, data29, data30; + FLOAT data33, data34, data35, data36, data37, data38; + FLOAT data41, data42, data43, data44, data45, data46; + + FLOAT *a1, *a2, *a3, *a4, *a5, *a6, *a7, *a8; jj = offset; - j = (n >> 2); + BLASLONG mmod6, nmod6, k; + mmod6 = m - (m/6)*6 ; + nmod6 = n - (n/6)*6 ; + + // j = (n >> 3); + j = (n / 6); while (j > 0){ a1 = a + 0 * lda; a2 = a + 1 * lda; a3 = a + 2 * lda; a4 = a + 3 * lda; + a5 = a + 4 * lda; + a6 = a + 5 * lda; - i = (m >> 2); + // i = (m >> 3); + i = (m / 6); ii = 0; while (i > 0) { @@ -72,37 +85,67 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT data01 = *(a1 + 0); #endif - data05 = *(a2 + 0); + data09 = *(a2 + 0); #ifndef UNIT - data06 = *(a2 + 1); + data10 = *(a2 + 1); #endif - data09 = *(a3 + 0); - data10 = *(a3 + 1); + data17 = *(a3 + 0); + data18 = *(a3 + 1); #ifndef UNIT - data11 = *(a3 + 2); + data19 = *(a3 + 2); #endif - data13 = *(a4 + 0); - data14 = *(a4 + 1); - data15 = *(a4 + 2); + data25 = *(a4 + 0); + data26 = *(a4 + 1); + data27 = *(a4 + 2); #ifndef UNIT - data16 = *(a4 + 3); + data28 = *(a4 + 3); #endif - *(b + 0) = INV(data01); + data33 = *(a5 + 0); + data34 = *(a5 + 1); + data35 = *(a5 + 2); + data36 = *(a5 + 3); +#ifndef UNIT + data37 = *(a5 + 4); +#endif - *(b + 4) = data05; - *(b + 5) = INV(data06); + data41 = *(a6 + 0); + data42 = *(a6 + 1); + data43 = *(a6 + 2); + data44 = *(a6 + 3); + data45 = *(a6 + 4); +#ifndef UNIT + data46 = *(a6 + 5); +#endif - *(b + 8) = data09; - *(b + 9) = data10; - *(b + 10) = INV(data11); + *(b + 0) = INV(data01); - *(b + 12) = data13; - *(b + 13) = data14; - *(b + 14) = data15; - *(b + 15) = INV(data16); + *(b + 6) = data09; + *(b + 7) = INV(data10); + + *(b + 12) = data17; + *(b + 13) = data18; + *(b + 14) = INV(data19); + + *(b + 18) = data25; + *(b + 19) = data26; + *(b + 20) = data27; + *(b + 21) = INV(data28); + + *(b + 24) = data33; + *(b + 25) = data34; + *(b + 26) = data35; + *(b + 27) = data36; + *(b + 28) = INV(data37); + + *(b + 30) = data41; + *(b + 31) = data42; + *(b + 32) = data43; + *(b + 33) = data44; + *(b + 34) = data45; + *(b + 35) = INV(data46); } if (ii > jj) { @@ -110,21 +153,166 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT data02 = *(a1 + 1); data03 = *(a1 + 2); data04 = *(a1 + 3); + data05 = *(a1 + 4); + data06 = *(a1 + 5); + + data09 = *(a2 + 0); + data10 = *(a2 + 1); + data11 = *(a2 + 2); + data12 = *(a2 + 3); + data13 = *(a2 + 4); + data14 = *(a2 + 5); + + data17 = *(a3 + 0); + data18 = *(a3 + 1); + data19 = *(a3 + 2); + data20 = *(a3 + 3); + data21 = *(a3 + 4); + data22 = *(a3 + 5); + + data25 = *(a4 + 0); + data26 = *(a4 + 1); + data27 = *(a4 + 2); + data28 = *(a4 + 3); + data29 = *(a4 + 4); + data30 = *(a4 + 5); + + data33 = *(a5 + 0); + data34 = *(a5 + 1); + data35 = *(a5 + 2); + data36 = *(a5 + 3); + data37 = *(a5 + 4); + data38 = *(a5 + 5); + + data41 = *(a6 + 0); + data42 = *(a6 + 1); + data43 = *(a6 + 2); + data44 = *(a6 + 3); + data45 = *(a6 + 4); + data46 = *(a6 + 5); + + *(b + 0) = data01; + *(b + 1) = data02; + *(b + 2) = data03; + *(b + 3) = data04; + *(b + 4) = data05; + *(b + 5) = data06; + *(b + 6) = data09; + *(b + 7) = data10; + *(b + 8) = data11; + *(b + 9) = data12; + *(b + 10) = data13; + *(b + 11) = data14; + + *(b + 12) = data17; + *(b + 13) = data18; + *(b + 14) = data19; + *(b + 15) = data20; + *(b + 16) = data21; + *(b + 17) = data22; + *(b + 18) = data25; + *(b + 19) = data26; + *(b + 20) = data27; + *(b + 21) = data28; + *(b + 22) = data29; + *(b + 23) = data30; + + *(b + 24) = data33; + *(b + 25) = data34; + *(b + 26) = data35; + *(b + 27) = data36; + *(b + 28) = data37; + *(b + 29) = data38; + *(b + 30) = data41; + *(b + 31) = data42; + *(b + 32) = data43; + *(b + 33) = data44; + *(b + 34) = data45; + *(b + 35) = data46; - data05 = *(a2 + 0); - data06 = *(a2 + 1); - data07 = *(a2 + 2); - data08 = *(a2 + 3); + } - data09 = *(a3 + 0); - data10 = *(a3 + 1); - data11 = *(a3 + 2); - data12 = *(a3 + 3); + a1 += 6 * lda; + a2 += 6 * lda; + a3 += 6 * lda; + a4 += 6 * lda; + a5 += 6 * lda; + a6 += 6 * lda; + a7 += 6 * lda; + a8 += 6 * lda; + b += 36; - data13 = *(a4 + 0); - data14 = *(a4 + 1); - data15 = *(a4 + 2); - data16 = *(a4 + 3); + i --; + ii += 6; + } + + if (mmod6 & 4) { + if (ii == jj) { +#ifndef UNIT + data01 = *(a1 + 0); +#endif + + data09 = *(a2 + 0); +#ifndef UNIT + data10 = *(a2 + 1); +#endif + + data17 = *(a3 + 0); + data18 = *(a3 + 1); +#ifndef UNIT + data19 = *(a3 + 2); +#endif + + data25 = *(a4 + 0); + data26 = *(a4 + 1); + data27 = *(a4 + 2); +#ifndef UNIT + data28 = *(a4 + 3); +#endif + + *(b + 0) = INV(data01); + + *(b + 6) = data09; + *(b + 7) = INV(data10); + + *(b + 12) = data17; + *(b + 13) = data18; + *(b + 14) = INV(data19); + + *(b + 18) = data25; + *(b + 19) = data26; + *(b + 20) = data27; + *(b + 21) = INV(data28); + } + + if (ii > jj) { + data01 = *(a1 + 0); + data02 = *(a1 + 1); + data03 = *(a1 + 2); + data04 = *(a1 + 3); + data05 = *(a1 + 4); + data06 = *(a1 + 5); + + data09 = *(a2 + 0); + data10 = *(a2 + 1); + data11 = *(a2 + 2); + data12 = *(a2 + 3); + data13 = *(a2 + 4); + data14 = *(a2 + 5); + + data17 = *(a3 + 0); + data18 = *(a3 + 1); + data19 = *(a3 + 2); + data20 = *(a3 + 3); + data21 = *(a3 + 4); + data22 = *(a3 + 5); + + data25 = *(a4 + 0); + data26 = *(a4 + 1); + data27 = *(a4 + 2); + data28 = *(a4 + 3); + data29 = *(a4 + 4); + data30 = *(a4 + 5); *(b + 0) = data01; *(b + 1) = data02; @@ -132,44 +320,49 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT *(b + 3) = data04; *(b + 4) = data05; *(b + 5) = data06; - *(b + 6) = data07; - *(b + 7) = data08; - - *(b + 8) = data09; - *(b + 9) = data10; - *(b + 10) = data11; - *(b + 11) = data12; - *(b + 12) = data13; - *(b + 13) = data14; - *(b + 14) = data15; - *(b + 15) = data16; + *(b + 6) = data09; + *(b + 7) = data10; + *(b + 8) = data11; + *(b + 9) = data12; + *(b + 10) = data13; + *(b + 11) = data14; + + *(b + 12) = data17; + *(b + 13) = data18; + *(b + 14) = data19; + *(b + 15) = data20; + *(b + 16) = data21; + *(b + 17) = data22; + *(b + 18) = data25; + *(b + 19) = data26; + *(b + 20) = data27; + *(b + 21) = data28; + *(b + 22) = data29; + *(b + 23) = data30; } a1 += 4 * lda; a2 += 4 * lda; - a3 += 4 * lda; - a4 += 4 * lda; - b += 16; - - i --; + /* a3 += 4 * lda; + a4 += 4 * lda; */ + b += 24; ii += 4; } - if ((m & 2) != 0) { - - if (ii== jj) { + if (mmod6 & 2) { + if (ii == jj) { #ifndef UNIT data01 = *(a1 + 0); #endif - data05 = *(a2 + 0); + + data09 = *(a2 + 0); #ifndef UNIT - data06 = *(a2 + 1); + data10 = *(a2 + 1); #endif *(b + 0) = INV(data01); - - *(b + 4) = data05; - *(b + 5) = INV(data06); + *(b + 6) = data09; + *(b + 7) = INV(data10); } if (ii > jj) { @@ -177,11 +370,15 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT data02 = *(a1 + 1); data03 = *(a1 + 2); data04 = *(a1 + 3); + data05 = *(a1 + 4); + data06 = *(a1 + 5); - data05 = *(a2 + 0); - data06 = *(a2 + 1); - data07 = *(a2 + 2); - data08 = *(a2 + 3); + data09 = *(a2 + 0); + data10 = *(a2 + 1); + data11 = *(a2 + 2); + data12 = *(a2 + 3); + data13 = *(a2 + 4); + data14 = *(a2 + 5); *(b + 0) = data01; *(b + 1) = data02; @@ -189,46 +386,84 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT *(b + 3) = data04; *(b + 4) = data05; *(b + 5) = data06; - *(b + 6) = data07; - *(b + 7) = data08; + *(b + 6) = data09; + *(b + 7) = data10; + *(b + 8) = data11; + *(b + 9) = data12; + *(b + 10) = data13; + *(b + 11) = data14; } a1 += 2 * lda; - a2 += 2 * lda; - b += 8; - + // a2 += 2 * lda; + b += 12; ii += 2; } - if ((m & 1) != 0) { - - if (ii== jj) { + if (mmod6 & 1) { + if (ii == jj) { #ifndef UNIT data01 = *(a1 + 0); #endif *(b + 0) = INV(data01); } - if (ii > jj) { + if (ii > jj) { data01 = *(a1 + 0); data02 = *(a1 + 1); data03 = *(a1 + 2); data04 = *(a1 + 3); + data05 = *(a1 + 4); + data06 = *(a1 + 5); *(b + 0) = data01; *(b + 1) = data02; *(b + 2) = data03; *(b + 3) = data04; + *(b + 4) = data05; + *(b + 5) = data06; } - b += 4; + b += 6; } + a += 6; + jj += 6; + j --; + } + + if (nmod6 & 4) { + + a1 = a; a += 4; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 4)) { + for (k = 0; k < ii - jj; k ++) { + *(b + k) = *(a1 + k); + } + + *(b + ii - jj) = INV(*(a1 + ii - jj)); + } + + if (ii - jj >= 4) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a1 + 1); + *(b + 2) = *(a1 + 2); + *(b + 3) = *(a1 + 3); + } + + b += 4; + a1 += lda; + ii ++; + } + jj += 4; - j --; } - if (n & 2) { + + if (nmod6 & 2) { a1 = a + 0 * lda; a2 = a + 1 * lda; @@ -240,58 +475,58 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT #ifndef UNIT data01 = *(a1 + 0); #endif - data03 = *(a2 + 0); + + data09 = *(a2 + 0); #ifndef UNIT - data04 = *(a2 + 1); + data10 = *(a2 + 1); #endif *(b + 0) = INV(data01); - *(b + 2) = data03; - *(b + 3) = INV(data04); + *(b + 2) = data09; + *(b + 3) = INV(data10); } if (ii > jj) { data01 = *(a1 + 0); data02 = *(a1 + 1); - data03 = *(a2 + 0); - data04 = *(a2 + 1); + data09 = *(a2 + 0); + data10 = *(a2 + 1); *(b + 0) = data01; *(b + 1) = data02; - *(b + 2) = data03; - *(b + 3) = data04; + *(b + 2) = data09; + *(b + 3) = data10; } a1 += 2 * lda; a2 += 2 * lda; b += 4; - i --; ii += 2; } - if ((m & 1) != 0) { - - if (ii== jj) { + if (m & 1) { + if (ii == jj) { #ifndef UNIT data01 = *(a1 + 0); #endif *(b + 0) = INV(data01); } - if (ii > jj) { + if (ii > jj) { data01 = *(a1 + 0); data02 = *(a1 + 1); + *(b + 0) = data01; *(b + 1) = data02; } - b += 2; + b += 2; } a += 2; jj += 2; } - if (n & 1) { + if (nmod6 & 1) { a1 = a + 0 * lda; i = m; @@ -310,9 +545,8 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT *(b + 0) = data01; } - a1 += 1 * lda; + a1 += lda; b += 1; - i --; ii += 1; } diff --git a/kernel/loongarch64/KERNEL.LOONGSON3R5 b/kernel/loongarch64/KERNEL.LOONGSON3R5 index 20d0769f48..2c1ab87e59 100644 --- a/kernel/loongarch64/KERNEL.LOONGSON3R5 +++ b/kernel/loongarch64/KERNEL.LOONGSON3R5 @@ -85,11 +85,11 @@ ZSWAPKERNEL = cswap_lasx.S CSUMKERNEL = csum_lasx.S ZSUMKERNEL = csum_lasx.S -DGEMMKERNEL = dgemm_kernel_16x4.S +DGEMMKERNEL = dgemm_kernel_16x6.S DGEMMINCOPY = dgemm_ncopy_16.S DGEMMITCOPY = dgemm_tcopy_16.S -DGEMMONCOPY = dgemm_ncopy_4.S -DGEMMOTCOPY = dgemm_tcopy_4.S +DGEMMONCOPY = gemm_ncopy_6.prefx.c +DGEMMOTCOPY = dgemm_tcopy_6.S DGEMMINCOPYOBJ = dgemm_incopy$(TSUFFIX).$(SUFFIX) DGEMMITCOPYOBJ = dgemm_itcopy$(TSUFFIX).$(SUFFIX) DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX) @@ -153,10 +153,10 @@ ZTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c ZTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c ZTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c -DTRSMKERNEL_LN = dtrsm_kernel_LN_16x4_lasx.S -DTRSMKERNEL_LT = dtrsm_kernel_LT_16x4_lasx.S -DTRSMKERNEL_RN = dtrsm_kernel_RN_16x4_lasx.S -DTRSMKERNEL_RT = dtrsm_kernel_RT_16x4_lasx.S +DTRSMKERNEL_LN = trsm_kernel_LN_UNROLLN6.c +DTRSMKERNEL_LT = trsm_kernel_LT_UNROLLN6.c +DTRSMKERNEL_RN = trsm_kernel_RN_UNROLLN6.c +DTRSMKERNEL_RT = trsm_kernel_RT_UNROLLN6.c STRSMKERNEL_LN = ../generic/trsm_kernel_LN.c STRSMKERNEL_LT = ../generic/trsm_kernel_LT.c diff --git a/kernel/loongarch64/dgemm_kernel_16x6.S b/kernel/loongarch64/dgemm_kernel_16x6.S new file mode 100644 index 0000000000..90da107377 --- /dev/null +++ b/kernel/loongarch64/dgemm_kernel_16x6.S @@ -0,0 +1,6256 @@ +/******************************************************************************* +Copyright (c) 2024, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ +#define ASSEMBLER + +#include "common.h" + +/* Function parameters */ +#define M $r4 // param 1: bm +#define N $r5 // param 2: bn +#define K $r6 // param 3: bk +#define ALPHA $f0 // param 4: alpha +#define A $r7 // param 5: ba +#define B $r8 // param 6: bb +#define C $r9 // param 7: bc +#define LDC $r10 // param 8: ldc + +#ifdef TRMMKERNEL +#define OFFSET $r11 // param 9: offset +#endif +#define OFF $r12 + +/* Cycle control parameters */ +#define I $r13 +#define J $r14 +#define L $r15 +#define TL $r16 +/* Matrix address */ +#define A0 $r17 +#define B0 $r18 +#define C0 $r19 +#define C1 $r20 +#define C2 $r23 +#define C3 $r24 +#define C4 $r25 +#define C5 $r26 +#define T0 $r27 /* !! DO NOT USE $r21 and $r22 !! */ +#define T1 $r28 +#define T2 $r29 +#define I48 $r30 +#define ZERO $r0 + +/* LASX vectors */ +#define U0 $xr0 +#define U1 $xr1 +#define U2 $xr2 +#define U3 $xr3 +#define U4 $xr4 +#define U5 $xr5 +#define U6 $xr6 +#define D0 $xr7 +#define D1 $xr8 +#define D2 $xr9 +#define D3 $xr10 +#define D4 $xr11 +#define D5 $xr12 +#define D6 $xr13 +#define D7 $xr14 +#define D8 $xr15 +#define D9 $xr16 +#define D10 $xr17 +#define D11 $xr18 +#define D12 $xr19 +#define D13 $xr20 +#define D14 $xr21 +#define D15 $xr22 +#define D16 $xr23 +#define D17 $xr24 +#define D18 $xr25 +#define D19 $xr26 +#define D20 $xr27 +#define D21 $xr28 +#define D22 $xr29 +#define D23 $xr30 +#define VALPHA $xr31 + +/* Prefetch interval */ +#define A_PRE 0x200 /* 0x200 / 0x80 = 4 */ +#define B_PRE 0x100 /* 0x100 / 0x30 = 4 */ + +.macro KERNEL_16x6 + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + /* Cumulative D0~D23 */ + xvldrepl.d U4, B0, 0x00 + xvldrepl.d U5, B0, 0x08 + xvldrepl.d U6, B0, 0x10 + + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + preld 0, B0, B_PRE + + xvfmadd.d D4, U0, U5, D4 + xvfmadd.d D5, U1, U5, D5 + xvfmadd.d D6, U2, U5, D6 + xvfmadd.d D7, U3, U5, D7 + + xvfmadd.d D8, U0, U6, D8 + xvfmadd.d D9, U1, U6, D9 + xvfmadd.d D10, U2, U6, D10 + xvfmadd.d D11, U3, U6, D11 + preld 0, A0, A_PRE + + xvldrepl.d U4, B0, 0x18 + xvldrepl.d U5, B0, 0x20 + xvldrepl.d U6, B0, 0x28 + + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + xvfmadd.d D14, U2, U4, D14 + xvfmadd.d D15, U3, U4, D15 + + xvfmadd.d D16, U0, U5, D16 + xvfmadd.d D17, U1, U5, D17 + xvfmadd.d D18, U2, U5, D18 + xvfmadd.d D19, U3, U5, D19 + preld 0, A0, A_PRE + 0x40 + + xvfmadd.d D20, U0, U6, D20 + xvfmadd.d D21, U1, U6, D21 + xvfmadd.d D22, U2, U6, D22 + xvfmadd.d D23, U3, U6, D23 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x30 +.endm + + PROLOGUE + + addi.d $sp, $sp, -160 + /* Store $r23~$31 */ + SDARG $r23, $sp, 0 + SDARG $r24, $sp, 8 + SDARG $r25, $sp, 16 + SDARG $r26, $sp, 24 + SDARG $r27, $sp, 32 + SDARG $r28, $sp, 40 + SDARG $r29, $sp, 48 + SDARG $r30, $sp, 56 + SDARG $r31, $sp, 64 + fst.d $f23, $sp, 72 + fst.d $f24, $sp, 80 + fst.d $f25, $sp, 96 + fst.d $f26, $sp, 104 + fst.d $f27, $sp, 112 + fst.d $f28, $sp, 120 + fst.d $f29, $sp, 128 + fst.d $f30, $sp, 136 + fst.d $f31, $sp, 144 + fst.d ALPHA, $sp, 152 + +#if defined (TRMMKERNEL) && !defined(LEFT) + sub.d OFF, ZERO, OFFSET +#else + xor OFF, OFF, OFF +#endif + + addi.d I48, ZERO, 48 + /* VALPHA = {ALPHA, ALPHA, ALPHA, ALPHA} */ + xvld VALPHA, $sp, 152 + xvreplve0.d VALPHA, VALPHA + xor T0, T0, T0 + addi.d T0, T0, 6 + /* if (!(N / 6)) goto L_N5 */ + div.d J, N, T0 /* J = bn / 6 */ + mul.d T0, J, T0 + sub.d N, N, T0 + beq ZERO, J, .L_N5 + +.L_J1: /* J-- && This loop include Condition 1 */ + +/************************* Condition 1 if((N / 6) && (M >> 4)) START !!! ************************* +* dgemm_core_16x6 */ + move C0, C + move A0, A + slli.d T0, LDC, 3 + add.d C1, C0, T0 + addi.d J, J, -1 /* J-- */ + add.d C2, C1, T0 + add.d C3, C2, T0 + add.d C4, C3, T0 + add.d C5, C4, T0 + +#if defined(TRMMKERNEL) && defined(LEFT) + move OFF, OFFSET +#endif + + /* if (!(M >> 3)) goto L_M8 */ + srai.d I, M, 4 /* I = bm >> 4 */ + beq ZERO, I, .L_M8 + +.L_I1: /* I-- */ +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move B0, B +#else + slli.d T0, OFF, 0x07 + add.d A0, A0, T0 + mul.d T0, OFF, I48 + add.d B0, B, T0 +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub.d L, K, OFF +#elif defined(LEFT) + /* number of values in A */ + addi.d L, OFF, 16 +#else + /* number of values in B */ + addi.d L, OFF, 6 +#endif +#else // #if !defined(TRMMKERNEL) + move B0, B + move L, K /* L = bk */ +#endif + /* Calculate the first set of D0~D23, + * avoidig set 0 operation + * Load 16 * 64 from A0 + * U0 = {a3, a2, a1, a0} + * U1 = {a7, a6, a5, a4} + * U2 = {a11, a10, a9, a8} + * U3 = {a15, a14, a13, a12} + */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvldrepl.d U5, B0, 0x08 + xvldrepl.d U6, B0, 0x10 + + preld 0, C0, 0x00 + /* line 1 */ + xvfmul.d D0, U0, U4 + xvfmul.d D1, U1, U4 + preld 0, C0, 0x40 + xvfmul.d D2, U2, U4 + xvfmul.d D3, U3, U4 + + preld 0, C1, 0x00 + /* line 2 */ + xvfmul.d D4, U0, U5 + xvfmul.d D5, U1, U5 + preld 0, C1, 0x40 + xvfmul.d D6, U2, U5 + xvfmul.d D7, U3, U5 + + preld 0, C2, 0x00 + /* line 3 */ + xvfmul.d D8, U0, U6 + xvfmul.d D9, U1, U6 + preld 0, C2, 0x40 + xvfmul.d D10, U2, U6 + xvfmul.d D11, U3, U6 + + xvldrepl.d U4, B0, 0x18 + xvldrepl.d U5, B0, 0x20 + xvldrepl.d U6, B0, 0x28 + + preld 0, C3, 0x00 + /* line 4 */ + xvfmul.d D12, U0, U4 + xvfmul.d D13, U1, U4 + preld 0, C3, 0x40 + xvfmul.d D14, U2, U4 + xvfmul.d D15, U3, U4 + + preld 0, C4, 0x00 + /* line 5 */ + xvfmul.d D16, U0, U5 + xvfmul.d D17, U1, U5 + preld 0, C4, 0x40 + xvfmul.d D18, U2, U5 + xvfmul.d D19, U3, U5 + + preld 0, C5, 0x00 + /* line 6 */ + xvfmul.d D20, U0, U6 + xvfmul.d D21, U1, U6 + preld 0, C5, 0x40 + xvfmul.d D22, U2, U6 + xvfmul.d D23, U3, U6 + + /* Add stride for A0 and B0 */ + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x30 + /* Reduce L */ + addi.d L, L, -1 + srai.d TL, L, 3 /* TL = (L-1) >> 3 */ + /* if (TL < 1) goto L_L7 */ + beq ZERO,TL, .L_L7 + + /* Calculate 8 sets of D0~D23 */ +.L_TL1: /* TL-- */ + KERNEL_16x6 + KERNEL_16x6 + KERNEL_16x6 + KERNEL_16x6 + KERNEL_16x6 + KERNEL_16x6 + KERNEL_16x6 + KERNEL_16x6 + addi.d TL, TL, -1 /* TL-- */ + blt ZERO,TL, .L_TL1 + + /* Maybe we need calculate the last + * 7 sets of D0~D23? + */ +.L_L7: + /* if (!(L & 7)) goto L_L0 */ + andi TL, L, 7 + beq TL, ZERO,.L_L0 + +.L_L71: + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + /* Cumulative D0~D23 */ + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + xvfmadd.d D6, U2, U4, D6 + xvfmadd.d D7, U3, U4, D7 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + xvfmadd.d D10, U2, U4, D10 + xvfmadd.d D11, U3, U4, D11 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + xvfmadd.d D14, U2, U4, D14 + xvfmadd.d D15, U3, U4, D15 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + xvfmadd.d D17, U1, U4, D17 + xvfmadd.d D18, U2, U4, D18 + xvfmadd.d D19, U3, U4, D19 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + xvfmadd.d D21, U1, U4, D21 + xvfmadd.d D22, U2, U4, D22 + xvfmadd.d D23, U3, U4, D23 + + /* Add stride for A0, B0 */ + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x30 + + addi.d TL, TL, -1 + blt ZERO,TL, .L_L71 + +.L_L0: +#if defined(TRMMKERNEL) + xvfmul.d D0, D0, VALPHA + xvfmul.d D1, D1, VALPHA + xvfmul.d D2, D2, VALPHA + xvfmul.d D3, D3, VALPHA + xvfmul.d D4, D4, VALPHA + xvfmul.d D5, D5, VALPHA + xvfmul.d D6, D6, VALPHA + xvfmul.d D7, D7, VALPHA + xvfmul.d D8, D8, VALPHA + xvfmul.d D9, D9, VALPHA + xvfmul.d D10, D10, VALPHA + xvfmul.d D11, D11, VALPHA + xvfmul.d D12, D12, VALPHA + xvfmul.d D13, D13, VALPHA + xvfmul.d D14, D14, VALPHA + xvfmul.d D15, D15, VALPHA + xvfmul.d D16, D16, VALPHA + xvfmul.d D17, D17, VALPHA + xvfmul.d D18, D18, VALPHA + xvfmul.d D19, D19, VALPHA + xvfmul.d D20, D20, VALPHA + xvfmul.d D21, D21, VALPHA + xvfmul.d D22, D22, VALPHA + xvfmul.d D23, D23, VALPHA +#else + /* Load C0 */ + xvld U0, C0, 0x00 + xvld U1, C0, 0x20 + xvld U2, C0, 0x40 + xvld U3, C0, 0x60 + xvfmadd.d D0, D0, VALPHA, U0 /* D0 = U0 + (D0 * VALPHA) */ + xvfmadd.d D1, D1, VALPHA, U1 + xvfmadd.d D2, D2, VALPHA, U2 + xvfmadd.d D3, D3, VALPHA, U3 + + /* Load C1 */ + xvld U0, C1, 0x00 + xvld U1, C1, 0x20 + xvld U2, C1, 0x40 + xvld U3, C1, 0x60 + xvfmadd.d D4, D4, VALPHA, U0 + xvfmadd.d D5, D5, VALPHA, U1 + xvfmadd.d D6, D6, VALPHA, U2 + xvfmadd.d D7, D7, VALPHA, U3 + + /* Load C2 */ + xvld U0, C2, 0x00 + xvld U1, C2, 0x20 + xvld U2, C2, 0x40 + xvld U3, C2, 0x60 + xvfmadd.d D8, D8, VALPHA, U0 + xvfmadd.d D9, D9, VALPHA, U1 + xvfmadd.d D10, D10, VALPHA, U2 + xvfmadd.d D11, D11, VALPHA, U3 + + /* Load C3 */ + xvld U0, C3, 0x00 + xvld U1, C3, 0x20 + xvld U2, C3, 0x40 + xvld U3, C3, 0x60 + xvfmadd.d D12, D12, VALPHA, U0 + xvfmadd.d D13, D13, VALPHA, U1 + xvfmadd.d D14, D14, VALPHA, U2 + xvfmadd.d D15, D15, VALPHA, U3 + + /* Load C4 */ + xvld U0, C4, 0x00 + xvld U1, C4, 0x20 + xvld U2, C4, 0x40 + xvld U3, C4, 0x60 + xvfmadd.d D16, D16, VALPHA, U0 + xvfmadd.d D17, D17, VALPHA, U1 + xvfmadd.d D18, D18, VALPHA, U2 + xvfmadd.d D19, D19, VALPHA, U3 + + /* Load C5 */ + xvld U0, C5, 0x00 + xvld U1, C5, 0x20 + xvld U2, C5, 0x40 + xvld U3, C5, 0x60 + xvfmadd.d D20, D20, VALPHA, U0 + xvfmadd.d D21, D21, VALPHA, U1 + xvfmadd.d D22, D22, VALPHA, U2 + xvfmadd.d D23, D23, VALPHA, U3 + #endif + + /* Store C0 */ + xvst D0, C0, 0x00 + xvst D1, C0, 0x20 + xvst D2, C0, 0x40 + xvst D3, C0, 0x60 + /* Store C1 */ + xvst D4, C1, 0x00 + xvst D5, C1, 0x20 + xvst D6, C1, 0x40 + xvst D7, C1, 0x60 + /* Store C2 */ + xvst D8, C2, 0x00 + xvst D9, C2, 0x20 + xvst D10, C2, 0x40 + xvst D11, C2, 0x60 + /* Store C3 */ + xvst D12, C3, 0x00 + xvst D13, C3, 0x20 + xvst D14, C3, 0x40 + xvst D15, C3, 0x60 + /* Store C4 */ + xvst D16, C4, 0x00 + xvst D17, C4, 0x20 + xvst D18, C4, 0x40 + xvst D19, C4, 0x60 + /* Store C5 */ + xvst D20, C5, 0x00 + xvst D21, C5, 0x20 + xvst D22, C5, 0x40 + xvst D23, C5, 0x60 + + /* Add stride for C */ + addi.d C0, C0, 0x80 + addi.d C1, C1, 0x80 + addi.d C2, C2, 0x80 + addi.d C3, C3, 0x80 + addi.d C4, C4, 0x80 + addi.d C5, C5, 0x80 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub.d L, K, OFF +#ifdef LEFT + /* number of values in A */ + addi.d L, L, -16 +#else + /* number of values in B */ + addi.d L, L, -6 +#endif + slli.d T0, L, 0x07 + add.d A0, A0, T0 + mul.d T0, L, I48 + add.d B0, B0, T0 +#endif + +#ifdef LEFT + addi.d OFF, OFF, 0x10 +#endif +#endif // #if defined(TRMMKERNEL) + + addi.d I, I, -1 /* I-- */ + blt ZERO,I, .L_I1 + +.L_M8: + /* We have done M & 16, considering M=8/4/2/1 */ + andi I, M, 15 + beq ZERO,I, .L_M0 + + andi I, M, 8 + beq ZERO,I, .L_M4 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move B0, B +#else + slli.d T0, OFF, 0x06 + add.d A0, A0, T0 + mul.d T0, OFF, I48 + add.d B0, B, T0 +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub.d L, K, OFF +#elif defined(LEFT) + /* number of values in A */ + addi.d L, OFF, 8 +#else + /* number of values in B */ + addi.d L, OFF, 6 +#endif +#else // #if !defined(TRMMKERNEL) + move B0, B + move L, K /* L = bk */ +#endif // #if defined(TRMMKERNEL) + + /* Load 8 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + /* line 1 */ + xvfmul.d D0, U0, U4 + xvfmul.d D1, U1, U4 + + xvldrepl.d U4, B0, 0x08 + /* line 2 */ + xvfmul.d D4, U0, U4 + xvfmul.d D5, U1, U4 + + xvldrepl.d U4, B0, 0x10 + /* line 3 */ + xvfmul.d D8, U0, U4 + xvfmul.d D9, U1, U4 + + xvldrepl.d U4, B0, 0x18 + /* line 4 */ + xvfmul.d D12, U0, U4 + xvfmul.d D13, U1, U4 + + xvldrepl.d U4, B0, 0x20 + /* line 5 */ + xvfmul.d D16, U0, U4 + xvfmul.d D17, U1, U4 + + xvldrepl.d U4, B0, 0x28 + /* line 6 */ + xvfmul.d D20, U0, U4 + xvfmul.d D21, U1, U4 + + /* Add stride for A0 and B0 */ + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x30 + /* Reduce L */ + addi.d L, L, -1 + srai.d TL, L, 3 /* TL = (L-1) >> 3 */ + /* if (TL < 1) goto L_M8_L7 */ + beq ZERO,TL, .L_M8_L7 + +.L_M8_TL1: /* TL-- */ + /***8-1***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + xvfmadd.d D17, U1, U4, D17 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + xvfmadd.d D21, U1, U4, D21 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x30 + + /***8-2***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + xvfmadd.d D17, U1, U4, D17 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + xvfmadd.d D21, U1, U4, D21 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x30 + + /***8-3***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + xvfmadd.d D17, U1, U4, D17 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + xvfmadd.d D21, U1, U4, D21 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x30 + + /***8-4***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + xvfmadd.d D17, U1, U4, D17 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + xvfmadd.d D21, U1, U4, D21 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x30 + + /***8-5***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + xvfmadd.d D17, U1, U4, D17 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + xvfmadd.d D21, U1, U4, D21 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x30 + + /***8-6***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + xvfmadd.d D17, U1, U4, D17 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + xvfmadd.d D21, U1, U4, D21 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x30 + + /***8-7***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + xvfmadd.d D17, U1, U4, D17 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + xvfmadd.d D21, U1, U4, D21 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x30 + + /***8-8***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + xvfmadd.d D17, U1, U4, D17 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + xvfmadd.d D21, U1, U4, D21 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x30 + + addi.d TL, TL, -1 /* TL-- */ + blt ZERO,TL, .L_M8_TL1 + +.L_M8_L7: + /* if (!(L & 7)) goto L_M8_L0 */ + andi TL, L, 7 + beq TL, ZERO,.L_M8_L0 + +.L_M8_L71: + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + xvfmadd.d D17, U1, U4, D17 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + xvfmadd.d D21, U1, U4, D21 + + /* Add stride for A0, B0 */ + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x30 + + addi.d TL, TL, -1 + blt ZERO,TL, .L_M8_L71 + +.L_M8_L0: +#if defined(TRMMKERNEL) + xvfmul.d D0, D0, VALPHA + xvfmul.d D1, D1, VALPHA + xvfmul.d D4, D4, VALPHA + xvfmul.d D5, D5, VALPHA + xvfmul.d D8, D8, VALPHA + xvfmul.d D9, D9, VALPHA + xvfmul.d D12, D12, VALPHA + xvfmul.d D13, D13, VALPHA + xvfmul.d D16, D16, VALPHA + xvfmul.d D17, D17, VALPHA + xvfmul.d D20, D20, VALPHA + xvfmul.d D21, D21, VALPHA +#else + /* Load C0 */ + xvld U0, C0, 0x00 + xvld U1, C0, 0x20 + xvfmadd.d D0, D0, VALPHA, U0 /* D0 = U0 + (D0 * VALPHA) */ + xvfmadd.d D1, D1, VALPHA, U1 + + /* Load C1 */ + xvld U0, C1, 0x00 + xvld U1, C1, 0x20 + xvfmadd.d D4, D4, VALPHA, U0 + xvfmadd.d D5, D5, VALPHA, U1 + + /* Load C2 */ + xvld U0, C2, 0x00 + xvld U1, C2, 0x20 + xvfmadd.d D8, D8, VALPHA, U0 + xvfmadd.d D9, D9, VALPHA, U1 + + /* Load C3 */ + xvld U0, C3, 0x00 + xvld U1, C3, 0x20 + xvfmadd.d D12, D12, VALPHA, U0 + xvfmadd.d D13, D13, VALPHA, U1 + + /* Load C4 */ + xvld U0, C4, 0x00 + xvld U1, C4, 0x20 + xvfmadd.d D16, D16, VALPHA, U0 + xvfmadd.d D17, D17, VALPHA, U1 + + /* Load C5 */ + xvld U0, C5, 0x00 + xvld U1, C5, 0x20 + xvfmadd.d D20, D20, VALPHA, U0 + xvfmadd.d D21, D21, VALPHA, U1 +#endif + + /* Store C0 */ + xvst D0, C0, 0x00 + xvst D1, C0, 0x20 + /* Store C1 */ + xvst D4, C1, 0x00 + xvst D5, C1, 0x20 + /* Store C2 */ + xvst D8, C2, 0x00 + xvst D9, C2, 0x20 + /* Store C3 */ + xvst D12, C3, 0x00 + xvst D13, C3, 0x20 + /* Store C4 */ + xvst D16, C4, 0x00 + xvst D17, C4, 0x20 + /* Store C5 */ + xvst D20, C5, 0x00 + xvst D21, C5, 0x20 + + /* Add stride for C */ + addi.d C0, C0, 0x40 + addi.d C1, C1, 0x40 + addi.d C2, C2, 0x40 + addi.d C3, C3, 0x40 + addi.d C4, C4, 0x40 + addi.d C5, C5, 0x40 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub.d L, K, OFF +#ifdef LEFT + /* number of values in A */ + addi.d L, L, -8 +#else + /* number of values in B */ + addi.d L, L, -6 +#endif + slli.d T0, L, 0x06 + add.d A0, A0, T0 + mul.d T0, L, I48 + add.d B0, B0, T0 +#endif + +#ifdef LEFT + /* number of values in A */ + addi.d OFF, OFF, 0x08 +#endif +#endif // #if defined(TRMMKERNEL) + +/********LOOP (if(N / 6 ) && (M & 8)) End************/ + +.L_M4: + andi I, M, 4 + beq ZERO,I, .L_M2 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move B0, B +#else + slli.d T0, OFF, 0x05 + add.d A0, A0, T0 + mul.d T0, OFF, I48 + add.d B0, B, T0 +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub.d L, K, OFF +#elif defined(LEFT) + /* number of values in A */ + addi.d L, OFF, 4 +#else + /* number of values in B */ + addi.d L, OFF, 6 +#endif +#else // #if !defined(TRMMKERNEL) + move B0, B + move L, K /* L = bk */ +#endif + + /* Load 4 * 64 from A0 */ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + /* line 1 */ + xvfmul.d D0, U0, U4 + + xvldrepl.d U4, B0, 0x08 + /* line 2 */ + xvfmul.d D4, U0, U4 + + xvldrepl.d U4, B0, 0x10 + /* line 3 */ + xvfmul.d D8, U0, U4 + + xvldrepl.d U4, B0, 0x18 + /* line 4 */ + xvfmul.d D12, U0, U4 + + xvldrepl.d U4, B0, 0x20 + /* line 5 */ + xvfmul.d D16, U0, U4 + + xvldrepl.d U4, B0, 0x28 + /* line 6 */ + xvfmul.d D20, U0, U4 + + /* Add stride for A0 and B0 */ + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x30 + /* Reduce L */ + addi.d L, L, -1 + srai.d TL, L, 3 /* TL = (L-1) >> 3 */ + /* if (TL < 1) goto L_M4_L7 */ + beq ZERO,TL, .L_M4_L7 + +.L_M4_TL1: /* TL-- */ + /***8-1***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x30 + + /***8-2***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x30 + + /***8-3***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x30 + + /***8-4***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x30 + + /***8-5***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x30 + + /***8-6***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x30 + + /***8-7***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x30 + + /***8-8***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x30 + + addi.d TL, TL, -1 /* TL-- */ + blt ZERO,TL, .L_M4_TL1 + +.L_M4_L7: + /* if (!(L & 7)) goto L_M4_L0 */ + andi TL, L, 7 + beq TL, ZERO,.L_M4_L0 + +.L_M4_L71: + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + /* Add stride for A0, B0 */ + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x30 + + addi.d TL, TL, -1 + blt ZERO,TL, .L_M4_L71 + +.L_M4_L0: +#if defined(TRMMKERNEL) + xvfmul.d D0, D0, VALPHA + xvfmul.d D4, D4, VALPHA + xvfmul.d D8, D8, VALPHA + xvfmul.d D12, D12, VALPHA + xvfmul.d D16, D16, VALPHA + xvfmul.d D20, D20, VALPHA +#else + /* Load C0 */ + xvld U0, C0, 0x00 + xvfmadd.d D0, D0, VALPHA, U0 /* D0 = U0 + (D0 * VALPHA) */ + + /* Load C1 */ + xvld U0, C1, 0x00 + xvfmadd.d D4, D4, VALPHA, U0 + + /* Load C2 */ + xvld U0, C2, 0x00 + xvfmadd.d D8, D8, VALPHA, U0 + + /* Load C3 */ + xvld U0, C3, 0x00 + xvfmadd.d D12, D12, VALPHA, U0 + + /* Load C4 */ + xvld U0, C4, 0x00 + xvfmadd.d D16, D16, VALPHA, U0 + + /* Load C5 */ + xvld U0, C5, 0x00 + xvfmadd.d D20, D20, VALPHA, U0 +#endif + + /* Store C0 */ + xvst D0, C0, 0x00 + /* Store C1 */ + xvst D4, C1, 0x00 + /* Store C2 */ + xvst D8, C2, 0x00 + /* Store C3 */ + xvst D12, C3, 0x00 + /* Store C4 */ + xvst D16, C4, 0x00 + /* Store C5 */ + xvst D20, C5, 0x00 + + /* Add stride for C */ + addi.d C0, C0, 0x20 + addi.d C1, C1, 0x20 + addi.d C2, C2, 0x20 + addi.d C3, C3, 0x20 + addi.d C4, C4, 0x20 + addi.d C5, C5, 0x20 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub.d L, K, OFF +#ifdef LEFT + /* number of values in A */ + addi.d L, L, -4 +#else + /* number of values in B */ + addi.d L, L, -6 +#endif + slli.d T0, L, 0x05 + add.d A0, A0, T0 + mul.d T0, L, I48 + add.d B0, B0, T0 +#endif + +#ifdef LEFT + /* number of values in A */ + addi.d OFF, OFF, 0x04 +#endif +#endif // #if defined(TRMMKERNEL) + +/********LOOP (if(N / 6 ) && (M & 4) ) End************/ + +.L_M2: + andi I, M, 2 + beq ZERO,I, .L_M1 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move B0, B +#else + slli.d T0, OFF, 0x04 + add.d A0, A0, T0 + mul.d T0, OFF, I48 + add.d B0, B, T0 +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub.d L, K, OFF +#elif defined(LEFT) + /* number of values in A */ + addi.d L, OFF, 2 +#else + /* number of values in B */ + addi.d L, OFF, 6 +#endif +#else // #if !defined(TRMMKERNEL) + move B0, B + move L, K /* L = bk */ +#endif + + /* Load 2 * 64 from A0 */ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + /* line 1 */ + xvfmul.d D0, U0, U4 + + xvldrepl.d U4, B0, 0x08 + /* line 2 */ + xvfmul.d D4, U0, U4 + + xvldrepl.d U4, B0, 0x10 + /* line 3 */ + xvfmul.d D8, U0, U4 + + xvldrepl.d U4, B0, 0x18 + /* line 4 */ + xvfmul.d D12, U0, U4 + + xvldrepl.d U4, B0, 0x20 + /* line 5 */ + xvfmul.d D16, U0, U4 + + xvldrepl.d U4, B0, 0x28 + /* line 6 */ + xvfmul.d D20, U0, U4 + + /* Add stride for A0 and B0 */ + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x30 + /* Reduce L */ + addi.d L, L, -1 + srai.d TL, L, 3 /* TL = (L-1) >> 3 */ + /* if (TL < 1) goto L_M2_L7 */ + beq ZERO,TL, .L_M2_L7 + +.L_M2_TL1: /* TL-- */ + /***8-1***/ + /* Load 2 * 64 from A0 */ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x30 + + /***8-2***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x30 + + /***8-3***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x30 + + /***8-4***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x30 + + /***8-5***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x30 + + /***8-6***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x30 + + /***8-7***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x30 + + /***8-8***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x30 + + addi.d TL, TL, -1 /* TL-- */ + blt ZERO,TL, .L_M2_TL1 + +.L_M2_L7: + /* if (!(L & 7)) goto L_M2_L0 */ + andi TL, L, 7 + beq TL, ZERO,.L_M2_L0 + +.L_M2_L71: + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + /* Add stride for A0, B0 */ + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x30 + + addi.d TL, TL, -1 + blt ZERO,TL, .L_M2_L71 + +.L_M2_L0: +#if defined(TRMMKERNEL) + xvfmul.d D0, D0, VALPHA + xvfmul.d D4, D4, VALPHA + xvfmul.d D8, D8, VALPHA + xvfmul.d D12, D12, VALPHA + xvfmul.d D16, D16, VALPHA + xvfmul.d D20, D20, VALPHA +#else + /* Load C0 */ + xvld U0, C0, 0x00 + xvfmadd.d D0, D0, VALPHA, U0 /* D0 = U0 + (D0 * VALPHA) */ + + /* Load C1 */ + xvld U0, C1, 0x00 + xvfmadd.d D4, D4, VALPHA, U0 + + /* Load C2 */ + xvld U0, C2, 0x00 + xvfmadd.d D8, D8, VALPHA, U0 + + /* Load C3 */ + xvld U0, C3, 0x00 + xvfmadd.d D12, D12, VALPHA, U0 + + /* Load C4 */ + xvld U0, C4, 0x00 + xvfmadd.d D16, D16, VALPHA, U0 + + /* Load C5 */ + xvld U0, C5, 0x00 + xvfmadd.d D20, D20, VALPHA, U0 +#endif + + xvstelm.d D0, C0, 0x00, 0x00 + xvstelm.d D4, C1, 0x00, 0x00 + xvstelm.d D8, C2, 0x00, 0x00 + xvstelm.d D12, C3, 0x00, 0x00 + xvstelm.d D16, C4, 0x00, 0x00 + xvstelm.d D20, C5, 0x00, 0x00 + xvstelm.d D0, C0, 0x08, 0x01 + xvstelm.d D4, C1, 0x08, 0x01 + xvstelm.d D8, C2, 0x08, 0x01 + xvstelm.d D12, C3, 0x08, 0x01 + xvstelm.d D16, C4, 0x08, 0x01 + xvstelm.d D20, C5, 0x08, 0x01 + + /* Add stride for C */ + addi.d C0, C0, 0x10 + addi.d C1, C1, 0x10 + addi.d C2, C2, 0x10 + addi.d C3, C3, 0x10 + addi.d C4, C4, 0x10 + addi.d C5, C5, 0x10 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub.d L, K, OFF +#ifdef LEFT + /* number of values in A */ + addi.d L, L, -2 +#else + /* number of values in B */ + addi.d L, L, -6 +#endif + slli.d T0, L, 0x04 + add.d A0, A0, T0 + mul.d T0, L, I48 + add.d B0, B0, T0 +#endif + +#ifdef LEFT + /* number of values in A */ + addi.d OFF, OFF, 0x02 +#endif +#endif // #if defined(TRMMKERNEL) + +/********LOOP (if(N / 6 ) && (M & 2) ) End************/ + +.L_M1: + andi I, M, 1 + beq ZERO,I, .L_M0 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move B0, B +#else + slli.d T0, OFF, 0x03 + add.d A0, A0, T0 + mul.d T0, OFF, I48 + add.d B0, B, T0 +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub.d L, K, OFF +#elif defined(LEFT) + /* number of values in A */ + addi.d L, OFF, 1 +#else + /* number of values in B */ + addi.d L, OFF, 6 +#endif +#else // #if !defined(TRMMKERNEL) + move B0, B + move L, K /* L = bk */ +#endif + + /* Load 1 * 64 from A0 */ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + /* line 1 */ + xvfmul.d D0, U0, U4 + + xvldrepl.d U4, B0, 0x08 + /* line 2 */ + xvfmul.d D4, U0, U4 + + xvldrepl.d U4, B0, 0x10 + /* line 3 */ + xvfmul.d D8, U0, U4 + + xvldrepl.d U4, B0, 0x18 + /* line 4 */ + xvfmul.d D12, U0, U4 + + xvldrepl.d U4, B0, 0x20 + /* line 5 */ + xvfmul.d D16, U0, U4 + + xvldrepl.d U4, B0, 0x28 + /* line 6 */ + xvfmul.d D20, U0, U4 + + /* Add stride for A0 and B0 */ + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x30 + /* Reduce L */ + addi.d L, L, -1 + srai.d TL, L, 3 /* TL = (L-1) >> 3 */ + /* if (TL < 1) goto L_M1_L7 */ + beq ZERO,TL, .L_M1_L7 + +.L_M1_TL1: /* TL-- */ + /***8-1***/ + /* Load 1 * 64 from A0 */ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x30 + + /***8-2***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x30 + + /***8-3***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x30 + + /***8-4***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x30 + + /***8-5***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x30 + + /***8-6***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x30 + + /***8-7***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x30 + + /***8-8***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x30 + + addi.d TL, TL, -1 /* TL-- */ + blt ZERO,TL, .L_M1_TL1 + +.L_M1_L7: + /* if (!(L & 7)) goto L_M1_L0 */ + andi TL, L, 7 + beq TL, ZERO,.L_M1_L0 + +.L_M1_L71: + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + xvldrepl.d U4, B0, 0x20 + xvfmadd.d D16, U0, U4, D16 + + xvldrepl.d U4, B0, 0x28 + xvfmadd.d D20, U0, U4, D20 + + /* Add stride for A0, B0 */ + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x30 + + addi.d TL, TL, -1 + blt ZERO,TL, .L_M1_L71 + +.L_M1_L0: +#ifdef TRMMKERNEL + xvfmul.d D0, D0, VALPHA + xvfmul.d D4, D4, VALPHA + xvfmul.d D8, D8, VALPHA + xvfmul.d D12, D12, VALPHA + xvfmul.d D16, D16, VALPHA + xvfmul.d D20, D20, VALPHA +#else + /* Load C0 */ + xvld U0, C0, 0x00 + xvfmadd.d D0, D0, VALPHA, U0 /* D0 = U0 + (D0 * VALPHA) */ + + /* Load C1 */ + xvld U0, C1, 0x00 + xvfmadd.d D4, D4, VALPHA, U0 + + /* Load C2 */ + xvld U0, C2, 0x00 + xvfmadd.d D8, D8, VALPHA, U0 + + /* Load C3 */ + xvld U0, C3, 0x00 + xvfmadd.d D12, D12, VALPHA, U0 + + /* Load C4 */ + xvld U0, C4, 0x00 + xvfmadd.d D16, D16, VALPHA, U0 + + /* Load C5 */ + xvld U0, C5, 0x00 + xvfmadd.d D20, D20, VALPHA, U0 +#endif + + xvstelm.d D0, C0, 0x00, 0x00 + xvstelm.d D4, C1, 0x00, 0x00 + xvstelm.d D8, C2, 0x00, 0x00 + xvstelm.d D12, C3, 0x00, 0x00 + xvstelm.d D16, C4, 0x00, 0x00 + xvstelm.d D20, C5, 0x00, 0x00 + + /* Add stride for C */ + addi.d C0, C0, 0x08 + addi.d C1, C1, 0x08 + addi.d C2, C2, 0x08 + addi.d C3, C3, 0x08 + addi.d C4, C4, 0x08 + addi.d C5, C5, 0x08 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub.d L, K, OFF +#ifdef LEFT + /* number of values in A */ + addi.d L, L, -1 +#else + /* number of values in B */ + addi.d L, L, -6 +#endif + slli.d T0, L, 0x03 + add.d A0, A0, T0 + mul.d T0, L, I48 + add.d B0, B0, T0 +#endif + +#ifdef LEFT + /* number of values in A */ + addi.d OFF, OFF, 0x01 +#endif +#endif // #if defined(TRMMKERNEL) + +/********LOOP (if(N / 6 ) && (M & 1) ) End************/ + +.L_M0: + /* Add stride for B and C + * B += (K * 6) + * C += (LDC * 6) + */ + /* since the array type is double, + * so we must mul 48 + */ + addi.d T2, ZERO,48 + mul.d T0, K, T2 + mul.d T1, LDC, T2 + add.d B, B, T0 + add.d C, C, T1 + +#if defined(TRMMKERNEL) && !defined(LEFT) + addi.d OFF, OFF, 0x06 +#endif + + blt ZERO, J, .L_J1 + +//////////////// go back to L_J1 ///////////////// +///////////////////////////////////////////////// +/************************ Condition 1 if((N >> 2) && (M >> 3)) END !!! ************************/ + +.L_N5: + andi J, N, 4 + beq ZERO, J, .L_N3 + +/************************* Condition 2 if((N & 4) && (M >> 4)) START !!! ************************* +* dgemm_core_16x4 */ + + move C0, C + move A0, A + slli.d T0, LDC, 3 + add.d C1, C0, T0 + add.d C2, C1, T0 + add.d C3, C2, T0 + +#if defined(TRMMKERNEL) && defined(LEFT) + move OFF, OFFSET +#endif + + /* if (!(M >> 3)) goto L_N5_M8 */ + srai.d I, M, 4 /* I = bm >> 4 */ + beq ZERO, I, .L_N5_M8 + +.L_N5_I1: +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move B0, B +#else + slli.d T0, OFF, 0x07 + add.d A0, A0, T0 + slli.d T0, OFF, 0x05 + add.d B0, B, T0 +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub.d L, K, OFF +#elif defined(LEFT) + /* number of values in A */ + addi.d L, OFF, 16 +#else + /* number of values in B */ + addi.d L, OFF, 4 +#endif +#else // #if !defined(TRMMKERNEL) + move B0, B + move L, K /* L = bk */ +#endif + /* Load 16 * 64 from A0 + * U0 = {a3, a2, a1, a0} + * U1 = {a7, a6, a5, a4} + * U2 = {a11, a10, a9, a8} + * U3 = {a15, a14, a13, a12} + */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + /* line 1 */ + xvfmul.d D0, U0, U4 + xvfmul.d D1, U1, U4 + xvfmul.d D2, U2, U4 + xvfmul.d D3, U3, U4 + + xvldrepl.d U4, B0, 0x08 + /* line 2 */ + xvfmul.d D4, U0, U4 + xvfmul.d D5, U1, U4 + xvfmul.d D6, U2, U4 + xvfmul.d D7, U3, U4 + + xvldrepl.d U4, B0, 0x10 + /* line 3 */ + xvfmul.d D8, U0, U4 + xvfmul.d D9, U1, U4 + xvfmul.d D10, U2, U4 + xvfmul.d D11, U3, U4 + + xvldrepl.d U4, B0, 0x18 + /* line 4 */ + xvfmul.d D12, U0, U4 + xvfmul.d D13, U1, U4 + xvfmul.d D14, U2, U4 + xvfmul.d D15, U3, U4 + + /* Add stride for A0 and B0 */ + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x20 + /* Reduce L */ + addi.d L, L, -1 + srai.d TL, L, 3 /* TL = (L-1) >> 3 */ + /* if (TL < 1) goto L_N5_L7 */ + beq ZERO,TL, .L_N5_L7 + +.L_N5_TL1: /* TL-- */ + /***8-1***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + xvfmadd.d D6, U2, U4, D6 + xvfmadd.d D7, U3, U4, D7 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + xvfmadd.d D10, U2, U4, D10 + xvfmadd.d D11, U3, U4, D11 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + xvfmadd.d D14, U2, U4, D14 + xvfmadd.d D15, U3, U4, D15 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x20 + + /***8-2***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + xvfmadd.d D6, U2, U4, D6 + xvfmadd.d D7, U3, U4, D7 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + xvfmadd.d D10, U2, U4, D10 + xvfmadd.d D11, U3, U4, D11 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + xvfmadd.d D14, U2, U4, D14 + xvfmadd.d D15, U3, U4, D15 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x20 + + /***8-3***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + xvfmadd.d D6, U2, U4, D6 + xvfmadd.d D7, U3, U4, D7 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + xvfmadd.d D10, U2, U4, D10 + xvfmadd.d D11, U3, U4, D11 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + xvfmadd.d D14, U2, U4, D14 + xvfmadd.d D15, U3, U4, D15 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x20 + + /***8-4***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + xvfmadd.d D6, U2, U4, D6 + xvfmadd.d D7, U3, U4, D7 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + xvfmadd.d D10, U2, U4, D10 + xvfmadd.d D11, U3, U4, D11 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + xvfmadd.d D14, U2, U4, D14 + xvfmadd.d D15, U3, U4, D15 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x20 + + /***8-5***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + xvfmadd.d D6, U2, U4, D6 + xvfmadd.d D7, U3, U4, D7 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + xvfmadd.d D10, U2, U4, D10 + xvfmadd.d D11, U3, U4, D11 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + xvfmadd.d D14, U2, U4, D14 + xvfmadd.d D15, U3, U4, D15 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x20 + + /***8-6***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + xvfmadd.d D6, U2, U4, D6 + xvfmadd.d D7, U3, U4, D7 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + xvfmadd.d D10, U2, U4, D10 + xvfmadd.d D11, U3, U4, D11 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + xvfmadd.d D14, U2, U4, D14 + xvfmadd.d D15, U3, U4, D15 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x20 + + /***8-7***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + xvfmadd.d D6, U2, U4, D6 + xvfmadd.d D7, U3, U4, D7 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + xvfmadd.d D10, U2, U4, D10 + xvfmadd.d D11, U3, U4, D11 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + xvfmadd.d D14, U2, U4, D14 + xvfmadd.d D15, U3, U4, D15 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x20 + + /***8-8***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + xvfmadd.d D6, U2, U4, D6 + xvfmadd.d D7, U3, U4, D7 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + xvfmadd.d D10, U2, U4, D10 + xvfmadd.d D11, U3, U4, D11 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + xvfmadd.d D14, U2, U4, D14 + xvfmadd.d D15, U3, U4, D15 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x20 + + addi.d TL, TL, -1 /* TL-- */ + blt ZERO,TL, .L_N5_TL1 + +.L_N5_L7: + /* if (!(L & 7)) goto L_N5_L0 */ + andi TL, L, 7 + beq TL, ZERO,.L_N5_L0 + +.L_N5_L71: + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + xvfmadd.d D6, U2, U4, D6 + xvfmadd.d D7, U3, U4, D7 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + xvfmadd.d D10, U2, U4, D10 + xvfmadd.d D11, U3, U4, D11 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + xvfmadd.d D14, U2, U4, D14 + xvfmadd.d D15, U3, U4, D15 + + /* Add stride for A0, B0 */ + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x20 + + addi.d TL, TL, -1 + blt ZERO,TL, .L_N5_L71 + +.L_N5_L0: +#if defined(TRMMKERNEL) + xvfmul.d D0, D0, VALPHA + xvfmul.d D1, D1, VALPHA + xvfmul.d D2, D2, VALPHA + xvfmul.d D3, D3, VALPHA + xvfmul.d D4, D4, VALPHA + xvfmul.d D5, D5, VALPHA + xvfmul.d D6, D6, VALPHA + xvfmul.d D7, D7, VALPHA + xvfmul.d D8, D8, VALPHA + xvfmul.d D9, D9, VALPHA + xvfmul.d D10, D10, VALPHA + xvfmul.d D11, D11, VALPHA + xvfmul.d D12, D12, VALPHA + xvfmul.d D13, D13, VALPHA + xvfmul.d D14, D14, VALPHA + xvfmul.d D15, D15, VALPHA +#else + /* Load C0 */ + xvld U0, C0, 0x00 + xvld U1, C0, 0x20 + xvld U2, C0, 0x40 + xvld U3, C0, 0x60 + xvfmadd.d D0, D0, VALPHA, U0 /* D0 = U0 + (D0 * VALPHA) */ + xvfmadd.d D1, D1, VALPHA, U1 + xvfmadd.d D2, D2, VALPHA, U2 + xvfmadd.d D3, D3, VALPHA, U3 + + /* Load C1 */ + xvld U0, C1, 0x00 + xvld U1, C1, 0x20 + xvld U2, C1, 0x40 + xvld U3, C1, 0x60 + xvfmadd.d D4, D4, VALPHA, U0 + xvfmadd.d D5, D5, VALPHA, U1 + xvfmadd.d D6, D6, VALPHA, U2 + xvfmadd.d D7, D7, VALPHA, U3 + + /* Load C2 */ + xvld U0, C2, 0x00 + xvld U1, C2, 0x20 + xvld U2, C2, 0x40 + xvld U3, C2, 0x60 + xvfmadd.d D8, D8, VALPHA, U0 + xvfmadd.d D9, D9, VALPHA, U1 + xvfmadd.d D10, D10, VALPHA, U2 + xvfmadd.d D11, D11, VALPHA, U3 + + /* Load C3 */ + xvld U0, C3, 0x00 + xvld U1, C3, 0x20 + xvld U2, C3, 0x40 + xvld U3, C3, 0x60 + xvfmadd.d D12, D12, VALPHA, U0 + xvfmadd.d D13, D13, VALPHA, U1 + xvfmadd.d D14, D14, VALPHA, U2 + xvfmadd.d D15, D15, VALPHA, U3 + #endif + + /* Store C0 */ + xvst D0, C0, 0x00 + xvst D1, C0, 0x20 + xvst D2, C0, 0x40 + xvst D3, C0, 0x60 + /* Store C1 */ + xvst D4, C1, 0x00 + xvst D5, C1, 0x20 + xvst D6, C1, 0x40 + xvst D7, C1, 0x60 + /* Store C2 */ + xvst D8, C2, 0x00 + xvst D9, C2, 0x20 + xvst D10, C2, 0x40 + xvst D11, C2, 0x60 + /* Store C3 */ + xvst D12, C3, 0x00 + xvst D13, C3, 0x20 + xvst D14, C3, 0x40 + xvst D15, C3, 0x60 + + /* Add stride for C */ + addi.d C0, C0, 0x80 + addi.d C1, C1, 0x80 + addi.d C2, C2, 0x80 + addi.d C3, C3, 0x80 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub.d L, K, OFF +#ifdef LEFT + /* number of values in A */ + addi.d L, L, -16 +#else + /* number of values in B */ + addi.d L, L, -4 +#endif + slli.d T0, L, 0x07 + add.d A0, A0, T0 + slli.d T0, L, 0x05 + add.d B0, B0, T0 +#endif + +#ifdef LEFT + addi.d OFF, OFF, 0x10 +#endif +#endif // #if defined(TRMMKERNEL) + + addi.d I, I, -1 /* I-- */ + blt ZERO,I, .L_N5_I1 + +.L_N5_M8: + /* We have done M & 16, considering M=8/4/2/1 */ + andi I, M, 15 + beq ZERO,I, .L_N5_M0 + + andi I, M, 8 + beq ZERO,I, .L_N5_M4 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move B0, B +#else + slli.d T0, OFF, 0x06 + add.d A0, A0, T0 + slli.d T0, OFF, 0x05 + add.d B0, B, T0 +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub.d L, K, OFF +#elif defined(LEFT) + /* number of values in A */ + addi.d L, OFF, 8 +#else + /* number of values in B */ + addi.d L, OFF, 4 +#endif +#else // #if !defined(TRMMKERNEL) + move B0, B + move L, K /* L = bk */ +#endif // #if defined(TRMMKERNEL) + + /* Load 8 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + /* line 1 */ + xvfmul.d D0, U0, U4 + xvfmul.d D1, U1, U4 + + xvldrepl.d U4, B0, 0x08 + /* line 2 */ + xvfmul.d D4, U0, U4 + xvfmul.d D5, U1, U4 + + xvldrepl.d U4, B0, 0x10 + /* line 3 */ + xvfmul.d D8, U0, U4 + xvfmul.d D9, U1, U4 + + xvldrepl.d U4, B0, 0x18 + /* line 4 */ + xvfmul.d D12, U0, U4 + xvfmul.d D13, U1, U4 + + /* Add stride for A0 and B0 */ + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x20 + /* Reduce L */ + addi.d L, L, -1 + srai.d TL, L, 3 /* TL = (L-1) >> 3 */ + /* if (TL < 1) goto L_N5_M8_L7 */ + beq ZERO,TL, .L_N5_M8_L7 + +.L_N5_M8_TL1: /* TL-- */ + /***8-1***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x20 + + /***8-2***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x20 + + /***8-3***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x20 + + /***8-4***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x20 + + /***8-5***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + /* Cumulative D0~D23 */ + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x20 + + /***8-6***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x20 + + /***8-7***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x20 + + /***8-8***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x20 + + addi.d TL, TL, -1 /* TL-- */ + blt ZERO,TL, .L_N5_M8_TL1 + +.L_N5_M8_L7: + /* if (!(L & 7)) goto L_N5_M8_L0 */ + andi TL, L, 7 + beq TL, ZERO,.L_N5_M8_L0 + +.L_N5_M8_L71: + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + xvfmadd.d D9, U1, U4, D9 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + xvfmadd.d D13, U1, U4, D13 + + /* Add stride for A0, B0 */ + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x20 + + addi.d TL, TL, -1 + blt ZERO,TL, .L_N5_M8_L71 + +.L_N5_M8_L0: +#if defined(TRMMKERNEL) + xvfmul.d D0, D0, VALPHA + xvfmul.d D1, D1, VALPHA + xvfmul.d D4, D4, VALPHA + xvfmul.d D5, D5, VALPHA + xvfmul.d D8, D8, VALPHA + xvfmul.d D9, D9, VALPHA + xvfmul.d D12, D12, VALPHA + xvfmul.d D13, D13, VALPHA +#else + /* Load C0 */ + xvld U0, C0, 0x00 + xvld U1, C0, 0x20 + xvfmadd.d D0, D0, VALPHA, U0 /* D0 = U0 + (D0 * VALPHA) */ + xvfmadd.d D1, D1, VALPHA, U1 + + /* Load C1 */ + xvld U0, C1, 0x00 + xvld U1, C1, 0x20 + xvfmadd.d D4, D4, VALPHA, U0 + xvfmadd.d D5, D5, VALPHA, U1 + + /* Load C2 */ + xvld U0, C2, 0x00 + xvld U1, C2, 0x20 + xvfmadd.d D8, D8, VALPHA, U0 + xvfmadd.d D9, D9, VALPHA, U1 + + /* Load C3 */ + xvld U0, C3, 0x00 + xvld U1, C3, 0x20 + xvfmadd.d D12, D12, VALPHA, U0 + xvfmadd.d D13, D13, VALPHA, U1 +#endif + + /* Store C0 */ + xvst D0, C0, 0x00 + xvst D1, C0, 0x20 + /* Store C1 */ + xvst D4, C1, 0x00 + xvst D5, C1, 0x20 + /* Store C2 */ + xvst D8, C2, 0x00 + xvst D9, C2, 0x20 + /* Store C3 */ + xvst D12, C3, 0x00 + xvst D13, C3, 0x20 + + /* Add stride for C */ + addi.d C0, C0, 0x40 + addi.d C1, C1, 0x40 + addi.d C2, C2, 0x40 + addi.d C3, C3, 0x40 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub.d L, K, OFF +#ifdef LEFT + /* number of values in A */ + addi.d L, L, -8 +#else + /* number of values in B */ + addi.d L, L, -4 +#endif + slli.d T0, L, 0x06 + add.d A0, A0, T0 + slli.d T0, L, 0x05 + add.d B0, B0, T0 +#endif + +#ifdef LEFT + /* number of values in A */ + addi.d OFF, OFF, 0x08 +#endif +#endif // #if defined(TRMMKERNEL) + +/********LOOP (if(N & 4 ) && (M & 8) ) End************/ + +.L_N5_M4: + andi I, M, 4 + beq ZERO,I, .L_N5_M2 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move B0, B +#else + slli.d T0, OFF, 0x05 + add.d A0, A0, T0 + add.d B0, B, T0 +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub.d L, K, OFF +#elif defined(LEFT) + /* number of values in A */ + addi.d L, OFF, 4 +#else + /* number of values in B */ + addi.d L, OFF, 4 +#endif +#else // #if !defined(TRMMKERNEL) + move B0, B + move L, K /* L = bk */ +#endif + + /* Load 4 * 64 from A0 */ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + /* line 1 */ + xvfmul.d D0, U0, U4 + + xvldrepl.d U4, B0, 0x08 + /* line 2 */ + xvfmul.d D4, U0, U4 + + xvldrepl.d U4, B0, 0x10 + /* line 3 */ + xvfmul.d D8, U0, U4 + + xvldrepl.d U4, B0, 0x18 + /* line 4 */ + xvfmul.d D12, U0, U4 + + /* Add stride for A0 and B0 */ + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x20 + /* Reduce L */ + addi.d L, L, -1 + srai.d TL, L, 3 /* TL = (L-1) >> 3 */ + /* if (TL < 1) goto L_N5_M4_L7 */ + beq ZERO,TL, .L_N5_M4_L7 + +.L_N5_M4_TL1: /* TL-- */ + /***8-1***/ + /* Load 8 * 64 from A0 */ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x20 + + /***8-2***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x20 + + /***8-3***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x20 + + /***8-4***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x20 + + /***8-5***/ + xvld U0, A0, 0x00 + + /* Cumulative D0~D23 */ + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x20 + + /***8-6***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x20 + + /***8-7***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x20 + + /***8-8***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x20 + + addi.d TL, TL, -1 /* TL-- */ + blt ZERO,TL, .L_N5_M4_TL1 + +.L_N5_M4_L7: + /* if (!(L & 7)) goto L_N5_M4_L0 */ + andi TL, L, 7 + beq TL, ZERO,.L_N5_M4_L0 + +.L_N5_M4_L71: + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + /* Add stride for A0, B0 */ + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x20 + + addi.d TL, TL, -1 + blt ZERO,TL, .L_N5_M4_L71 + +.L_N5_M4_L0: +#if defined(TRMMKERNEL) + xvfmul.d D0, D0, VALPHA + xvfmul.d D4, D4, VALPHA + xvfmul.d D8, D8, VALPHA + xvfmul.d D12, D12, VALPHA +#else + /* Load C0 */ + xvld U0, C0, 0x00 + xvfmadd.d D0, D0, VALPHA, U0 /* D0 = U0 + (D0 * VALPHA) */ + + /* Load C1 */ + xvld U0, C1, 0x00 + xvfmadd.d D4, D4, VALPHA, U0 + + /* Load C2 */ + xvld U0, C2, 0x00 + xvfmadd.d D8, D8, VALPHA, U0 + + /* Load C3 */ + xvld U0, C3, 0x00 + xvfmadd.d D12, D12, VALPHA, U0 +#endif + + /* Store C0 */ + xvst D0, C0, 0x00 + /* Store C1 */ + xvst D4, C1, 0x00 + /* Store C2 */ + xvst D8, C2, 0x00 + /* Store C3 */ + xvst D12, C3, 0x00 + + /* Add stride for C */ + addi.d C0, C0, 0x20 + addi.d C1, C1, 0x20 + addi.d C2, C2, 0x20 + addi.d C3, C3, 0x20 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub.d L, K, OFF +#ifdef LEFT + /* number of values in A */ + addi.d L, L, -4 +#else + /* number of values in B */ + addi.d L, L, -4 +#endif + slli.d T0, L, 0x05 + add.d A0, A0, T0 + add.d B0, B0, T0 +#endif + +#ifdef LEFT + /* number of values in A */ + addi.d OFF, OFF, 0x04 +#endif +#endif // #if defined(TRMMKERNEL) + +/********LOOP (if(N & 4 ) && (M & 4) ) End************/ + +.L_N5_M2: + andi I, M, 2 + beq ZERO,I, .L_N5_M1 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move B0, B +#else + slli.d T0, OFF, 0x04 + add.d A0, A0, T0 + slli.d T0, OFF, 0x05 + add.d B0, B, T0 +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub.d L, K, OFF +#elif defined(LEFT) + /* number of values in A */ + addi.d L, OFF, 2 +#else + /* number of values in B */ + addi.d L, OFF, 4 +#endif +#else // #if !defined(TRMMKERNEL) + move B0, B + move L, K /* L = bk */ +#endif + + /* Load 2 * 64 from A0 */ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + /* line 1 */ + xvfmul.d D0, U0, U4 + + xvldrepl.d U4, B0, 0x08 + /* line 2 */ + xvfmul.d D4, U0, U4 + + xvldrepl.d U4, B0, 0x10 + /* line 3 */ + xvfmul.d D8, U0, U4 + + xvldrepl.d U4, B0, 0x18 + /* line 4 */ + xvfmul.d D12, U0, U4 + + /* Add stride for A0 and B0 */ + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x20 + /* Reduce L */ + addi.d L, L, -1 + srai.d TL, L, 3 /* TL = (L-1) >> 3 */ + /* if (TL < 1) goto L_N5_M2_L7 */ + beq ZERO,TL, .L_N5_M2_L7 + +.L_N5_M2_TL1: /* TL-- */ + /***8-1***/ + /* Load 2 * 64 from A0 */ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x20 + + /***8-2***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x20 + + /***8-3***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x20 + + /***8-4***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x20 + + /***8-5***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x20 + + /***8-6***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x20 + + /***8-7***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x20 + + /***8-8***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x20 + + addi.d TL, TL, -1 /* TL-- */ + blt ZERO,TL, .L_N5_M2_TL1 + +.L_N5_M2_L7: + /* if (!(L & 7)) goto L_N5_M2_L0 */ + andi TL, L, 7 + beq TL, ZERO,.L_N5_M2_L0 + +.L_N5_M2_L71: + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + /* Add stride for A0, B0 */ + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x20 + + addi.d TL, TL, -1 + blt ZERO,TL, .L_N5_M2_L71 + +.L_N5_M2_L0: +#if defined(TRMMKERNEL) + xvfmul.d D0, D0, VALPHA + xvfmul.d D4, D4, VALPHA + xvfmul.d D8, D8, VALPHA + xvfmul.d D12, D12, VALPHA + #else + /* Load C0 */ + xvld U0, C0, 0x00 + xvfmadd.d D0, D0, VALPHA, U0 /* D0 = U0 + (D0 * VALPHA) */ + + /* Load C1 */ + xvld U0, C1, 0x00 + xvfmadd.d D4, D4, VALPHA, U0 + + /* Load C2 */ + xvld U0, C2, 0x00 + xvfmadd.d D8, D8, VALPHA, U0 + + /* Load C3 */ + xvld U0, C3, 0x00 + xvfmadd.d D12, D12, VALPHA, U0 + #endif + + xvstelm.d D0, C0, 0x00, 0x00 + xvstelm.d D4, C1, 0x00, 0x00 + xvstelm.d D8, C2, 0x00, 0x00 + xvstelm.d D12, C3, 0x00, 0x00 + xvstelm.d D0, C0, 0x08, 0x01 + xvstelm.d D4, C1, 0x08, 0x01 + xvstelm.d D8, C2, 0x08, 0x01 + xvstelm.d D12, C3, 0x08, 0x01 + + /* Add stride for C */ + addi.d C0, C0, 0x10 + addi.d C1, C1, 0x10 + addi.d C2, C2, 0x10 + addi.d C3, C3, 0x10 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub.d L, K, OFF +#ifdef LEFT + /* number of values in A */ + addi.d L, L, -2 +#else + /* number of values in B */ + addi.d L, L, -4 +#endif + slli.d T0, L, 0x04 + add.d A0, A0, T0 + slli.d T0, L, 0x05 + add.d B0, B0, T0 +#endif + +#ifdef LEFT + /* number of values in A */ + addi.d OFF, OFF, 0x02 +#endif +#endif // #if defined(TRMMKERNEL) + +/********LOOP (if(N & 4 ) && (M & 2) ) End************/ + +.L_N5_M1: + andi I, M, 1 + beq ZERO,I, .L_N5_M0 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move B0, B +#else + slli.d T0, OFF, 0x03 + add.d A0, A0, T0 + slli.d T0, OFF, 0x05 + add.d B0, B, T0 +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub.d L, K, OFF +#elif defined(LEFT) + /* number of values in A */ + addi.d L, OFF, 1 +#else + /* number of values in B */ + addi.d L, OFF, 4 +#endif +#else // #if !defined(TRMMKERNEL) + move B0, B + move L, K /* L = bk */ +#endif + + /* Load 1 * 64 from A0 */ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + /* line 1 */ + xvfmul.d D0, U0, U4 + + xvldrepl.d U4, B0, 0x08 + /* line 2 */ + xvfmul.d D4, U0, U4 + + xvldrepl.d U4, B0, 0x10 + /* line 3 */ + xvfmul.d D8, U0, U4 + + xvldrepl.d U4, B0, 0x18 + /* line 4 */ + xvfmul.d D12, U0, U4 + + /* Add stride for A0 and B0 */ + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x20 + /* Reduce L */ + addi.d L, L, -1 + srai.d TL, L, 3 /* TL = (L-1) >> 3 */ + /* if (TL < 1) goto L_N5_M1_L7 */ + beq ZERO,TL, .L_N5_M1_L7 + +.L_N5_M1_TL1: /* TL-- */ + /***8-1***/ + /* Load 1 * 64 from A0 */ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x20 + + /***8-2***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x20 + + /***8-3***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x20 + + /***8-4***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x20 + + /***8-5***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x20 + + /***8-6***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x20 + + /***8-7***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x20 + + /***8-8***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x20 + + addi.d TL, TL, -1 /* TL-- */ + blt ZERO,TL, .L_N5_M1_TL1 + +.L_N5_M1_L7: + /* if (!(L & 7)) goto L_N5_M1_L0 */ + andi TL, L, 7 + beq TL, ZERO,.L_N5_M1_L0 + +.L_N5_M1_L71: + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + xvldrepl.d U4, B0, 0x10 + xvfmadd.d D8, U0, U4, D8 + + xvldrepl.d U4, B0, 0x18 + xvfmadd.d D12, U0, U4, D12 + + /* Add stride for A0, B0 */ + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x20 + + addi.d TL, TL, -1 + blt ZERO,TL, .L_N5_M1_L71 + +.L_N5_M1_L0: +#if defined(TRMMKERNEL) + xvfmul.d D0, D0, VALPHA + xvfmul.d D4, D4, VALPHA + xvfmul.d D8, D8, VALPHA + xvfmul.d D12, D12, VALPHA + #else + /* Load C0 */ + xvld U0, C0, 0x00 + xvfmadd.d D0, D0, VALPHA, U0 /* D0 = U0 + (D0 * VALPHA) */ + + /* Load C1 */ + xvld U0, C1, 0x00 + xvfmadd.d D4, D4, VALPHA, U0 + + /* Load C2 */ + xvld U0, C2, 0x00 + xvfmadd.d D8, D8, VALPHA, U0 + + /* Load C3 */ + xvld U0, C3, 0x00 + xvfmadd.d D12, D12, VALPHA, U0 +#endif + + xvstelm.d D0, C0, 0x00, 0x00 + xvstelm.d D4, C1, 0x00, 0x00 + xvstelm.d D8, C2, 0x00, 0x00 + xvstelm.d D12, C3, 0x00, 0x00 + + /* Add stride for C */ + addi.d C0, C0, 0x08 + addi.d C1, C1, 0x08 + addi.d C2, C2, 0x08 + addi.d C3, C3, 0x08 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub.d L, K, OFF +#ifdef LEFT + /* number of values in A */ + addi.d L, L, -1 +#else + /* number of values in B */ + addi.d L, L, -4 +#endif + slli.d T0, L, 0x03 + add.d A0, A0, T0 + slli.d T0, L, 0x05 + add.d B0, B0, T0 +#endif + +#ifdef LEFT + /* number of values in A */ + addi.d OFF, OFF, 0x01 +#endif +#endif // #if defined(TRMMKERNEL) + +/********LOOP (if(N & 4 ) && (M & 1) ) End************/ + +.L_N5_M0: + /* Add stride for B and C + * B += (K * 32) + * C += (LDC * 32) + */ + /* since the array type is double, + * so we must mul 32 + */ + addi.d T2, ZERO,32 + mul.d T0, K, T2 + mul.d T1, LDC, T2 + add.d B, B, T0 + add.d C, C, T1 + +#if defined(TRMMKERNEL) && !defined(LEFT) + addi.d OFF, OFF, 0x04 +#endif + + /* We must reinit I */ + srai.d I, M, 4 /* I = bm >> 4 */ + +/************************* Condition 2 if((N & 4) && (M >> 4)) End !!! ************************* +* dgemm_core_16x4 */ + +.L_N3: + andi J, N, 2 + beq ZERO, J, .L_N1 + +/************************* Condition 3 if((N & 2) && (M >> 4)) START !!! ************************* +* dgemm_core_16x2 */ + + move C0, C + move A0, A + slli.d T0, LDC, 3 + add.d C1, C0, T0 + +#if defined(TRMMKERNEL) && defined(LEFT) + move OFF, OFFSET +#endif + + /* if (!(M >> 4)) goto L_N3_M8 */ + srai.d I, M, 4 /* I = bm >> 4 */ + beq ZERO, I, .L_N3_M8 + +.L_N3_I1: +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move B0, B +#else + slli.d T0, OFF, 0x07 + add.d A0, A0, T0 + slli.d T0, OFF, 0x04 + add.d B0, B, T0 +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub.d L, K, OFF +#elif defined(LEFT) + /* number of values in A */ + addi.d L, OFF, 16 +#else + /* number of values in B */ + addi.d L, OFF, 2 +#endif +#else // #if !defined(TRMMKERNEL) + move B0, B + move L, K /* L = bk */ +#endif + + /* Load 16 * 64 from A0 + * U0 = {a3, a2, a1, a0} + * U1 = {a7, a6, a5, a4} + * U2 = {a11, a10, a9, a8} + * U3 = {a15, a14, a13, a12} + */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + /* line 1 */ + xvfmul.d D0, U0, U4 + xvfmul.d D1, U1, U4 + xvfmul.d D2, U2, U4 + xvfmul.d D3, U3, U4 + + xvldrepl.d U4, B0, 0x08 + /* line 2 */ + xvfmul.d D4, U0, U4 + xvfmul.d D5, U1, U4 + xvfmul.d D6, U2, U4 + xvfmul.d D7, U3, U4 + + /* Add stride for A0 and B0 */ + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x10 + /* Reduce L */ + addi.d L, L, -1 + srai.d TL, L, 3 /* TL = (L-1) >> 3 */ + /* if (TL < 1) goto L_N3_L7 */ + beq ZERO,TL, .L_N3_L7 + +.L_N3_TL1: /* TL-- */ + /***8-1***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + xvfmadd.d D6, U2, U4, D6 + xvfmadd.d D7, U3, U4, D7 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x10 + + /***8-2***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + xvfmadd.d D6, U2, U4, D6 + xvfmadd.d D7, U3, U4, D7 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x10 + + /***8-3***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + xvfmadd.d D6, U2, U4, D6 + xvfmadd.d D7, U3, U4, D7 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x10 + + /***8-4***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + xvfmadd.d D6, U2, U4, D6 + xvfmadd.d D7, U3, U4, D7 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x10 + + /***8-5***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + xvfmadd.d D6, U2, U4, D6 + xvfmadd.d D7, U3, U4, D7 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x10 + + /***8-6***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + xvfmadd.d D6, U2, U4, D6 + xvfmadd.d D7, U3, U4, D7 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x10 + + /***8-7***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + xvfmadd.d D6, U2, U4, D6 + xvfmadd.d D7, U3, U4, D7 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x10 + + /***8-8***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + xvfmadd.d D6, U2, U4, D6 + xvfmadd.d D7, U3, U4, D7 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x10 + + addi.d TL, TL, -1 /* TL-- */ + blt ZERO,TL, .L_N3_TL1 + +.L_N3_L7: + /* if (!(L & 7)) goto L_N3_L0 */ + andi TL, L, 7 + beq TL, ZERO,.L_N3_L0 + +.L_N3_L71: + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + xvfmadd.d D6, U2, U4, D6 + xvfmadd.d D7, U3, U4, D7 + + /* Add stride for A0, B0 */ + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x10 + + addi.d TL, TL, -1 + blt ZERO,TL, .L_N3_L71 + +.L_N3_L0: +#if defined(TRMMKERNEL) + xvfmul.d D0, D0, VALPHA + xvfmul.d D1, D1, VALPHA + xvfmul.d D2, D2, VALPHA + xvfmul.d D3, D3, VALPHA + xvfmul.d D4, D4, VALPHA + xvfmul.d D5, D5, VALPHA + xvfmul.d D6, D6, VALPHA + xvfmul.d D7, D7, VALPHA +#else + /* Load C0 */ + xvld U0, C0, 0x00 + xvld U1, C0, 0x20 + xvld U2, C0, 0x40 + xvld U3, C0, 0x60 + xvfmadd.d D0, D0, VALPHA, U0 /* D0 = U0 + (D0 * VALPHA) */ + xvfmadd.d D1, D1, VALPHA, U1 + xvfmadd.d D2, D2, VALPHA, U2 + xvfmadd.d D3, D3, VALPHA, U3 + + /* Load C1 */ + xvld U0, C1, 0x00 + xvld U1, C1, 0x20 + xvld U2, C1, 0x40 + xvld U3, C1, 0x60 + xvfmadd.d D4, D4, VALPHA, U0 + xvfmadd.d D5, D5, VALPHA, U1 + xvfmadd.d D6, D6, VALPHA, U2 + xvfmadd.d D7, D7, VALPHA, U3 +#endif + + /* Store C0 */ + xvst D0, C0, 0x00 + xvst D1, C0, 0x20 + xvst D2, C0, 0x40 + xvst D3, C0, 0x60 + /* Store C1 */ + xvst D4, C1, 0x00 + xvst D5, C1, 0x20 + xvst D6, C1, 0x40 + xvst D7, C1, 0x60 + + /* Add stride for C */ + addi.d C0, C0, 0x80 + addi.d C1, C1, 0x80 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub.d L, K, OFF +#ifdef LEFT + addi.d L, L, -16 +#else + addi.d L, L, -2 +#endif + slli.d T0, L, 0x07 + add.d A0, A0, T0 + slli.d T0, L, 0x04 + add.d B0, B0, T0 +#endif + +#ifdef LEFT + addi.d OFF, OFF, 0x10 +#endif +#endif // #if defined(TRMMKERNEL) + + addi.d I, I, -1 /* I-- */ + blt ZERO,I, .L_N3_I1 + +.L_N3_M8: + /* We have done M & 16, considering M=8/4/2/1 */ + andi I, M, 15 + beq ZERO,I, .L_N3_M0 + + andi I, M, 8 + beq ZERO,I, .L_N3_M4 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move B0, B +#else + slli.d T0, OFF, 0x06 + add.d A0, A0, T0 + slli.d T0, OFF, 0x04 + add.d B0, B, T0 +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub.d L, K, OFF +#elif defined(LEFT) + /* number of values in A */ + addi.d L, OFF, 8 +#else + /* number of values in B */ + addi.d L, OFF, 2 +#endif +#else // #if !defined(TRMMKERNEL) + move B0, B + move L, K /* L = bk */ +#endif + + /* Load 8 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + /* line 1 */ + xvfmul.d D0, U0, U4 + xvfmul.d D1, U1, U4 + + xvldrepl.d U4, B0, 0x08 + /* line 2 */ + xvfmul.d D4, U0, U4 + xvfmul.d D5, U1, U4 + + /* Add stride for A0 and B0 */ + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x10 + /* Reduce L */ + addi.d L, L, -1 + srai.d TL, L, 3 /* TL = (L-1) >> 3 */ + /* if (TL < 1) goto L_N3_M8_L7 */ + beq ZERO,TL, .L_N3_M8_L7 + +.L_N3_M8_TL1: /* TL-- */ + /***8-1***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x10 + + /***8-2***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x10 + + /***8-3***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x10 + + /***8-4***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x10 + + /***8-5***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + /* Cumulative D0~D23 */ + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x10 + + /***8-6***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x10 + + /***8-7***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x10 + + /***8-8***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x10 + + addi.d TL, TL, -1 /* TL-- */ + blt ZERO,TL, .L_N3_M8_TL1 + +.L_N3_M8_L7: + /* if (!(L & 7)) goto L_N3_M8_L0 */ + andi TL, L, 7 + beq TL, ZERO,.L_N3_M8_L0 + +.L_N3_M8_L71: + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + xvfmadd.d D5, U1, U4, D5 + + /* Add stride for A0, B0 */ + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x10 + + addi.d TL, TL, -1 + blt ZERO,TL, .L_N3_M8_L71 + +.L_N3_M8_L0: +#if defined(TRMMKERNEL) + xvfmul.d D0, D0, VALPHA + xvfmul.d D1, D1, VALPHA + xvfmul.d D4, D4, VALPHA + xvfmul.d D5, D5, VALPHA +#else + /* Load C0 */ + xvld U0, C0, 0x00 + xvld U1, C0, 0x20 + xvfmadd.d D0, D0, VALPHA, U0 /* D0 = U0 + (D0 * VALPHA) */ + xvfmadd.d D1, D1, VALPHA, U1 + + /* Load C1 */ + xvld U0, C1, 0x00 + xvld U1, C1, 0x20 + xvfmadd.d D4, D4, VALPHA, U0 + xvfmadd.d D5, D5, VALPHA, U1 +#endif + + /* Store C0 */ + xvst D0, C0, 0x00 + xvst D1, C0, 0x20 + /* Store C1 */ + xvst D4, C1, 0x00 + xvst D5, C1, 0x20 + + /* Add stride for C */ + addi.d C0, C0, 0x40 + addi.d C1, C1, 0x40 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub.d L, K, OFF +#ifdef LEFT + addi.d L, L, -8 +#else + addi.d L, L, -2 +#endif + slli.d T0, L, 0x06 + add.d A0, A0, T0 + slli.d T0, L, 0x04 + add.d B0, B0, T0 +#endif + +#ifdef LEFT + addi.d OFF, OFF, 0x08 +#endif +#endif // #if defined(TRMMKERNEL) + +/********LOOP (if(N & 2) && (M & 8) ) End************/ + +.L_N3_M4: + andi I, M, 4 + beq ZERO,I, .L_N3_M2 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move B0, B +#else + slli.d T0, OFF, 0x05 + add.d A0, A0, T0 + slli.d T0, OFF, 0x04 + add.d B0, B, T0 +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub.d L, K, OFF +#elif defined(LEFT) + /* number of values in A */ + addi.d L, OFF, 4 +#else + /* number of values in B */ + addi.d L, OFF, 2 +#endif +#else // #if !defined(TRMMKERNEL) + move B0, B + move L, K /* L = bk */ +#endif + + /* Load 4 * 64 from A0 */ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + /* line 1 */ + xvfmul.d D0, U0, U4 + + xvldrepl.d U4, B0, 0x08 + /* line 2 */ + xvfmul.d D4, U0, U4 + + /* Add stride for A0 and B0 */ + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x10 + /* Reduce L */ + addi.d L, L, -1 + srai.d TL, L, 3 /* TL = (L-1) >> 3 */ + /* if (TL < 1) goto L_N3_M4_L7 */ + beq ZERO,TL, .L_N3_M4_L7 + +.L_N3_M4_TL1: /* TL-- */ + /***8-1***/ + /* Load 8 * 64 from A0 */ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x10 + + /***8-2***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x10 + + /***8-3***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x10 + + /***8-4***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x10 + + /***8-5***/ + xvld U0, A0, 0x00 + + /* Cumulative D0~D23 */ + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x10 + + /***8-6***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x10 + + /***8-7***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x10 + + /***8-8***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x10 + + addi.d TL, TL, -1 /* TL-- */ + blt ZERO,TL, .L_N3_M4_TL1 + +.L_N3_M4_L7: + /* if (!(L & 7)) goto L_N3_M4_L0 */ + andi TL, L, 7 + beq TL, ZERO,.L_N3_M4_L0 + +.L_N3_M4_L71: + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + /* Add stride for A0, B0 */ + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x10 + + addi.d TL, TL, -1 + blt ZERO,TL, .L_N3_M4_L71 + +.L_N3_M4_L0: +#if defined(TRMMKERNEL) + xvfmul.d D0, D0, VALPHA + xvfmul.d D4, D4, VALPHA +#else + /* Load C0 */ + xvld U0, C0, 0x00 + xvfmadd.d D0, D0, VALPHA, U0 /* D0 = U0 + (D0 * VALPHA) */ + + /* Load C1 */ + xvld U0, C1, 0x00 + xvfmadd.d D4, D4, VALPHA, U0 +#endif + + /* Store C0 */ + xvst D0, C0, 0x00 + /* Store C1 */ + xvst D4, C1, 0x00 + + /* Add stride for C */ + addi.d C0, C0, 0x20 + addi.d C1, C1, 0x20 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub.d L, K, OFF +#ifdef LEFT + addi.d L, L, -4 +#else + addi.d L, L, -2 +#endif + slli.d T0, L, 0x05 + add.d A0, A0, T0 + slli.d T0, L, 0x04 + add.d B0, B0, T0 +#endif + +#ifdef LEFT + addi.d OFF, OFF, 0x04 +#endif +#endif // #if defined(TRMMKERNEL) + +/********LOOP (if(N & 2 ) && (M & 4) ) End************/ + +.L_N3_M2: + andi I, M, 2 + beq ZERO,I, .L_N3_M1 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move B0, B +#else + slli.d T0, OFF, 0x04 + add.d A0, A0, T0 + add.d B0, B, T0 +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub.d L, K, OFF +#elif defined(LEFT) + /* number of values in A */ + addi.d L, OFF, 2 +#else + /* number of values in B */ + addi.d L, OFF, 2 +#endif +#else // #if !defined(TRMMKERNEL) + move B0, B + move L, K /* L = bk */ +#endif + + /* Load 2 * 64 from A0 */ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + /* line 1 */ + xvfmul.d D0, U0, U4 + + xvldrepl.d U4, B0, 0x08 + /* line 2 */ + xvfmul.d D4, U0, U4 + + /* Add stride for A0 and B0 */ + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x10 + /* Reduce L */ + addi.d L, L, -1 + srai.d TL, L, 3 /* TL = (L-1) >> 3 */ + /* if (TL < 1) goto L_N3_M2_L7 */ + beq ZERO,TL, .L_N3_M2_L7 + +.L_N3_M2_TL1: /* TL-- */ + /***8-1***/ + /* Load 2 * 64 from A0 */ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x10 + + /***8-2***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x10 + + /***8-3***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x10 + + /***8-4***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x10 + + /***8-5***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x10 + + /***8-6***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x10 + + /***8-7***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x10 + + /***8-8***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x10 + + addi.d TL, TL, -1 /* TL-- */ + blt ZERO,TL, .L_N3_M2_TL1 + +.L_N3_M2_L7: + /* if (!(L & 7)) goto L_N3_M2_L0 */ + andi TL, L, 7 + beq TL, ZERO,.L_N3_M2_L0 + +.L_N3_M2_L71: + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + /* Add stride for A0, B0 */ + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x10 + + addi.d TL, TL, -1 + blt ZERO,TL, .L_N3_M2_L71 + +.L_N3_M2_L0: +#if defined(TRMMKERNEL) + xvfmul.d D0, D0, VALPHA + xvfmul.d D4, D4, VALPHA +#else + /* Load C0 */ + xvld U0, C0, 0x00 + xvfmadd.d D0, D0, VALPHA, U0 /* D0 = U0 + (D0 * VALPHA) */ + + /* Load C1 */ + xvld U0, C1, 0x00 + xvfmadd.d D4, D4, VALPHA, U0 +#endif + + xvstelm.d D0, C0, 0x00, 0x00 + xvstelm.d D4, C1, 0x00, 0x00 + xvstelm.d D0, C0, 0x08, 0x01 + xvstelm.d D4, C1, 0x08, 0x01 + + /* Add stride for C */ + addi.d C0, C0, 0x10 + addi.d C1, C1, 0x10 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub.d L, K, OFF +#ifdef LEFT + addi.d L, L, -2 +#else + addi.d L, L, -2 +#endif + slli.d T0, L, 0x04 + add.d A0, A0, T0 + add.d B0, B0, T0 +#endif + +#ifdef LEFT + addi.d OFF, OFF, 0x02 +#endif +#endif // #if defined(TRMMKERNEL) + +/********LOOP (if(N & 2 ) && (M & 2) ) End************/ + +.L_N3_M1: + andi I, M, 1 + beq ZERO,I, .L_N3_M0 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move B0, B +#else + slli.d T0, OFF, 0x03 + add.d A0, A0, T0 + slli.d T0, OFF, 0x04 + add.d B0, B, T0 +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub.d L, K, OFF +#elif defined(LEFT) + /* number of values in A */ + addi.d L, OFF, 1 +#else + /* number of values in B */ + addi.d L, OFF, 2 +#endif +#else // #if !defined(TRMMKERNEL) + move B0, B + move L, K /* L = bk */ +#endif + + /* Load 1 * 64 from A0 */ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + /* line 1 */ + xvfmul.d D0, U0, U4 + + xvldrepl.d U4, B0, 0x08 + /* line 2 */ + xvfmul.d D4, U0, U4 + + /* Add stride for A0 and B0 */ + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x10 + /* Reduce L */ + addi.d L, L, -1 + srai.d TL, L, 3 /* TL = (L-1) >> 3 */ + /* if (TL < 1) goto L_N3_M1_L7 */ + beq ZERO,TL, .L_N3_M1_L7 + +.L_N3_M1_TL1: /* TL-- */ + /***8-1***/ + /* Load 1 * 64 from A0 */ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x10 + + /***8-2***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x10 + + /***8-3***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x10 + + /***8-4***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x10 + + /***8-5***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x10 + + /***8-6***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x10 + + /***8-7***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x10 + + /***8-8***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x10 + + addi.d TL, TL, -1 /* TL-- */ + blt ZERO,TL, .L_N3_M1_TL1 + +.L_N3_M1_L7: + /* if (!(L & 7)) goto L_N3_M1_L0 */ + andi TL, L, 7 + beq TL, ZERO,.L_N3_M1_L0 + +.L_N3_M1_L71: + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + xvldrepl.d U4, B0, 0x08 + xvfmadd.d D4, U0, U4, D4 + + /* Add stride for A0, B0 */ + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x10 + + addi.d TL, TL, -1 + blt ZERO,TL, .L_N3_M1_L71 + +.L_N3_M1_L0: +#if defined(TRMMKERNEL) + xvfmul.d D0, D0, VALPHA + xvfmul.d D4, D4, VALPHA +#else + /* Load C0 */ + xvld U0, C0, 0x00 + xvfmadd.d D0, D0, VALPHA, U0 /* D0 = U0 + (D0 * VALPHA) */ + + /* Load C1 */ + xvld U0, C1, 0x00 + xvfmadd.d D4, D4, VALPHA, U0 +#endif + + xvstelm.d D0, C0, 0x00, 0x00 + xvstelm.d D4, C1, 0x00, 0x00 + + /* Add stride for C */ + addi.d C0, C0, 0x08 + addi.d C1, C1, 0x08 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub.d L, K, OFF +#ifdef LEFT + addi.d L, L, -1 +#else + addi.d L, L, -2 +#endif + slli.d T0, L, 0x03 + add.d A0, A0, T0 + slli.d T0, L, 0x04 + add.d B0, B0, T0 +#endif + +#ifdef LEFT + addi.d OFF, OFF, 0x01 +#endif +#endif // #if defined(TRMMKERNEL) + +/********LOOP (if(N & 2 ) && (M & 1) ) End************/ + +.L_N3_M0: + /* Add stride for B and C + * B += (K * 16) + * C += (LDC * 16) + */ + /* since the array type is double, + * so we must mul 32 + */ + addi.d T2, ZERO,16 + mul.d T0, K, T2 + mul.d T1, LDC, T2 + add.d B, B, T0 + add.d C, C, T1 + +#if defined(TRMMKERNEL) && !defined(LEFT) + addi.d OFF, OFF, 0x02 +#endif + + /* We must reinit I */ + srai.d I, M, 4 /* I = bm >> 4 */ + +/************************* Condition 3 if((N & 2) && (M >> 4)) End !!! ************************* +* dgemm_core_16x2 */ + +.L_N1: + andi J, N, 1 + beq ZERO, J, .L_N0 + +/************************* Condition 4 if((N & 1) && (M >> 4)) START !!! ************************* +* dgemm_core_16x1 */ + + move C0, C + move A0, A + +#if defined(TRMMKERNEL) && defined(LEFT) + move OFF, OFFSET +#endif + + /* if (!(M >> 4)) goto L_N1_M8 */ + srai.d I, M, 4 /* I = bm >> 4 */ + beq ZERO, I, .L_N1_M8 + +.L_N1_I1: +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move B0, B +#else + slli.d T0, OFF, 0x07 + add.d A0, A0, T0 + slli.d T0, OFF, 0x03 + add.d B0, B, T0 +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub.d L, K, OFF +#elif defined(LEFT) + /* number of values in A */ + addi.d L, OFF, 16 +#else + /* number of values in B */ + addi.d L, OFF, 1 +#endif +#else // #if !defined(TRMMKERNEL) + move B0, B + move L, K /* L = bk */ +#endif + + /* Load 16 * 64 from A0 + * U0 = {a3, a2, a1, a0} + * U1 = {a7, a6, a5, a4} + * U2 = {a11, a10, a9, a8} + * U3 = {a15, a14, a13, a12} + */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + /* line 1 */ + xvfmul.d D0, U0, U4 + xvfmul.d D1, U1, U4 + xvfmul.d D2, U2, U4 + xvfmul.d D3, U3, U4 + + /* Add stride for A0 and B0 */ + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x08 + /* Reduce L */ + addi.d L, L, -1 + srai.d TL, L, 3 /* TL = (L-1) >> 3 */ + /* if (TL < 1) goto L_N1_L7 */ + beq ZERO,TL, .L_N1_L7 + +.L_N1_TL1: /* TL-- */ + /***8-1***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x08 + + /***8-2***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x08 + + /***8-3***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x08 + + /***8-4***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x08 + + /***8-5***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x08 + + /***8-6***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x08 + + /***8-7***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x08 + + /***8-8***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x08 + + addi.d TL, TL, -1 /* TL-- */ + blt ZERO,TL, .L_N1_TL1 + +.L_N1_L7: + /* if (!(L & 7)) goto L_N1_L0 */ + andi TL, L, 7 + beq TL, ZERO,.L_N1_L0 + +.L_N1_L71: + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + xvld U2, A0, 0x40 + xvld U3, A0, 0x60 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + xvfmadd.d D2, U2, U4, D2 + xvfmadd.d D3, U3, U4, D3 + + /* Add stride for A0, B0 */ + addi.d A0, A0, 0x80 + addi.d B0, B0, 0x08 + + addi.d TL, TL, -1 + blt ZERO,TL, .L_N1_L71 + +.L_N1_L0: +#if defined(TRMMKERNEL) + xvfmul.d D0, D0, VALPHA + xvfmul.d D1, D1, VALPHA + xvfmul.d D2, D2, VALPHA + xvfmul.d D3, D3, VALPHA +#else + /* Load C0 */ + xvld U0, C0, 0x00 + xvld U1, C0, 0x20 + xvld U2, C0, 0x40 + xvld U3, C0, 0x60 + xvfmadd.d D0, D0, VALPHA, U0 /* D0 = U0 + (D0 * VALPHA) */ + xvfmadd.d D1, D1, VALPHA, U1 + xvfmadd.d D2, D2, VALPHA, U2 + xvfmadd.d D3, D3, VALPHA, U3 +#endif + + /* Store C0 */ + xvst D0, C0, 0x00 + xvst D1, C0, 0x20 + xvst D2, C0, 0x40 + xvst D3, C0, 0x60 + + /* Add stride for C */ + addi.d C0, C0, 0x80 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub.d L, K, OFF +#ifdef LEFT + addi.d L, L, -16 +#else + addi.d L, L, -1 +#endif + slli.d T0, L, 0x07 + add.d A0, A0, T0 + slli.d T0, L, 0x03 + add.d B0, B0, T0 +#endif + +#ifdef LEFT + addi.d OFF, OFF, 0x10 +#endif +#endif // #if defined(TRMMKERNEL) + + addi.d I, I, -1 /* I-- */ + blt ZERO,I, .L_N1_I1 + +.L_N1_M8: + /* We have done M & 16, considering M=8/4/2/1 */ + andi I, M, 15 + beq ZERO,I, .L_N1_M0 + + andi I, M, 8 + beq ZERO,I, .L_N1_M4 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move B0, B +#else + slli.d T0, OFF, 0x06 + add.d A0, A0, T0 + slli.d T0, OFF, 0x03 + add.d B0, B, T0 +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub.d L, K, OFF +#elif defined(LEFT) + /* number of values in A */ + addi.d L, OFF, 8 +#else + /* number of values in B */ + addi.d L, OFF, 1 +#endif +#else // #if !defined(TRMMKERNEL) + move B0, B + move L, K /* L = bk */ +#endif + + /* Load 8 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + /* line 1 */ + xvfmul.d D0, U0, U4 + xvfmul.d D1, U1, U4 + + /* Add stride for A0 and B0 */ + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x08 + /* Reduce L */ + addi.d L, L, -1 + srai.d TL, L, 3 /* TL = (L-1) >> 3 */ + /* if (TL < 1) goto L_N1_M8_L7 */ + beq ZERO,TL, .L_N1_M8_L7 + +.L_N1_M8_TL1: /* TL-- */ + /***8-1***/ + /* Load 16 * 64 from A0 */ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x08 + + /***8-2***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x08 + + /***8-3***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x08 + + /***8-4***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x08 + + /***8-5***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x08 + + /***8-6***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x08 + + /***8-7***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x08 + + /***8-8***/ + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x08 + + addi.d TL, TL, -1 /* TL-- */ + blt ZERO,TL, .L_N1_M8_TL1 + +.L_N1_M8_L7: + /* if (!(L & 7)) goto L_N1_M8_L0 */ + andi TL, L, 7 + beq TL, ZERO,.L_N1_M8_L0 + +.L_N1_M8_L71: + xvld U0, A0, 0x00 + xvld U1, A0, 0x20 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + xvfmadd.d D1, U1, U4, D1 + + /* Add stride for A0, B0 */ + addi.d A0, A0, 0x40 + addi.d B0, B0, 0x08 + + addi.d TL, TL, -1 + blt ZERO,TL, .L_N1_M8_L71 + +.L_N1_M8_L0: +#if defined(TRMMKERNEL) + xvfmul.d D0, D0, VALPHA + xvfmul.d D1, D1, VALPHA +#else + /* Load C0 */ + xvld U0, C0, 0x00 + xvld U1, C0, 0x20 + xvfmadd.d D0, D0, VALPHA, U0 /* D0 = U0 + (D0 * VALPHA) */ + xvfmadd.d D1, D1, VALPHA, U1 +#endif + + /* Store C0 */ + xvst D0, C0, 0x00 + xvst D1, C0, 0x20 + + /* Add stride for C */ + addi.d C0, C0, 0x40 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub.d L, K, OFF +#ifdef LEFT + addi.d L, L, -8 +#else + addi.d L, L, -1 +#endif + slli.d T0, L, 0x06 + add.d A0, A0, T0 + slli.d T0, L, 0x03 + add.d B0, B0, T0 +#endif + +#ifdef LEFT + addi.d OFF, OFF, 0x08 +#endif +#endif // #if defined(TRMMKERNEL) + +/********LOOP (if(N & 2) && (M & 8) ) End************/ + +.L_N1_M4: + andi I, M, 4 + beq ZERO,I, .L_N1_M2 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move B0, B +#else + slli.d T0, OFF, 0x05 + add.d A0, A0, T0 + slli.d T0, OFF, 0x03 + add.d B0, B, T0 +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub.d L, K, OFF +#elif defined(LEFT) + /* number of values in A */ + addi.d L, OFF, 4 +#else + /* number of values in B */ + addi.d L, OFF, 1 +#endif +#else // #if !defined(TRMMKERNEL) + move B0, B + move L, K /* L = bk */ +#endif + + /* Load 4 * 64 from A0 */ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + /* line 1 */ + xvfmul.d D0, U0, U4 + + /* Add stride for A0 and B0 */ + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x08 + /* Reduce L */ + addi.d L, L, -1 + srai.d TL, L, 3 /* TL = (L-1) >> 3 */ + /* if (TL < 1) goto L_N1_M4_L7 */ + beq ZERO,TL, .L_N1_M4_L7 + +.L_N1_M4_TL1: /* TL-- */ + /***8-1***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x08 + + /***8-2***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x08 + + /***8-3***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x08 + + /***8-4***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x08 + + /***8-5***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x08 + + /***8-6***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x08 + + /***8-7***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x08 + + /***8-8***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x08 + + addi.d TL, TL, -1 /* TL-- */ + blt ZERO,TL, .L_N1_M4_TL1 + +.L_N1_M4_L7: + /* if (!(L & 7)) goto L_N1_M4_L0 */ + andi TL, L, 7 + beq TL, ZERO,.L_N1_M4_L0 + +.L_N1_M4_L71: + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + /* Add stride for A0, B0 */ + addi.d A0, A0, 0x20 + addi.d B0, B0, 0x08 + + addi.d TL, TL, -1 + blt ZERO,TL, .L_N1_M4_L71 + +.L_N1_M4_L0: +#if defined(TRMMKERNEL) + xvfmul.d D0, D0, VALPHA +#else + /* Load C0 */ + xvld U0, C0, 0x00 + xvfmadd.d D0, D0, VALPHA, U0 /* D0 = U0 + (D0 * VALPHA) */ + #endif + + /* Store C0 */ + xvst D0, C0, 0x00 + + /* Add stride for C */ + addi.d C0, C0, 0x20 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub.d L, K, OFF +#ifdef LEFT + addi.d L, L, -4 +#else + addi.d L, L, -1 +#endif + slli.d T0, L, 0x05 + add.d A0, A0, T0 + slli.d T0, L, 0x03 + add.d B0, B0, T0 +#endif + +#ifdef LEFT + addi.d OFF, OFF, 0x04 +#endif +#endif // #if defined(TRMMKERNEL) + +/********LOOP (if(N & 2 ) && (M & 4) ) End************/ + +.L_N1_M2: + andi I, M, 2 + beq ZERO,I, .L_N1_M1 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move B0, B +#else + slli.d T0, OFF, 0x04 + add.d A0, A0, T0 + slli.d T0, OFF, 0x03 + add.d B0, B, T0 +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub.d L, K, OFF +#elif defined(LEFT) + /* number of values in A */ + addi.d L, OFF, 2 +#else + /* number of values in B */ + addi.d L, OFF, 1 +#endif +#else // #if !defined(TRMMKERNEL) + move B0, B + move L, K /* L = bk */ +#endif + + /* Load 2 * 64 from A0 */ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + /* line 1 */ + xvfmul.d D0, U0, U4 + + /* Add stride for A0 and B0 */ + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x08 + /* Reduce L */ + addi.d L, L, -1 + srai.d TL, L, 3 /* TL = (L-1) >> 3 */ + /* if (TL < 1) goto L_N1_M2_L7 */ + beq ZERO,TL, .L_N1_M2_L7 + +.L_N1_M2_TL1: /* TL-- */ + /***8-1***/ + /* Load 2 * 64 from A0 */ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x08 + + /***8-2***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x08 + + /***8-3***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x08 + + /***8-4***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x08 + + /***8-5***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x08 + + /***8-6***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x08 + + /***8-7***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x08 + + /***8-8***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x08 + + addi.d TL, TL, -1 /* TL-- */ + blt ZERO,TL, .L_N1_M2_TL1 + +.L_N1_M2_L7: + /* if (!(L & 7)) goto L_N1_M2_L0 */ + andi TL, L, 7 + beq TL, ZERO,.L_N1_M2_L0 + +.L_N1_M2_L71: + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + /* Add stride for A0, B0 */ + addi.d A0, A0, 0x10 + addi.d B0, B0, 0x08 + + addi.d TL, TL, -1 + blt ZERO,TL, .L_N1_M2_L71 + +.L_N1_M2_L0: +#if defined(TRMMKERNEL) + xvfmul.d D0, D0, VALPHA +#else + /* Load C0 */ + xvld U0, C0, 0x00 + xvfmadd.d D0, D0, VALPHA, U0 /* D0 = U0 + (D0 * VALPHA) */ +#endif + + xvstelm.d D0, C0, 0x00, 0x00 + xvstelm.d D0, C0, 0x08, 0x01 + + /* Add stride for C */ + addi.d C0, C0, 0x10 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub.d L, K, OFF +#ifdef LEFT + addi.d L, L, -2 +#else + addi.d L, L, -1 +#endif + slli.d T0, L, 0x04 + add.d A0, A0, T0 + slli.d T0, L, 0x03 + add.d B0, B0, T0 +#endif + +#ifdef LEFT + addi.d OFF, OFF, 0x02 +#endif +#endif // #if defined(TRMMKERNEL) + +/********LOOP (if(N & 2 ) && (M & 2) ) End************/ + +.L_N1_M1: + andi I, M, 1 + beq ZERO,I, .L_N1_M0 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + move B0, B +#else + slli.d T0, OFF, 0x03 + add.d A0, A0, T0 + add.d B0, B, T0 +#endif + +#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + sub.d L, K, OFF +#elif defined(LEFT) + /* number of values in A */ + addi.d L, OFF, 1 +#else + /* number of values in B */ + addi.d L, OFF, 1 +#endif +#else // #if !defined(TRMMKERNEL) + move B0, B + move L, K /* L = bk */ +#endif + + /* Load 1 * 64 from A0 */ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + /* line 1 */ + xvfmul.d D0, U0, U4 + + /* Add stride for A0 and B0 */ + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x08 + /* Reduce L */ + addi.d L, L, -1 + srai.d TL, L, 3 /* TL = (L-1) >> 3 */ + /* if (TL < 1) goto L_N1_M1_L7 */ + beq ZERO,TL, .L_N1_M1_L7 + +.L_N1_M1_TL1: /* TL-- */ + /***8-1***/ + /* Load 1 * 64 from A0 */ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x08 + + /***8-2***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x08 + + /***8-3***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x08 + + /***8-4***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x08 + + /***8-5***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x08 + + /***8-6***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x08 + + /***8-7***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x08 + + /***8-8***/ + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x08 + + addi.d TL, TL, -1 /* TL-- */ + blt ZERO,TL, .L_N1_M1_TL1 + +.L_N1_M1_L7: + /* if (!(L & 7)) goto L_N1_M1_L0 */ + andi TL, L, 7 + beq TL, ZERO,.L_N1_M1_L0 + +.L_N1_M1_L71: + xvld U0, A0, 0x00 + + xvldrepl.d U4, B0, 0x00 + xvfmadd.d D0, U0, U4, D0 + + /* Add stride for A0, B0 */ + addi.d A0, A0, 0x08 + addi.d B0, B0, 0x08 + + addi.d TL, TL, -1 + blt ZERO,TL, .L_N1_M1_L71 + +.L_N1_M1_L0: +#if defined(TRMMKERNEL) + xvfmul.d D0, D0, VALPHA +#else + /* Load C0 */ + xvld U0, C0, 0x00 + xvfmadd.d D0, D0, VALPHA, U0 /* D0 = U0 + (D0 * VALPHA) */ +#endif + + xvstelm.d D0, C0, 0x00, 0x00 + + /* Add stride for C */ + addi.d C0, C0, 0x08 + +#if defined(TRMMKERNEL) +#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + sub.d L, K, OFF +#ifdef LEFT + addi.d L, L, -1 +#else + addi.d L, L, -1 +#endif + slli.d T0, L, 0x03 + add.d A0, A0, T0 + add.d B0, B0, T0 +#endif + +#ifdef LEFT + addi.d OFF, OFF, 0x01 +#endif +#endif // #if defined(TRMMKERNEL) + +/********LOOP (if(N & 2 ) && (M & 1) ) End************/ + +.L_N1_M0: + +/************************* Condition 4 if((N & 1) && (M >> 4)) End !!! ************************* +* dgemm_core_16x1 */ + +.L_N0: + /* Restore $r23~$31 */ + LDARG $r23, $sp, 0 + LDARG $r24, $sp, 8 + LDARG $r25, $sp, 16 + LDARG $r26, $sp, 24 + LDARG $r27, $sp, 32 + LDARG $r28, $sp, 40 + LDARG $r29, $sp, 48 + LDARG $r30, $sp, 56 + LDARG $r31, $sp, 64 + fld.d $f23, $sp, 72 + fld.d $f24, $sp, 80 + fld.d $f25, $sp, 96 + fld.d $f26, $sp, 104 + fld.d $f27, $sp, 112 + fld.d $f28, $sp, 120 + fld.d $f29, $sp, 128 + fld.d $f30, $sp, 136 + fld.d $f31, $sp, 144 + addi.d $sp, $sp, 160 + + /* Back home */ + jirl $r0, $r1, 0x0 + + EPILOGUE diff --git a/kernel/loongarch64/dgemm_tcopy_6.S b/kernel/loongarch64/dgemm_tcopy_6.S new file mode 100644 index 0000000000..d3bb4a2a64 --- /dev/null +++ b/kernel/loongarch64/dgemm_tcopy_6.S @@ -0,0 +1,555 @@ +/******************************************************************************* +Copyright (c) 2024, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ +#define ASSEMBLER + +#include "common.h" +/* Function parameters */ +#define M $r4 // param 1: m +#define N $r5 // param 2: n +#define SRC $r6 // param 3: src +#define LDA $r7 // param 4: lda +#define DST $r8 // param 5: dst + +#define I $r9 +#define J $r10 +#define S0 $r11 +#define S1 $r12 +#define S2 $r13 +#define S3 $r14 +#define S4 $r15 +#define S5 $r16 +#define S6 $r17 +#define S7 $r18 +#define S8 $r19 +#define P0 $r20 +#define P1 $r23 +#define P2 $r24 +#define P3 $r25 +#define P4 $r26 +#define T0 $r27 +#define T1 $r28 +#define T2 $r29 +#define TL $r7 +#define ZERO $r0 + +#define F0 $f0 +#define F1 $f1 +#define F2 $f2 +#define F3 $f3 +#define F4 $f4 +#define F5 $f5 +#define F6 $f6 +#define F7 $f7 +/* LSX vectors */ +#define V0 $vr0 +#define V1 $vr1 +#define V2 $vr2 +#define V3 $vr3 +/* LASX vectors */ +#define U0 $xr4 +#define U1 $xr5 +#define U2 $xr6 +#define U3 $xr7 +#define U4 $xr8 +#define U5 $xr9 +#define U6 $xr10 +#define U7 $xr11 + + PROLOGUE + + addi.d $sp, $sp, -56 + SDARG $r23, $sp, 0 + SDARG $r24, $sp, 8 + SDARG $r25, $sp, 16 + SDARG $r26, $sp, 24 + SDARG $r27, $sp, 32 + SDARG $r28, $sp, 40 + SDARG $r29, $sp, 48 + + move S0, SRC + move P0, DST + + addi.d I, ZERO, 0x06 + div.d T0, N, I // 1 + mul.d T1, I, T0 // 6 + sub.d N, N, T1 // 1 + + srai.d T2, N, 0x02 + slli.d T2, T2, 0x02 + add.d T2, T1, T2 + + mul.d P2, M, T1 + mul.d P3, M, T2 + slli.d P2, P2, 0x03 + slli.d P3, P3, 0x03 + + srai.d T2, N, 0x01 + srai.d J, M, 0x03 + slli.d T2, T2, 0x01 + add.d T2, T1, T2 + + add.d P2, DST, P2 + mul.d P4, M, T2 + add.d P3, DST, P3 + slli.d P4, P4, 0x03 + slli.d TL, LDA, 0x03 + add.d P4, DST, P4 + + slli.d T2, TL, 0x01 + slli.d T1, M, 0x03 + mul.d T1, T1, I + beq ZERO, J, .L_M7 + +.L_J1: /* J-- */ + move S1, S0 + add.d S2, S0, TL + add.d S3, S1, T2 + add.d S4, S2, T2 + add.d S5, S3, T2 + add.d S6, S4, T2 + add.d S7, S5, T2 + add.d S8, S6, T2 + add.d S0, S7, T2 + + move P1, P0 + addi.d P0, P0, 0x180 + + move I, T0 + addi.d J, J, -1 + beq ZERO, I, .L_N7 + +.L_I1: /* I-- */ + xvld U0, S1, 0x00 + vld V0, S1, 0x20 + xvld U1, S2, 0x00 + vld V1, S2, 0x20 + xvld U2, S3, 0x00 + vld V2, S3, 0x20 + xvld U3, S4, 0x00 + vld V3, S4, 0x20 + + xvst U0, P1, 0x00 + vst V0, P1, 0x20 + + xvst U1, P1, 0x30 + vst V1, P1, 0x50 + + xvst U2, P1, 0x60 + vst V2, P1, 0x80 + + xvst U3, P1, 0x90 + vst V3, P1, 0xB0 + + xvld U0, S5, 0x00 + vld V0, S5, 0x20 + xvld U1, S6, 0x00 + vld V1, S6, 0x20 + xvld U2, S7, 0x00 + vld V2, S7, 0x20 + xvld U3, S8, 0x00 + vld V3, S8, 0x20 + + xvst U0, P1, 0xC0 + vst V0, P1, 0xE0 + + xvst U1, P1, 0xF0 + vst V1, P1, 0x110 + + xvst U2, P1, 0x120 + vst V2, P1, 0x140 + + xvst U3, P1, 0x150 + vst V3, P1, 0x170 + + addi.d S1, S1, 0x30 + addi.d S2, S2, 0x30 + addi.d S3, S3, 0x30 + addi.d S4, S4, 0x30 + addi.d S5, S5, 0x30 + addi.d S6, S6, 0x30 + addi.d S7, S7, 0x30 + addi.d S8, S8, 0x30 + addi.d I, I, -1 + + add.d P1, P1, T1 + blt ZERO, I, .L_I1 + +.L_N7: + andi I, N, 0x04 + beq ZERO, I, .L_N3 + + xvld U0, S1, 0x00 + xvld U1, S2, 0x00 + xvld U2, S3, 0x00 + xvld U3, S4, 0x00 + xvld U4, S5, 0x00 + xvld U5, S6, 0x00 + xvld U6, S7, 0x00 + xvld U7, S8, 0x00 + + xvst U0, P2, 0x00 + xvst U1, P2, 0x20 + xvst U2, P2, 0x40 + xvst U3, P2, 0x60 + xvst U4, P2, 0x80 + xvst U5, P2, 0xA0 + xvst U6, P2, 0xC0 + xvst U7, P2, 0xE0 + + addi.d S1, S1, 0x20 + addi.d S2, S2, 0x20 + addi.d S3, S3, 0x20 + addi.d S4, S4, 0x20 + addi.d S5, S5, 0x20 + addi.d S6, S6, 0x20 + addi.d S7, S7, 0x20 + addi.d S8, S8, 0x20 + addi.d P2, P2, 0x100 + +.L_N3: + andi I, N, 0x02 + beq ZERO, I, .L_N1 + + xvld U0, S1, 0x00 + xvld U1, S2, 0x00 + xvld U2, S3, 0x00 + xvld U3, S4, 0x00 + xvld U4, S5, 0x00 + xvld U5, S6, 0x00 + xvld U6, S7, 0x00 + xvld U7, S8, 0x00 + + xvpermi.q U0, U1, 0x02 + xvpermi.q U2, U3, 0x02 + xvpermi.q U4, U5, 0x02 + xvpermi.q U6, U7, 0x02 + + xvst U0, P3, 0x00 + xvst U2, P3, 0x20 + xvst U4, P3, 0x40 + xvst U6, P3, 0x60 + + addi.d S1, S1, 0x10 + addi.d S2, S2, 0x10 + addi.d S3, S3, 0x10 + addi.d S4, S4, 0x10 + addi.d S5, S5, 0x10 + addi.d S6, S6, 0x10 + addi.d S7, S7, 0x10 + addi.d S8, S8, 0x10 + addi.d P3, P3, 0x80 + +.L_N1: + andi I, N, 0x01 + beq ZERO, I, .L_N0 + + fld.d F0, S1, 0x00 + fld.d F1, S2, 0x00 + fld.d F2, S3, 0x00 + fld.d F3, S4, 0x00 + fld.d F4, S5, 0x00 + fld.d F5, S6, 0x00 + fld.d F6, S7, 0x00 + fld.d F7, S8, 0x00 + + fst.d F0, P4, 0x00 + fst.d F1, P4, 0x08 + fst.d F2, P4, 0x10 + fst.d F3, P4, 0x18 + fst.d F4, P4, 0x20 + fst.d F5, P4, 0x28 + fst.d F6, P4, 0x30 + fst.d F7, P4, 0x38 + + addi.d S1, S1, 0x08 + addi.d S2, S2, 0x08 + addi.d S3, S3, 0x08 + addi.d S4, S4, 0x08 + addi.d S5, S5, 0x08 + addi.d S6, S6, 0x08 + addi.d S7, S7, 0x08 + addi.d S8, S8, 0x08 + addi.d P4, P4, 0x40 + +.L_N0: + blt ZERO, J, .L_J1 + +.L_M7: + andi J, M, 0x04 + beq ZERO, J, .L_M3 + + move S1, S0 + add.d S2, S0, TL + add.d S3, S1, T2 + add.d S4, S2, T2 + add.d S0, S3, T2 + + move P1, P0 + addi.d P0, P0, 0xC0 + + move I, T0 + beq ZERO, I, .L_4N7 + +.L_4I1: /* I-- */ + xvld U0, S1, 0x00 + vld V0, S1, 0x20 + xvld U1, S2, 0x00 + vld V1, S2, 0x20 + xvld U2, S3, 0x00 + vld V2, S3, 0x20 + xvld U3, S4, 0x00 + vld V3, S4, 0x20 + + xvst U0, P1, 0x00 + vst V0, P1, 0x20 + + xvst U1, P1, 0x30 + vst V1, P1, 0x50 + + xvst U2, P1, 0x60 + vst V2, P1, 0x80 + + xvst U3, P1, 0x90 + vst V3, P1, 0xB0 + + addi.d S1, S1, 0x30 + addi.d S2, S2, 0x30 + addi.d S3, S3, 0x30 + addi.d S4, S4, 0x30 + + addi.d I, I, -1 + add.d P1, P1, T1 + blt ZERO, I, .L_4I1 + +.L_4N7: + andi I, N, 0x04 + beq ZERO, I, .L_4N3 + + xvld U0, S1, 0x00 + xvld U1, S2, 0x00 + xvld U2, S3, 0x00 + xvld U3, S4, 0x00 + + xvst U0, P2, 0x00 + xvst U1, P2, 0x20 + xvst U2, P2, 0x40 + xvst U3, P2, 0x60 + + addi.d S1, S1, 0x20 + addi.d S2, S2, 0x20 + addi.d S3, S3, 0x20 + addi.d S4, S4, 0x20 + addi.d P2, P2, 0x80 + +.L_4N3: + andi I, N, 0x02 + beq ZERO, I, .L_4N1 + + xvld U0, S1, 0x00 + xvld U1, S2, 0x00 + xvld U2, S3, 0x00 + xvld U3, S4, 0x00 + + xvpermi.q U0, U1, 0x02 + xvpermi.q U2, U3, 0x02 + + xvst U0, P3, 0x00 + xvst U2, P3, 0x20 + + addi.d S1, S1, 0x10 + addi.d S2, S2, 0x10 + addi.d S3, S3, 0x10 + addi.d S4, S4, 0x10 + addi.d P3, P3, 0x40 + +.L_4N1: + andi I, N, 0x01 + beq ZERO, I, .L_M3 + + fld.d F0, S1, 0x00 + fld.d F1, S2, 0x00 + fld.d F2, S3, 0x00 + fld.d F3, S4, 0x00 + + fst.d F0, P4, 0x00 + fst.d F1, P4, 0x08 + fst.d F2, P4, 0x10 + fst.d F3, P4, 0x18 + + addi.d S1, S1, 0x08 + addi.d S2, S2, 0x08 + addi.d S3, S3, 0x08 + addi.d S4, S4, 0x08 + addi.d P4, P4, 0x20 + +.L_M3: + andi J, M, 0x02 + beq ZERO, J, .L_M1 + + move S1, S0 + add.d S2, S0, TL + add.d S0, S0, T2 + + move P1, P0 + addi.d P0, P0, 0x60 + + move I, T0 + beq ZERO, I, .L_2N7 + +.L_2I1: /* I-- */ + xvld U0, S1, 0x00 + vld V0, S1, 0x20 + xvld U1, S2, 0x00 + vld V1, S2, 0x20 + + xvst U0, P1, 0x00 + vst V0, P1, 0x20 + + xvst U1, P1, 0x30 + vst V1, P1, 0x50 + + addi.d S1, S1, 0x30 + addi.d S2, S2, 0x30 + addi.d I, I, -1 + add.d P1, P1, T1 + blt ZERO, I, .L_2I1 + +.L_2N7: + andi I, N, 0x04 + beq ZERO, I, .L_2N3 + + xvld U0, S1, 0x00 + xvld U1, S2, 0x00 + + xvst U0, P2, 0x00 + xvst U1, P2, 0x20 + + addi.d S1, S1, 0x20 + addi.d S2, S2, 0x20 + addi.d P2, P2, 0x40 + +.L_2N3: + andi I, N, 0x02 + beq ZERO, I, .L_2N1 + + xvld U0, S1, 0x00 + xvld U1, S2, 0x00 + + xvpermi.q U0, U1, 0x02 + + xvst U0, P3, 0x00 + + addi.d S1, S1, 0x10 + addi.d S2, S2, 0x10 + addi.d P3, P3, 0x20 + +.L_2N1: + andi I, N, 0x01 + beq ZERO, I, .L_M1 + + fld.d F0, S1, 0x00 + fld.d F1, S2, 0x00 + + fst.d F0, P4, 0x00 + fst.d F1, P4, 0x08 + + addi.d S1, S1, 0x08 + addi.d S2, S2, 0x08 + addi.d P4, P4, 0x10 + +.L_M1: + andi J, M, 0x01 + beq ZERO, J, .L_M0 + + move S1, S0 + add.d S2, S0, TL + + move P1, P0 + addi.d P0, P0, 0x30 + + move I, T0 + beq ZERO, I, .L_1N7 + +.L_1I1: /* I-- */ + xvld U0, S1, 0x00 + vld V0, S1, 0x20 + + xvst U0, P1, 0x00 + vst V0, P1, 0x20 + + addi.d S1, S1, 0x30 + addi.d I, I, -1 + add.d P1, P1, T1 + blt ZERO, I, .L_1I1 + +.L_1N7: + andi I, N, 0x04 + beq ZERO, I, .L_1N3 + + xvld U0, S1, 0x00 + + xvst U0, P2, 0x00 + + addi.d S1, S1, 0x20 + addi.d P2, P2, 0x20 + +.L_1N3: + andi I, N, 0x02 + beq ZERO, I, .L_1N1 + + fld.d F0, S1, 0x00 + fld.d F1, S1, 0x08 + + fst.d F0, P3, 0x00 + fst.d F1, P3, 0x08 + + addi.d S1, S1, 0x10 + addi.d P3, P3, 0x10 + +.L_1N1: + andi I, N, 0x01 + beq ZERO, I, .L_M0 + + fld.d F0, S1, 0x00 + + fst.d F0, P4, 0x00 + + addi.d S1, S1, 0x08 + addi.d P4, P4, 0x08 + +.L_M0: + LDARG $r23, $sp, 0 + LDARG $r24, $sp, 8 + LDARG $r25, $sp, 16 + LDARG $r26, $sp, 24 + LDARG $r27, $sp, 32 + LDARG $r28, $sp, 40 + LDARG $r29, $sp, 48 + addi.d $sp, $sp, 56 + jirl $r0, $r1, 0x00 + + EPILOGUE diff --git a/kernel/loongarch64/gemm_ncopy_6.prefx.c b/kernel/loongarch64/gemm_ncopy_6.prefx.c new file mode 100644 index 0000000000..65680d4e32 --- /dev/null +++ b/kernel/loongarch64/gemm_ncopy_6.prefx.c @@ -0,0 +1,299 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#include +#include "common.h" + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, FLOAT *b){ + BLASLONG i, j; + BLASLONG nmod6; + + FLOAT *aoffset; + FLOAT *aoffset1, *aoffset2, *aoffset3, *aoffset4; + FLOAT *aoffset5, *aoffset6 ; + + FLOAT *boffset; + FLOAT ctemp01, ctemp02, ctemp03, ctemp04; + FLOAT ctemp05, ctemp06, ctemp07, ctemp08; + FLOAT ctemp09, ctemp10, ctemp11, ctemp12; + FLOAT ctemp13, ctemp14, ctemp15, ctemp16; + FLOAT ctemp17, ctemp18, ctemp19, ctemp20; + FLOAT ctemp21, ctemp22, ctemp23, ctemp24; + + nmod6 = n - (n / 6)* 6 ; + aoffset = a; + boffset = b; + + // prefex A: 1 block, block size: 4*8 bytes, offset: 16*8 bytes, base: aoffset1,2,,6; + BLASULONG index = 0x100080; //( (1<<20)|(16<<3)&0xffff) ) ; + // prefex B: 1 block, block size: 24*8 bytes, offset: 96*8 bytes, base: boffset; + BLASULONG index_b = 0xb00300; //(11<<20) | ((96*8)&0xffff) ; + + j = (n / 6); + if (j > 0){ + do{ + aoffset1 = aoffset; + aoffset2 = aoffset1 + lda; + aoffset3 = aoffset2 + lda; + aoffset4 = aoffset3 + lda; + aoffset5 = aoffset4 + lda; + aoffset6 = aoffset5 + lda; + aoffset += 6 * lda; + + i = (m >> 2); + if (i > 0){ + do{ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + ctemp03 = *(aoffset1 + 2); + ctemp04 = *(aoffset1 + 3); + + ctemp05 = *(aoffset2 + 0); + ctemp06 = *(aoffset2 + 1); + ctemp07 = *(aoffset2 + 2); + ctemp08 = *(aoffset2 + 3); + + ctemp09 = *(aoffset3 + 0); + ctemp10 = *(aoffset3 + 1); + ctemp11 = *(aoffset3 + 2); + ctemp12 = *(aoffset3 + 3); + + ctemp13 = *(aoffset4 + 0); + ctemp14 = *(aoffset4 + 1); + ctemp15 = *(aoffset4 + 2); + ctemp16 = *(aoffset4 + 3); + + ctemp17 = *(aoffset5 + 0); + ctemp18 = *(aoffset5 + 1); + ctemp19 = *(aoffset5 + 2); + ctemp20 = *(aoffset5 + 3); + + ctemp21 = *(aoffset6 + 0); + ctemp22 = *(aoffset6 + 1); + ctemp23 = *(aoffset6 + 2); + ctemp24 = *(aoffset6 + 3); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp05; + *(boffset + 2) = ctemp09; + *(boffset + 3) = ctemp13; + *(boffset + 4) = ctemp17; + *(boffset + 5) = ctemp21; + + *(boffset + 6) = ctemp02; + *(boffset + 7) = ctemp06; + *(boffset + 8) = ctemp10; + *(boffset + 9) = ctemp14; + *(boffset + 10) = ctemp18; + *(boffset + 11) = ctemp22; + + *(boffset + 12) = ctemp03; + *(boffset + 13) = ctemp07; + *(boffset + 14) = ctemp11; + *(boffset + 15) = ctemp15; + *(boffset + 16) = ctemp19; + *(boffset + 17) = ctemp23; + + *(boffset + 18) = ctemp04; + *(boffset + 19) = ctemp08; + *(boffset + 20) = ctemp12; + *(boffset + 21) = ctemp16; + *(boffset + 22) = ctemp20; + *(boffset + 23) = ctemp24; + + aoffset1 += 4; + aoffset2 += 4; + aoffset3 += 4; + aoffset4 += 4; + aoffset5 += 4; + aoffset6 += 4; + + boffset += 24; + i --; + }while(i > 0); + } + + i = (m & 3); + if (i > 0){ + do{ + ctemp01 = *(aoffset1 + 0); + ctemp03 = *(aoffset2 + 0); + ctemp05 = *(aoffset3 + 0); + ctemp07 = *(aoffset4 + 0); + ctemp09 = *(aoffset5 + 0); + ctemp11 = *(aoffset6 + 0); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp03; + *(boffset + 2) = ctemp05; + *(boffset + 3) = ctemp07; + *(boffset + 4) = ctemp09; + *(boffset + 5) = ctemp11; + + aoffset1 ++; + aoffset2 ++; + aoffset3 ++; + aoffset4 ++; + aoffset5 ++; + aoffset6 ++; + boffset += 6; + i --; + }while(i > 0); + } + + j--; + }while(j > 0); + } /* end of if(j > 0) */ + + if (nmod6 & 4){ + aoffset1 = aoffset; + aoffset2 = aoffset1 + lda; + aoffset3 = aoffset2 + lda; + aoffset4 = aoffset3 + lda; + aoffset += 4 * lda; + + i = (m >> 1); + if (i > 0){ + do{ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + ctemp03 = *(aoffset2 + 0); + ctemp04 = *(aoffset2 + 1); + + ctemp05 = *(aoffset3 + 0); + ctemp06 = *(aoffset3 + 1); + ctemp07 = *(aoffset4 + 0); + ctemp08 = *(aoffset4 + 1); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp03; + *(boffset + 2) = ctemp05; + *(boffset + 3) = ctemp07; + *(boffset + 4) = ctemp02; + *(boffset + 5) = ctemp04; + *(boffset + 6) = ctemp06; + *(boffset + 7) = ctemp08; + + aoffset1 += 2; + aoffset2 += 2; + aoffset3 += 2; + aoffset4 += 2; + boffset += 8; + + i --; + }while(i > 0); + } + + if (m & 1){ + ctemp01 = *(aoffset1 + 0); + ctemp03 = *(aoffset2 + 0); + ctemp05 = *(aoffset3 + 0); + ctemp07 = *(aoffset4 + 0); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp03; + *(boffset + 2) = ctemp05; + *(boffset + 3) = ctemp07; + boffset += 4; + } + } + + if (nmod6 & 2){ + aoffset1 = aoffset; + aoffset2 = aoffset1 + lda; + aoffset += 2 * lda; + + i = (m >> 1); + if (i > 0){ + do{ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + ctemp03 = *(aoffset2 + 0); + ctemp04 = *(aoffset2 + 1); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp03; + *(boffset + 2) = ctemp02; + *(boffset + 3) = ctemp04; + + aoffset1 += 2; + aoffset2 += 2; + boffset += 4; + + i --; + }while(i > 0); + } + + if (m & 1){ + ctemp01 = *(aoffset1 + 0); + ctemp03 = *(aoffset2 + 0); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp03; + boffset += 2; + } + } + + if (nmod6 & 1){ + aoffset1 = aoffset; + + i = (m >> 1); + if (i > 0){ + do{ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp02; + + aoffset1 += 2; + boffset += 2; + + i --; + }while(i > 0); + } + + if (m & 1){ + ctemp01 = *(aoffset1 + 0); + + *(boffset + 0) = ctemp01; + } + } + + return 0; +} diff --git a/kernel/loongarch64/trsm_kernel_LN_UNROLLN6.c b/kernel/loongarch64/trsm_kernel_LN_UNROLLN6.c new file mode 100644 index 0000000000..5e25a5e3e4 --- /dev/null +++ b/kernel/loongarch64/trsm_kernel_LN_UNROLLN6.c @@ -0,0 +1,342 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#include "common.h" + +static FLOAT dm1 = -1.; + +#ifdef CONJ +#define GEMM_KERNEL GEMM_KERNEL_L +#else +#define GEMM_KERNEL GEMM_KERNEL_N +#endif + +#if GEMM_DEFAULT_UNROLL_M == 1 +#define GEMM_UNROLL_M_SHIFT 0 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 2 +#define GEMM_UNROLL_M_SHIFT 1 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 4 +#define GEMM_UNROLL_M_SHIFT 2 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 6 +#define GEMM_UNROLL_M_SHIFT 2 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 8 +#define GEMM_UNROLL_M_SHIFT 3 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 16 +#define GEMM_UNROLL_M_SHIFT 4 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 1 +#define GEMM_UNROLL_N_SHIFT 0 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 2 +#define GEMM_UNROLL_N_SHIFT 1 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 4 +#define GEMM_UNROLL_N_SHIFT 2 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 8 +#define GEMM_UNROLL_N_SHIFT 3 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 16 +#define GEMM_UNROLL_N_SHIFT 4 +#endif + +#ifndef COMPLEX + +static inline void solve(BLASLONG m, BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc) { + + FLOAT aa, bb; + + int i, j, k; + + a += (m - 1) * m; + b += (m - 1) * n; + + for (i = m - 1; i >= 0; i--) { + + aa = *(a + i); + + for (j = 0; j < n; j ++) { + bb = *(c + i + j * ldc); + bb *= aa; + *b = bb; + *(c + i + j * ldc) = bb; + b ++; + + for (k = 0; k < i; k ++){ + *(c + k + j * ldc) -= bb * *(a + k); + } + + } + a -= m; + b -= 2 * n; + } + +} + +#else + +static inline void solve(BLASLONG m, BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc) { + + FLOAT aa1, aa2; + FLOAT bb1, bb2; + FLOAT cc1, cc2; + + int i, j, k; + + ldc *= 2; + a += (m - 1) * m * 2; + b += (m - 1) * n * 2; + + for (i = m - 1; i >= 0; i--) { + + aa1 = *(a + i * 2 + 0); + aa2 = *(a + i * 2 + 1); + + for (j = 0; j < n; j ++) { + bb1 = *(c + i * 2 + 0 + j * ldc); + bb2 = *(c + i * 2 + 1 + j * ldc); + +#ifndef CONJ + cc1 = aa1 * bb1 - aa2 * bb2; + cc2 = aa1 * bb2 + aa2 * bb1; +#else + cc1 = aa1 * bb1 + aa2 * bb2; + cc2 = aa1 * bb2 - aa2 * bb1; +#endif + + + *(b + 0) = cc1; + *(b + 1) = cc2; + *(c + i * 2 + 0 + j * ldc) = cc1; + *(c + i * 2 + 1 + j * ldc) = cc2; + b += 2; + + for (k = 0; k < i; k ++){ +#ifndef CONJ + *(c + k * 2 + 0 + j * ldc) -= cc1 * *(a + k * 2 + 0) - cc2 * *(a + k * 2 + 1); + *(c + k * 2 + 1 + j * ldc) -= cc1 * *(a + k * 2 + 1) + cc2 * *(a + k * 2 + 0); +#else + *(c + k * 2 + 0 + j * ldc) -= cc1 * *(a + k * 2 + 0) + cc2 * *(a + k * 2 + 1); + *(c + k * 2 + 1 + j * ldc) -= - cc1 * *(a + k * 2 + 1) + cc2 * *(a + k * 2 + 0); +#endif + } + + } + a -= m * 2; + b -= 4 * n; + } + +} + +#endif + + +int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, +#ifdef COMPLEX + FLOAT dummy2, +#endif + FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG offset){ + + BLASLONG i, j; + FLOAT *aa, *cc; + BLASLONG kk; + +#if 0 + fprintf(stderr, "TRSM KERNEL LN : m = %3ld n = %3ld k = %3ld offset = %3ld\n", + m, n, k, offset); +#endif + + // j = (n >> GEMM_UNROLL_N_SHIFT); + j = (n / 6); + + while (j > 0) { + + kk = m + offset; + + if (m & (GEMM_UNROLL_M - 1)) { + for (i = 1; i < GEMM_UNROLL_M; i *= 2){ + if (m & i) { + aa = a + ((m & ~(i - 1)) - i) * k * COMPSIZE; + cc = c + ((m & ~(i - 1)) - i) * COMPSIZE; + + if (k - kk > 0) { + GEMM_KERNEL(i, GEMM_UNROLL_N, k - kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa + i * kk * COMPSIZE, + b + GEMM_UNROLL_N * kk * COMPSIZE, + cc, + ldc); + } + + solve(i, GEMM_UNROLL_N, + aa + (kk - i) * i * COMPSIZE, + b + (kk - i) * GEMM_UNROLL_N * COMPSIZE, + cc, ldc); + + kk -= i; + } + } + } + + i = (m >> GEMM_UNROLL_M_SHIFT); + if (i > 0) { + aa = a + ((m & ~(GEMM_UNROLL_M - 1)) - GEMM_UNROLL_M) * k * COMPSIZE; + cc = c + ((m & ~(GEMM_UNROLL_M - 1)) - GEMM_UNROLL_M) * COMPSIZE; + + do { + if (k - kk > 0) { + GEMM_KERNEL(GEMM_UNROLL_M, GEMM_UNROLL_N, k - kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa + GEMM_UNROLL_M * kk * COMPSIZE, + b + GEMM_UNROLL_N * kk * COMPSIZE, + cc, + ldc); + } + + solve(GEMM_UNROLL_M, GEMM_UNROLL_N, + aa + (kk - GEMM_UNROLL_M) * GEMM_UNROLL_M * COMPSIZE, + b + (kk - GEMM_UNROLL_M) * GEMM_UNROLL_N * COMPSIZE, + cc, ldc); + + aa -= GEMM_UNROLL_M * k * COMPSIZE; + cc -= GEMM_UNROLL_M * COMPSIZE; + kk -= GEMM_UNROLL_M; + i --; + } while (i > 0); + } + + b += GEMM_UNROLL_N * k * COMPSIZE; + c += GEMM_UNROLL_N * ldc * COMPSIZE; + j --; + } + + BLASLONG nmodN = n - n/6*6 ; + + // if (n & (GEMM_UNROLL_N - 1)) { + if (nmodN) { + + // j = (GEMM_UNROLL_N >> 1); + j = 4; + while (j > 0) { + if (nmodN & j) { + + kk = m + offset; + + if (m & (GEMM_UNROLL_M - 1)) { + for (i = 1; i < GEMM_UNROLL_M; i *= 2){ + if (m & i) { + aa = a + ((m & ~(i - 1)) - i) * k * COMPSIZE; + cc = c + ((m & ~(i - 1)) - i) * COMPSIZE; + + if (k - kk > 0) { + GEMM_KERNEL(i, j, k - kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa + i * kk * COMPSIZE, + b + j * kk * COMPSIZE, + cc, ldc); + } + + solve(i, j, + aa + (kk - i) * i * COMPSIZE, + b + (kk - i) * j * COMPSIZE, + cc, ldc); + + kk -= i; + } + } + } + + i = (m >> GEMM_UNROLL_M_SHIFT); + if (i > 0) { + aa = a + ((m & ~(GEMM_UNROLL_M - 1)) - GEMM_UNROLL_M) * k * COMPSIZE; + cc = c + ((m & ~(GEMM_UNROLL_M - 1)) - GEMM_UNROLL_M) * COMPSIZE; + + do { + if (k - kk > 0) { + GEMM_KERNEL(GEMM_UNROLL_M, j, k - kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa + GEMM_UNROLL_M * kk * COMPSIZE, + b + j * kk * COMPSIZE, + cc, + ldc); + } + + solve(GEMM_UNROLL_M, j, + aa + (kk - GEMM_UNROLL_M) * GEMM_UNROLL_M * COMPSIZE, + b + (kk - GEMM_UNROLL_M) * j * COMPSIZE, + cc, ldc); + + aa -= GEMM_UNROLL_M * k * COMPSIZE; + cc -= GEMM_UNROLL_M * COMPSIZE; + kk -= GEMM_UNROLL_M; + i --; + } while (i > 0); + } + + b += j * k * COMPSIZE; + c += j * ldc * COMPSIZE; + } + j >>= 1; + } + } + + return 0; +} diff --git a/kernel/loongarch64/trsm_kernel_LT_UNROLLN6.c b/kernel/loongarch64/trsm_kernel_LT_UNROLLN6.c new file mode 100644 index 0000000000..2106c88cae --- /dev/null +++ b/kernel/loongarch64/trsm_kernel_LT_UNROLLN6.c @@ -0,0 +1,327 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#include "common.h" + +static FLOAT dm1 = -1.; + +#ifdef CONJ +#define GEMM_KERNEL GEMM_KERNEL_L +#else +#define GEMM_KERNEL GEMM_KERNEL_N +#endif + +#if GEMM_DEFAULT_UNROLL_M == 1 +#define GEMM_UNROLL_M_SHIFT 0 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 2 +#define GEMM_UNROLL_M_SHIFT 1 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 4 +#define GEMM_UNROLL_M_SHIFT 2 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 6 +#define GEMM_UNROLL_M_SHIFT 2 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 8 +#define GEMM_UNROLL_M_SHIFT 3 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 16 +#define GEMM_UNROLL_M_SHIFT 4 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 1 +#define GEMM_UNROLL_N_SHIFT 0 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 2 +#define GEMM_UNROLL_N_SHIFT 1 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 4 +#define GEMM_UNROLL_N_SHIFT 2 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 8 +#define GEMM_UNROLL_N_SHIFT 3 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 16 +#define GEMM_UNROLL_N_SHIFT 4 +#endif + +#ifndef COMPLEX + +static inline void solve(BLASLONG m, BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc) { + + FLOAT aa, bb; + + int i, j, k; + + for (i = 0; i < m; i++) { + + aa = *(a + i); + + for (j = 0; j < n; j ++) { + bb = *(c + i + j * ldc); + bb *= aa; + *b = bb; + *(c + i + j * ldc) = bb; + b ++; + + for (k = i + 1; k < m; k ++){ + *(c + k + j * ldc) -= bb * *(a + k); + } + + } + a += m; + } +} + +#else + +static inline void solve(BLASLONG m, BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc) { + + FLOAT aa1, aa2; + FLOAT bb1, bb2; + FLOAT cc1, cc2; + + int i, j, k; + + ldc *= 2; + + for (i = 0; i < m; i++) { + + aa1 = *(a + i * 2 + 0); + aa2 = *(a + i * 2 + 1); + + for (j = 0; j < n; j ++) { + bb1 = *(c + i * 2 + 0 + j * ldc); + bb2 = *(c + i * 2 + 1 + j * ldc); + +#ifndef CONJ + cc1 = aa1 * bb1 - aa2 * bb2; + cc2 = aa1 * bb2 + aa2 * bb1; +#else + cc1 = aa1 * bb1 + aa2 * bb2; + cc2 = aa1 * bb2 - aa2 * bb1; +#endif + + *(b + 0) = cc1; + *(b + 1) = cc2; + *(c + i * 2 + 0 + j * ldc) = cc1; + *(c + i * 2 + 1 + j * ldc) = cc2; + b += 2; + + for (k = i + 1; k < m; k ++){ +#ifndef CONJ + *(c + k * 2 + 0 + j * ldc) -= cc1 * *(a + k * 2 + 0) - cc2 * *(a + k * 2 + 1); + *(c + k * 2 + 1 + j * ldc) -= cc1 * *(a + k * 2 + 1) + cc2 * *(a + k * 2 + 0); +#else + *(c + k * 2 + 0 + j * ldc) -= cc1 * *(a + k * 2 + 0) + cc2 * *(a + k * 2 + 1); + *(c + k * 2 + 1 + j * ldc) -= -cc1 * *(a + k * 2 + 1) + cc2 * *(a + k * 2 + 0); +#endif + } + + } + a += m * 2; + } +} + +#endif + + +int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, +#ifdef COMPLEX + FLOAT dummy2, +#endif + FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG offset){ + + FLOAT *aa, *cc; + BLASLONG kk; + BLASLONG i, j, jj; + +#if 0 + fprintf(stderr, "TRSM KERNEL LT : m = %3ld n = %3ld k = %3ld offset = %3ld\n", + m, n, k, offset); +#endif + + jj = 0; + + // j = (n >> GEMM_UNROLL_N_SHIFT); + j = (n / 6); + + while (j > 0) { + + kk = offset; + aa = a; + cc = c; + + i = (m >> GEMM_UNROLL_M_SHIFT); + + while (i > 0) { + + if (kk > 0) { + GEMM_KERNEL(GEMM_UNROLL_M, GEMM_UNROLL_N, kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa, b, cc, ldc); + } + + solve(GEMM_UNROLL_M, GEMM_UNROLL_N, + aa + kk * GEMM_UNROLL_M * COMPSIZE, + b + kk * GEMM_UNROLL_N * COMPSIZE, + cc, ldc); + + aa += GEMM_UNROLL_M * k * COMPSIZE; + cc += GEMM_UNROLL_M * COMPSIZE; + kk += GEMM_UNROLL_M; + i --; + } + + if (m & (GEMM_UNROLL_M - 1)) { + i = (GEMM_UNROLL_M >> 1); + while (i > 0) { + if (m & i) { + if (kk > 0) { + GEMM_KERNEL(i, GEMM_UNROLL_N, kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa, b, cc, ldc); + } + solve(i, GEMM_UNROLL_N, + aa + kk * i * COMPSIZE, + b + kk * GEMM_UNROLL_N * COMPSIZE, + cc, ldc); + + aa += i * k * COMPSIZE; + cc += i * COMPSIZE; + kk += i; + } + i >>= 1; + } + } + + b += GEMM_UNROLL_N * k * COMPSIZE; + c += GEMM_UNROLL_N * ldc * COMPSIZE; + j --; + jj += GEMM_UNROLL_M; + } + + BLASLONG nmodN = n - n/6*6 ; + + // if (n & (GEMM_UNROLL_N - 1)) { + if (nmodN) { + + // j = (GEMM_UNROLL_N >> 1); + j = 4; + + while (j > 0) { + if (nmodN & j) { + + kk = offset; + aa = a; + cc = c; + + i = (m >> GEMM_UNROLL_M_SHIFT); + + while (i > 0) { + if (kk > 0) { + GEMM_KERNEL(GEMM_UNROLL_M, j, kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa, + b, + cc, + ldc); + } + + solve(GEMM_UNROLL_M, j, + aa + kk * GEMM_UNROLL_M * COMPSIZE, + b + kk * j * COMPSIZE, cc, ldc); + + aa += GEMM_UNROLL_M * k * COMPSIZE; + cc += GEMM_UNROLL_M * COMPSIZE; + kk += GEMM_UNROLL_M; + i --; + } + + if (m & (GEMM_UNROLL_M - 1)) { + i = (GEMM_UNROLL_M >> 1); + while (i > 0) { + if (m & i) { + if (kk > 0) { + GEMM_KERNEL(i, j, kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa, + b, + cc, + ldc); + } + + solve(i, j, + aa + kk * i * COMPSIZE, + b + kk * j * COMPSIZE, cc, ldc); + + aa += i * k * COMPSIZE; + cc += i * COMPSIZE; + kk += i; + } + i >>= 1; + } + } + + b += j * k * COMPSIZE; + c += j * ldc * COMPSIZE; + } + j >>= 1; + } + } + + return 0; +} diff --git a/kernel/loongarch64/trsm_kernel_RN_UNROLLN6.c b/kernel/loongarch64/trsm_kernel_RN_UNROLLN6.c new file mode 100644 index 0000000000..42d5155c33 --- /dev/null +++ b/kernel/loongarch64/trsm_kernel_RN_UNROLLN6.c @@ -0,0 +1,325 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#include "common.h" + +static FLOAT dm1 = -1.; + +#ifdef CONJ +#define GEMM_KERNEL GEMM_KERNEL_R +#else +#define GEMM_KERNEL GEMM_KERNEL_N +#endif + +#if GEMM_DEFAULT_UNROLL_M == 1 +#define GEMM_UNROLL_M_SHIFT 0 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 2 +#define GEMM_UNROLL_M_SHIFT 1 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 4 +#define GEMM_UNROLL_M_SHIFT 2 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 6 +#define GEMM_UNROLL_M_SHIFT 2 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 8 +#define GEMM_UNROLL_M_SHIFT 3 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 16 +#define GEMM_UNROLL_M_SHIFT 4 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 1 +#define GEMM_UNROLL_N_SHIFT 0 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 2 +#define GEMM_UNROLL_N_SHIFT 1 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 4 +#define GEMM_UNROLL_N_SHIFT 2 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 8 +#define GEMM_UNROLL_N_SHIFT 3 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 16 +#define GEMM_UNROLL_N_SHIFT 4 +#endif + +#ifndef COMPLEX + +static inline void solve(BLASLONG m, BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc) { + + FLOAT aa, bb; + + int i, j, k; + + for (i = 0; i < n; i++) { + + bb = *(b + i); + + for (j = 0; j < m; j ++) { + aa = *(c + j + i * ldc); + aa *= bb; + *a = aa; + *(c + j + i * ldc) = aa; + a ++; + + for (k = i + 1; k < n; k ++){ + *(c + j + k * ldc) -= aa * *(b + k); + } + + } + b += n; + } +} + +#else + +static inline void solve(BLASLONG m, BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc) { + + FLOAT aa1, aa2; + FLOAT bb1, bb2; + FLOAT cc1, cc2; + + int i, j, k; + + ldc *= 2; + + for (i = 0; i < n; i++) { + + bb1 = *(b + i * 2 + 0); + bb2 = *(b + i * 2 + 1); + + for (j = 0; j < m; j ++) { + aa1 = *(c + j * 2 + 0 + i * ldc); + aa2 = *(c + j * 2 + 1 + i * ldc); + +#ifndef CONJ + cc1 = aa1 * bb1 - aa2 * bb2; + cc2 = aa1 * bb2 + aa2 * bb1; +#else + cc1 = aa1 * bb1 + aa2 * bb2; + cc2 = -aa1 * bb2 + aa2 * bb1; +#endif + + *(a + 0) = cc1; + *(a + 1) = cc2; + *(c + j * 2 + 0 + i * ldc) = cc1; + *(c + j * 2 + 1 + i * ldc) = cc2; + a += 2; + + for (k = i + 1; k < n; k ++){ +#ifndef CONJ + *(c + j * 2 + 0 + k * ldc) -= cc1 * *(b + k * 2 + 0) - cc2 * *(b + k * 2 + 1); + *(c + j * 2 + 1 + k * ldc) -= cc1 * *(b + k * 2 + 1) + cc2 * *(b + k * 2 + 0); +#else + *(c + j * 2 + 0 + k * ldc) -= cc1 * *(b + k * 2 + 0) + cc2 * *(b + k * 2 + 1); + *(c + j * 2 + 1 + k * ldc) -= - cc1 * *(b + k * 2 + 1) + cc2 * *(b + k * 2 + 0); +#endif + } + + } + b += n * 2; + } +} + +#endif + + +int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, +#ifdef COMPLEX + FLOAT dummy2, +#endif + FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG offset){ + + FLOAT *aa, *cc; + BLASLONG kk; + BLASLONG i, j, jj; + +#if 0 + fprintf(stderr, "TRSM RN KERNEL m = %3ld n = %3ld k = %3ld offset = %3ld\n", + m, n, k, offset); +#endif + + jj = 0; + // j = (n >> GEMM_UNROLL_N_SHIFT); + j = (n / 6); + kk = -offset; + + while (j > 0) { + + aa = a; + cc = c; + + i = (m >> GEMM_UNROLL_M_SHIFT); + + if (i > 0) { + do { + if (kk > 0) { + GEMM_KERNEL(GEMM_UNROLL_M, GEMM_UNROLL_N, kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa, b, cc, ldc); + } + + solve(GEMM_UNROLL_M, GEMM_UNROLL_N, + aa + kk * GEMM_UNROLL_M * COMPSIZE, + b + kk * GEMM_UNROLL_N * COMPSIZE, + cc, ldc); + + aa += GEMM_UNROLL_M * k * COMPSIZE; + cc += GEMM_UNROLL_M * COMPSIZE; + i --; + } while (i > 0); + } + + + if (m & (GEMM_UNROLL_M - 1)) { + i = (GEMM_UNROLL_M >> 1); + while (i > 0) { + if (m & i) { + if (kk > 0) { + GEMM_KERNEL(i, GEMM_UNROLL_N, kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa, b, cc, ldc); + } + solve(i, GEMM_UNROLL_N, + aa + kk * i * COMPSIZE, + b + kk * GEMM_UNROLL_N * COMPSIZE, + cc, ldc); + + aa += i * k * COMPSIZE; + cc += i * COMPSIZE; + } + i >>= 1; + } + } + + kk += GEMM_UNROLL_N; + b += GEMM_UNROLL_N * k * COMPSIZE; + c += GEMM_UNROLL_N * ldc * COMPSIZE; + j --; + jj += GEMM_UNROLL_M; + } + + BLASLONG nmodN = n - n/6*6 ; + + // if (n & (GEMM_UNROLL_N - 1)) { + if (nmodN) { + + // j = (GEMM_UNROLL_N >> 1); + j = 4; + + while (j > 0) { + if (nmodN & j) { + + aa = a; + cc = c; + + i = (m >> GEMM_UNROLL_M_SHIFT); + + while (i > 0) { + if (kk > 0) { + GEMM_KERNEL(GEMM_UNROLL_M, j, kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa, + b, + cc, + ldc); + } + + solve(GEMM_UNROLL_M, j, + aa + kk * GEMM_UNROLL_M * COMPSIZE, + b + kk * j * COMPSIZE, cc, ldc); + + aa += GEMM_UNROLL_M * k * COMPSIZE; + cc += GEMM_UNROLL_M * COMPSIZE; + i --; + } + + if (m & (GEMM_UNROLL_M - 1)) { + i = (GEMM_UNROLL_M >> 1); + while (i > 0) { + if (m & i) { + if (kk > 0) { + GEMM_KERNEL(i, j, kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa, + b, + cc, + ldc); + } + + solve(i, j, + aa + kk * i * COMPSIZE, + b + kk * j * COMPSIZE, cc, ldc); + + aa += i * k * COMPSIZE; + cc += i * COMPSIZE; + } + i >>= 1; + } + } + + b += j * k * COMPSIZE; + c += j * ldc * COMPSIZE; + kk += j; + } + j >>= 1; + } + } + + return 0; +} diff --git a/kernel/loongarch64/trsm_kernel_RT_UNROLLN6.c b/kernel/loongarch64/trsm_kernel_RT_UNROLLN6.c new file mode 100644 index 0000000000..7424ad5791 --- /dev/null +++ b/kernel/loongarch64/trsm_kernel_RT_UNROLLN6.c @@ -0,0 +1,351 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#include "common.h" + +static FLOAT dm1 = -1.; + +#ifdef CONJ +#define GEMM_KERNEL GEMM_KERNEL_R +#else +#define GEMM_KERNEL GEMM_KERNEL_N +#endif + +#if GEMM_DEFAULT_UNROLL_M == 1 +#define GEMM_UNROLL_M_SHIFT 0 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 2 +#define GEMM_UNROLL_M_SHIFT 1 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 4 +#define GEMM_UNROLL_M_SHIFT 2 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 6 +#define GEMM_UNROLL_M_SHIFT 2 +#endif + + +#if GEMM_DEFAULT_UNROLL_M == 8 +#define GEMM_UNROLL_M_SHIFT 3 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 16 +#define GEMM_UNROLL_M_SHIFT 4 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 1 +#define GEMM_UNROLL_N_SHIFT 0 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 2 +#define GEMM_UNROLL_N_SHIFT 1 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 4 +#define GEMM_UNROLL_N_SHIFT 2 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 8 +#define GEMM_UNROLL_N_SHIFT 3 +#endif + +#if GEMM_DEFAULT_UNROLL_N == 16 +#define GEMM_UNROLL_N_SHIFT 4 +#endif + + +#ifndef COMPLEX + +static inline void solve(BLASLONG m, BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc) { + + FLOAT aa, bb; + + int i, j, k; + + a += (n - 1) * m; + b += (n - 1) * n; + + for (i = n - 1; i >= 0; i--) { + + bb = *(b + i); + + for (j = 0; j < m; j ++) { + aa = *(c + j + i * ldc); + aa *= bb; + *a = aa; + *(c + j + i * ldc) = aa; + a ++; + + for (k = 0; k < i; k ++){ + *(c + j + k * ldc) -= aa * *(b + k); + } + + } + b -= n; + a -= 2 * m; + } + +} + +#else + +static inline void solve(BLASLONG m, BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc) { + + FLOAT aa1, aa2; + FLOAT bb1, bb2; + FLOAT cc1, cc2; + + int i, j, k; + + ldc *= 2; + + a += (n - 1) * m * 2; + b += (n - 1) * n * 2; + + for (i = n - 1; i >= 0; i--) { + + bb1 = *(b + i * 2 + 0); + bb2 = *(b + i * 2 + 1); + + for (j = 0; j < m; j ++) { + + aa1 = *(c + j * 2 + 0 + i * ldc); + aa2 = *(c + j * 2 + 1 + i * ldc); + +#ifndef CONJ + cc1 = aa1 * bb1 - aa2 * bb2; + cc2 = aa1 * bb2 + aa2 * bb1; +#else + cc1 = aa1 * bb1 + aa2 * bb2; + cc2 = - aa1 * bb2 + aa2 * bb1; +#endif + + *(a + 0) = cc1; + *(a + 1) = cc2; + + *(c + j * 2 + 0 + i * ldc) = cc1; + *(c + j * 2 + 1 + i * ldc) = cc2; + a += 2; + + for (k = 0; k < i; k ++){ +#ifndef CONJ + *(c + j * 2 + 0 + k * ldc) -= cc1 * *(b + k * 2 + 0) - cc2 * *(b + k * 2 + 1); + *(c + j * 2 + 1 + k * ldc) -= cc1 * *(b + k * 2 + 1) + cc2 * *(b + k * 2 + 0); +#else + *(c + j * 2 + 0 + k * ldc) -= cc1 * *(b + k * 2 + 0) + cc2 * *(b + k * 2 + 1); + *(c + j * 2 + 1 + k * ldc) -= -cc1 * *(b + k * 2 + 1) + cc2 * *(b + k * 2 + 0); +#endif + } + + } + b -= n * 2; + a -= 4 * m; + } + +} + +#endif + +int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, +#ifdef COMPLEX + FLOAT dummy2, +#endif + FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG offset){ + + BLASLONG i, j; + FLOAT *aa, *cc; + BLASLONG kk; + +#if 0 + fprintf(stderr, "TRSM RT KERNEL m = %3ld n = %3ld k = %3ld offset = %3ld\n", + m, n, k, offset); +#endif + + kk = n - offset; + c += n * ldc * COMPSIZE; + b += n * k * COMPSIZE; + + + BLASLONG nmodN = n - n/6*6 ; + + // if (n & (GEMM_UNROLL_N - 1)) { + if (nmodN) { + + j = 1; + while (j < GEMM_UNROLL_N) { + if (nmodN & j) { + + aa = a; + b -= j * k * COMPSIZE; + c -= j * ldc* COMPSIZE; + cc = c; + + i = (m >> GEMM_UNROLL_M_SHIFT); + if (i > 0) { + + do { + if (k - kk > 0) { + GEMM_KERNEL(GEMM_UNROLL_M, j, k - kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa + GEMM_UNROLL_M * kk * COMPSIZE, + b + j * kk * COMPSIZE, + cc, + ldc); + } + + solve(GEMM_UNROLL_M, j, + aa + (kk - j) * GEMM_UNROLL_M * COMPSIZE, + b + (kk - j) * j * COMPSIZE, + cc, ldc); + + aa += GEMM_UNROLL_M * k * COMPSIZE; + cc += GEMM_UNROLL_M * COMPSIZE; + i --; + } while (i > 0); + } + + if (m & (GEMM_UNROLL_M - 1)) { + i = (GEMM_UNROLL_M >> 1); + do { + if (m & i) { + + if (k - kk > 0) { + GEMM_KERNEL(i, j, k - kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa + i * kk * COMPSIZE, + b + j * kk * COMPSIZE, + cc, ldc); + } + + solve(i, j, + aa + (kk - j) * i * COMPSIZE, + b + (kk - j) * j * COMPSIZE, + cc, ldc); + + aa += i * k * COMPSIZE; + cc += i * COMPSIZE; + + } + i >>= 1; + } while (i > 0); + } + kk -= j; + } + j <<= 1; + } + } + + // j = (n >> GEMM_UNROLL_N_SHIFT); + j = (n / 6); + + if (j > 0) { + + do { + aa = a; + b -= GEMM_UNROLL_N * k * COMPSIZE; + c -= GEMM_UNROLL_N * ldc * COMPSIZE; + cc = c; + + i = (m >> GEMM_UNROLL_M_SHIFT); + if (i > 0) { + do { + if (k - kk > 0) { + GEMM_KERNEL(GEMM_UNROLL_M, GEMM_UNROLL_N, k - kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa + GEMM_UNROLL_M * kk * COMPSIZE, + b + GEMM_UNROLL_N * kk * COMPSIZE, + cc, + ldc); + } + + solve(GEMM_UNROLL_M, GEMM_UNROLL_N, + aa + (kk - GEMM_UNROLL_N) * GEMM_UNROLL_M * COMPSIZE, + b + (kk - GEMM_UNROLL_N) * GEMM_UNROLL_N * COMPSIZE, + cc, ldc); + + aa += GEMM_UNROLL_M * k * COMPSIZE; + cc += GEMM_UNROLL_M * COMPSIZE; + i --; + } while (i > 0); + } + + if (m & (GEMM_UNROLL_M - 1)) { + i = (GEMM_UNROLL_M >> 1); + do { + if (m & i) { + if (k - kk > 0) { + GEMM_KERNEL(i, GEMM_UNROLL_N, k - kk, dm1, +#ifdef COMPLEX + ZERO, +#endif + aa + i * kk * COMPSIZE, + b + GEMM_UNROLL_N * kk * COMPSIZE, + cc, + ldc); + } + + solve(i, GEMM_UNROLL_N, + aa + (kk - GEMM_UNROLL_N) * i * COMPSIZE, + b + (kk - GEMM_UNROLL_N) * GEMM_UNROLL_N * COMPSIZE, + cc, ldc); + + aa += i * k * COMPSIZE; + cc += i * COMPSIZE; + } + i >>= 1; + } while (i > 0); + } + + kk -= GEMM_UNROLL_N; + j --; + } while (j > 0); + } + + return 0; +} + + diff --git a/param.h b/param.h index 84e241fcd2..2ebe824db5 100644 --- a/param.h +++ b/param.h @@ -2856,7 +2856,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define ZGEMM_DEFAULT_UNROLL_N 4 #define ZGEMM_DEFAULT_UNROLL_M 1 #else -#define DGEMM_DEFAULT_UNROLL_N 4 +#define DGEMM_DEFAULT_UNROLL_N 6 #define DGEMM_DEFAULT_UNROLL_M 16 #define SGEMM_DEFAULT_UNROLL_N 8 #define SGEMM_DEFAULT_UNROLL_M 16 @@ -2864,6 +2864,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define CGEMM_DEFAULT_UNROLL_M 16 #define ZGEMM_DEFAULT_UNROLL_N 4 #define ZGEMM_DEFAULT_UNROLL_M 8 +#define DGEMM_DEFAULT_UNROLL_MN 96 #endif #define QGEMM_DEFAULT_UNROLL_N 2 From b45d8e1ab2dd46b6c77e4c6e11b34258d032b771 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 9 May 2024 12:33:19 +0200 Subject: [PATCH 065/267] remove stray comma --- kernel/loongarch64/dot_lsx.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/loongarch64/dot_lsx.S b/kernel/loongarch64/dot_lsx.S index f7f6135531..8a74d82e7d 100644 --- a/kernel/loongarch64/dot_lsx.S +++ b/kernel/loongarch64/dot_lsx.S @@ -165,7 +165,7 @@ PROLOGUE /* store dot in s1 $f8 */ #ifdef DSDOT vfadd.d $vr8, $vr8, $vr9 - fsub.s s2, s2, s2, /* set s2 to 0.0 */ + fsub.s s2, s2, s2 /* set s2 to 0.0 */ vpackod.d $vr0, $vr8, $vr8 vfadd.d $vr8, $vr8, $vr0 #else From 637c650f4fb620cc42724cbd8a7ec1282abc56c4 Mon Sep 17 00:00:00 2001 From: gxw Date: Wed, 8 May 2024 14:40:48 +0800 Subject: [PATCH 066/267] loongarch64: Add buffer offset for target LOONGSON3R5 --- driver/others/blas_server.c | 13 +++++-------- interface/gemm.c | 13 ++++--------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/driver/others/blas_server.c b/driver/others/blas_server.c index e3ebc7a7e8..80c28ebda9 100644 --- a/driver/others/blas_server.c +++ b/driver/others/blas_server.c @@ -1074,13 +1074,10 @@ fprintf(STDERR, "Server[%2ld] Calculation started. Mode = 0x%03x M = %3ld N=%3l main_status[cpu] = MAIN_RUNNING1; #endif -//For Loongson servers, like the 3C5000 (featuring 16 cores), applying an -//offset to the buffer is essential for minimizing cache conflicts and optimizing performance. -#if defined(LOONGSON3R5) && !defined(NO_AFFINITY) - char model_name[128]; - get_cpu_model(model_name); - if ((strstr(model_name, "3C5000") != NULL) || (strstr(model_name, "3D5000") != NULL)) - if (sa == NULL) sa = (void *)((BLASLONG)buffer + (WhereAmI() & 0xf) * GEMM_OFFSET_A); +//For target LOONGSON3R5, applying an offset to the buffer is essential +//for minimizing cache conflicts and optimizing performance. +#if defined(ARCH_LOONGARCH64) && !defined(NO_AFFINITY) + if (sa == NULL) sa = (void *)((BLASLONG)buffer + (WhereAmI() & 0xf) * GEMM_OFFSET_A); #endif if (sa == NULL) sa = (void *)((BLASLONG)buffer + GEMM_OFFSET_A); @@ -1157,4 +1154,4 @@ if (queue -> mode & BLAS_PTHREAD) { } -#endif \ No newline at end of file +#endif diff --git a/interface/gemm.c b/interface/gemm.c index c402836ca3..4537b6a78f 100644 --- a/interface/gemm.c +++ b/interface/gemm.c @@ -521,15 +521,10 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANS buffer = (XFLOAT *)blas_memory_alloc(0); -//For Loongson servers, like the 3C5000 (featuring 16 cores), applying an -//offset to the buffer is essential for minimizing cache conflicts and optimizing performance. -#if defined(LOONGSON3R5) && !defined(NO_AFFINITY) - char model_name[128]; - get_cpu_model(model_name); - if ((strstr(model_name, "3C5000") != NULL) || (strstr(model_name, "3D5000") != NULL)) - sa = (XFLOAT *)((BLASLONG)buffer + (WhereAmI() & 0xf) * GEMM_OFFSET_A); - else - sa = (XFLOAT *)((BLASLONG)buffer + GEMM_OFFSET_A); +//For target LOONGSON3R5, applying an offset to the buffer is essential +//for minimizing cache conflicts and optimizing performance. +#if defined(ARCH_LOONGARCH64) && !defined(NO_AFFINITY) + sa = (XFLOAT *)((BLASLONG)buffer + (WhereAmI() & 0xf) * GEMM_OFFSET_A); #else sa = (XFLOAT *)((BLASLONG)buffer +GEMM_OFFSET_A); #endif From cb0f70740961c40c9e05be4ac4a197217c5c5c0e Mon Sep 17 00:00:00 2001 From: gxw Date: Thu, 9 May 2024 15:36:25 +0800 Subject: [PATCH 067/267] loongarch64: Fixed utest fork:safety --- kernel/loongarch64/dgemm_ncopy_16.S | 2 ++ kernel/loongarch64/dgemm_ncopy_8_lsx.S | 2 ++ 2 files changed, 4 insertions(+) diff --git a/kernel/loongarch64/dgemm_ncopy_16.S b/kernel/loongarch64/dgemm_ncopy_16.S index 95c879031a..4c32e0ec79 100644 --- a/kernel/loongarch64/dgemm_ncopy_16.S +++ b/kernel/loongarch64/dgemm_ncopy_16.S @@ -655,6 +655,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addi.d TD, TD, 0x10 .L_N1: + andi J, N, 0x01 + beq ZERO, J, .L_N0 move S1, TS beq ZERO, M, .L_N0 diff --git a/kernel/loongarch64/dgemm_ncopy_8_lsx.S b/kernel/loongarch64/dgemm_ncopy_8_lsx.S index 203c3eb27d..4ca485508a 100644 --- a/kernel/loongarch64/dgemm_ncopy_8_lsx.S +++ b/kernel/loongarch64/dgemm_ncopy_8_lsx.S @@ -268,6 +268,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. addi.d S2, S2, 0x08 addi.d TD, TD, 0x10 .L_N1: + andi J, N, 0x01 + beq ZERO, J, .L_N0 move S1, TS beq ZERO, M, .L_N0 .L_M1: From f9a26240a7004f426e2d39f36459e54c356a4c31 Mon Sep 17 00:00:00 2001 From: gxw Date: Thu, 9 May 2024 19:15:03 +0800 Subject: [PATCH 068/267] loongarch64: Fixed icamax_lsx --- kernel/loongarch64/icamax_lsx.S | 2 -- 1 file changed, 2 deletions(-) diff --git a/kernel/loongarch64/icamax_lsx.S b/kernel/loongarch64/icamax_lsx.S index a2fc9dbbd8..c22ade4b38 100644 --- a/kernel/loongarch64/icamax_lsx.S +++ b/kernel/loongarch64/icamax_lsx.S @@ -308,8 +308,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. vinsgr2vr.w x1, t3, 3 vinsgr2vr.w x2, t4, 3 addi.d I, I, -1 - vpickev.w x1, VX1, VX0 - vpickod.w x2, VX1, VX0 vfmul.s x3, VI4, x1 vfmul.s x4, VI4, x2 vfcmp.clt.s VT0, x1, VI3 From ecf8b588a95ca5affba8edbdbe20637e33fc67f9 Mon Sep 17 00:00:00 2001 From: gxw Date: Sat, 11 May 2024 16:10:38 +0800 Subject: [PATCH 069/267] loongarch64: Fixed GCC14 compilation issue --- cpuid_loongarch64.c | 1 + 1 file changed, 1 insertion(+) diff --git a/cpuid_loongarch64.c b/cpuid_loongarch64.c index 0ad32ae4e0..3b7a9c82ea 100644 --- a/cpuid_loongarch64.c +++ b/cpuid_loongarch64.c @@ -33,6 +33,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include /* If LASX extension instructions supported, * using core LOONGSON3R5 From a6c184d150d006a912ded59fad181d7ec84295d6 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 11 May 2024 13:07:30 +0200 Subject: [PATCH 070/267] forward NO_CFLAGS to the CFLAGS, if set --- utest/Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utest/Makefile b/utest/Makefile index d4154e0c2c..36acf96cdb 100644 --- a/utest/Makefile +++ b/utest/Makefile @@ -61,6 +61,10 @@ OBJS = utest_main2.o OBJS_EXT = $(DIR_EXT)/utest_main2.o endif +ifeq ($(NO_CBLAS), 1) +override CFLAGS += -DNO_CBLAS +endif + all : run_test ifeq ($(OSNAME), AIX) From d8baf2f2ea5c80d1de4a0769fa4b3cdec569e50d Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 11 May 2024 13:10:54 +0200 Subject: [PATCH 071/267] Support compilation without CBLAS --- utest/test_extensions/common.c | 6 +++--- utest/test_extensions/test_caxpby.c | 6 ++++-- utest/test_extensions/test_cgeadd.c | 12 +++++++++--- utest/test_extensions/test_cgemm.c | 7 ++++--- utest/test_extensions/test_cgemmt.c | 12 +++++++++++- utest/test_extensions/test_cgemv_t.c | 9 +++++++-- utest/test_extensions/test_cimatcopy.c | 6 +++++- utest/test_extensions/test_comatcopy.c | 4 ++++ utest/test_extensions/test_crot.c | 4 +++- utest/test_extensions/test_crotg.c | 5 +++-- utest/test_extensions/test_cscal.c | 6 +++++- utest/test_extensions/test_ctrmv.c | 5 +++-- utest/test_extensions/test_ctrsv.c | 7 ++++--- utest/test_extensions/test_daxpby.c | 7 +++++-- utest/test_extensions/test_dgeadd.c | 11 +++++++++-- utest/test_extensions/test_dgemmt.c | 12 +++++++++++- utest/test_extensions/test_dimatcopy.c | 6 +++++- utest/test_extensions/test_domatcopy.c | 4 ++++ utest/test_extensions/test_drotmg.c | 4 +++- utest/test_extensions/test_dsum.c | 2 ++ utest/test_extensions/test_dzsum.c | 2 ++ utest/test_extensions/test_icamin.c | 4 +++- utest/test_extensions/test_idamin.c | 2 ++ utest/test_extensions/test_isamin.c | 3 ++- utest/test_extensions/test_izamin.c | 2 ++ utest/test_extensions/test_saxpby.c | 6 +++++- utest/test_extensions/test_scsum.c | 2 ++ utest/test_extensions/test_sgeadd.c | 12 +++++++++--- utest/test_extensions/test_sgemmt.c | 12 +++++++++++- utest/test_extensions/test_simatcopy.c | 6 +++++- utest/test_extensions/test_somatcopy.c | 6 +++++- utest/test_extensions/test_srotmg.c | 4 +++- utest/test_extensions/test_ssum.c | 2 ++ utest/test_extensions/test_zaxpby.c | 4 ++++ utest/test_extensions/test_zgeadd.c | 11 +++++++++-- utest/test_extensions/test_zgemm.c | 7 ++++--- utest/test_extensions/test_zgemmt.c | 13 +++++++++++-- utest/test_extensions/test_zgemv_t.c | 10 +++++++--- utest/test_extensions/test_zimatcopy.c | 6 +++++- utest/test_extensions/test_zomatcopy.c | 6 +++++- utest/test_extensions/test_zrot.c | 4 +++- utest/test_extensions/test_zrotg.c | 4 +++- utest/test_extensions/test_zscal.c | 6 +++++- utest/test_extensions/test_ztrmv.c | 6 +++--- utest/test_extensions/test_ztrsv.c | 7 ++++--- 45 files changed, 220 insertions(+), 62 deletions(-) diff --git a/utest/test_extensions/common.c b/utest/test_extensions/common.c index 8a6a47795a..808aa54557 100644 --- a/utest/test_extensions/common.c +++ b/utest/test_extensions/common.c @@ -69,7 +69,7 @@ float smatrix_difference(float *a, float *b, blasint cols, blasint rows, blasint for (j = 0; j < cols; j++) { a_ptr[j] -= b_ptr[j]; } - norm += cblas_snrm2(cols, a_ptr, inc); + norm += BLASFUNC(snrm2)(&cols, a_ptr, &inc); a_ptr += ld; b_ptr += ld; @@ -92,7 +92,7 @@ double dmatrix_difference(double *a, double *b, blasint cols, blasint rows, blas for (j = 0; j < cols; j++) { a_ptr[j] -= b_ptr[j]; } - norm += cblas_dnrm2(cols, a_ptr, inc); + norm += BLASFUNC(dnrm2)(&cols, a_ptr, &inc); a_ptr += ld; b_ptr += ld; @@ -256,4 +256,4 @@ void zcopy(blasint rows, blasint cols, double *alpha, double *a_src, int lda_src a_dst[i*lda_dst+j+1] = (-1.0) * conj *alpha[0] * a_src[i*lda_src+j+1] + alpha[1] * a_src[i*lda_src+j]; } } -} \ No newline at end of file +} diff --git a/utest/test_extensions/test_caxpby.c b/utest/test_extensions/test_caxpby.c index 221a48ac71..8adf5b3e6c 100644 --- a/utest/test_extensions/test_caxpby.c +++ b/utest/test_extensions/test_caxpby.c @@ -96,7 +96,7 @@ static float check_caxpby(blasint n, float *alpha, blasint incx, float *beta, bl // Find the norm of differences return BLASFUNC(scnrm2)(&n, data_caxpby.y_test, &incy_abs); } - +#ifndef NO_CBLAS /** * C API specific function * Test caxpby by comparing it with cscal and caxpy. @@ -146,7 +146,7 @@ static float c_api_check_caxpby(blasint n, float *alpha, blasint incx, float *be // Find the norm of differences return cblas_scnrm2(n, data_caxpby.y_test, incy_abs); } - +#endif /** * Fortran API specific test * Test caxpby by comparing it with cscal and caxpy. @@ -388,6 +388,7 @@ CTEST(caxpby, check_n_zero) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test caxpby by comparing it with cscal and caxpy. @@ -629,3 +630,4 @@ CTEST(caxpby, c_api_check_n_zero) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } #endif +#endif diff --git a/utest/test_extensions/test_cgeadd.c b/utest/test_extensions/test_cgeadd.c index 9b87ad9f37..55c52137a2 100644 --- a/utest/test_extensions/test_cgeadd.c +++ b/utest/test_extensions/test_cgeadd.c @@ -62,13 +62,14 @@ static void cgeadd_trusted(blasint m, blasint n, float *alpha, float *aptr, blasint lda, float *beta, float *cptr, blasint ldc) { blasint i; + blasint one=1; lda *= 2; ldc *= 2; for (i = 0; i < n; i++) { - cblas_caxpby(m, alpha, aptr, 1, beta, cptr, 1); + BLASFUNC(caxpby)(&m, alpha, aptr, &one, beta, cptr, &one); aptr += lda; cptr += ldc; } @@ -116,9 +117,11 @@ static float check_cgeadd(char api, OPENBLAS_CONST enum CBLAS_ORDER order, if (api == 'F') BLASFUNC(cgeadd)(&m, &n, alpha, data_cgeadd.a_test, &lda, beta, data_cgeadd.c_test, &ldc); +#ifndef NO_CBLAS else cblas_cgeadd(order, m, n, alpha, data_cgeadd.a_test, lda, beta, data_cgeadd.c_test, ldc); +#endif // Find the differences between output matrix caculated by cgeadd and sgemm return smatrix_difference(data_cgeadd.c_test, data_cgeadd.c_verify, cols, rows, ldc*2); @@ -150,9 +153,11 @@ static int check_badargs(char api, OPENBLAS_CONST enum CBLAS_ORDER order, if (api == 'F') BLASFUNC(cgeadd)(&m, &n, alpha, data_cgeadd.a_test, &lda, beta, data_cgeadd.c_test, &ldc); +#ifndef NO_CBLAS else cblas_cgeadd(order, m, n, alpha, data_cgeadd.a_test, lda, beta, data_cgeadd.c_test, ldc); +#endif return check_error(); } @@ -419,7 +424,7 @@ CTEST(cgeadd, m_zero) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } - +#ifndef NO_CBLAS /** * C API specific test * Test cgeadd by comparing it against sgemm @@ -877,4 +882,5 @@ CTEST(cgeadd, c_api_m_zero) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } -#endif \ No newline at end of file +#endif +#endif diff --git a/utest/test_extensions/test_cgemm.c b/utest/test_extensions/test_cgemm.c index cd38d710bf..08a26554a0 100644 --- a/utest/test_extensions/test_cgemm.c +++ b/utest/test_extensions/test_cgemm.c @@ -73,6 +73,7 @@ static float check_cgemm(char transa, char transb, blasint m, blasint n, blasint float alpha_conj[] = {1.0f, 0.0f}; char transa_verify = transa; char transb_verify = transb; + char cc[2]="C", cr[2]="R"; int arows = k, acols = m; int brows = n, bcols = k; @@ -99,12 +100,12 @@ static float check_cgemm(char transa, char transb, blasint m, blasint n, blasint data_cgemm.c_verify[i] = data_cgemm.c_test[i]; if (transa == 'R'){ - cblas_cimatcopy(CblasColMajor, CblasConjNoTrans, arows, acols, alpha_conj, data_cgemm.a_verify, lda, lda); + BLASFUNC(cimatcopy)(cc, cr, &arows, &acols, alpha_conj, data_cgemm.a_verify, &lda, &lda); transa_verify = 'N'; } if (transb == 'R'){ - cblas_cimatcopy(CblasColMajor, CblasConjNoTrans, brows, bcols, alpha_conj, data_cgemm.b_verify, ldb, ldb); + BLASFUNC(cimatcopy)(cc, cr, &brows, &bcols, alpha_conj, data_cgemm.b_verify, &ldb, &ldb); transb_verify = 'N'; } @@ -270,4 +271,4 @@ CTEST(cgemm, transa_conjnotransb) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } -#endif \ No newline at end of file +#endif diff --git a/utest/test_extensions/test_cgemmt.c b/utest/test_extensions/test_cgemmt.c index ed9279933d..dfeb06ff6e 100644 --- a/utest/test_extensions/test_cgemmt.c +++ b/utest/test_extensions/test_cgemmt.c @@ -73,9 +73,11 @@ static void cgemmt_trusted(char api, enum CBLAS_ORDER order, char uplo, char tra if(api == 'F') BLASFUNC(cgemm)(&transa, &transb, &m, &m, &k, alpha, data_cgemmt.a_test, &lda, data_cgemmt.b_test, &ldb, beta, data_cgemmt.c_gemm, &ldc); +#ifndef NO_CBLAS else cblas_cgemm(order, transa, transb, m, m, k, alpha, data_cgemmt.a_test, lda, data_cgemmt.b_test, ldb, beta, data_cgemmt.c_gemm, ldc); +#endif ldc *= 2; @@ -160,9 +162,11 @@ static float check_cgemmt(char api, enum CBLAS_ORDER order, char uplo, char tran if (api == 'F') BLASFUNC(cgemmt)(&uplo, &transa, &transb, &m, &k, alpha, data_cgemmt.a_test, &lda, data_cgemmt.b_test, &ldb, beta, data_cgemmt.c_test, &ldc); +#ifndef NO_CBLAS else cblas_cgemmt(order, uplo, transa, transb, m, k, alpha, data_cgemmt.a_test, lda, data_cgemmt.b_test, ldb, beta, data_cgemmt.c_test, ldc); +#endif for (i = 0; i < m * ldc * 2; i++) data_cgemmt.c_verify[i] -= data_cgemmt.c_test[i]; @@ -197,9 +201,11 @@ static int check_badargs(char api, enum CBLAS_ORDER order, char uplo, char trans if (api == 'F') BLASFUNC(cgemmt)(&uplo, &transa, &transb, &m, &k, alpha, data_cgemmt.a_test, &lda, data_cgemmt.b_test, &ldb, beta, data_cgemmt.c_test, &ldc); +#ifndef NO_CBLAS else cblas_cgemmt(order, uplo, transa, transb, m, k, alpha, data_cgemmt.a_test, lda, data_cgemmt.b_test, ldb, beta, data_cgemmt.c_test, ldc); +#endif return check_error(); } @@ -680,6 +686,7 @@ CTEST(cgemmt, lower_beta_one) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test cgemmt by comparing it against sgemm @@ -1591,6 +1598,7 @@ CTEST(cgemmt, c_api_rowmajor_lower_beta_one) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } +#endif /** * Fortran API specific test @@ -1736,6 +1744,7 @@ CTEST(cgemmt, xerbla_ldc_invalid) ASSERT_EQUAL(TRUE, passed); } +#ifndef NO_CBLAS /** * C API specific test. * Test error function for an invalid param order. @@ -2007,4 +2016,5 @@ CTEST(cgemmt, xerbla_c_api_rowmajor_ldc_invalid) M, K, lda, ldb, ldc, expected_info); ASSERT_EQUAL(TRUE, passed); } -#endif \ No newline at end of file +#endif +#endif diff --git a/utest/test_extensions/test_cgemv_t.c b/utest/test_extensions/test_cgemv_t.c index cb4e5ad9e4..ceb0b4bc59 100644 --- a/utest/test_extensions/test_cgemv_t.c +++ b/utest/test_extensions/test_cgemv_t.c @@ -65,6 +65,7 @@ static struct DATA_CGEMV_T data_cgemv_t; static void matrix_vector_product(blasint n, blasint m, blasint lda, blasint inc_x) { blasint i; + blasint one=1; float *a_ptr = data_cgemv_t.a_verify; float *x_ptr = data_cgemv_t.x_test; float *x_res = data_cgemv_t.x_verify; @@ -73,7 +74,7 @@ static void matrix_vector_product(blasint n, blasint m, blasint lda, blasint inc for (i = 0; i < n * inc_x; i+= inc_x) { - result = cblas_cdotu(lda, a_ptr, 1, x_ptr, inc_x); + result = BLASFUNC(cdotu)(&lda, a_ptr, &one, x_ptr, &inc_x); x_res[0] = CREAL(result); x_res[1] = CIMAG(result); a_ptr += lda * 2; @@ -153,6 +154,7 @@ static float check_cgemv(char api, char order, char trans, blasint m, blasint n, BLASFUNC(cgemv)(&trans, &m, &n, alpha, data_cgemv_t.a_test, &lda, data_cgemv_t.x_test, &inc_x, beta, data_cgemv_t.y_test, &inc_y); } +#ifndef NO_CBLAS else { if (order == 'C') corder = CblasColMajor; if (order == 'R') corder = CblasRowMajor; @@ -173,13 +175,14 @@ static float check_cgemv(char api, char order, char trans, blasint m, blasint n, cblas_cgemv(corder, ctrans, m, n, alpha, data_cgemv_t.a_test, lda, data_cgemv_t.x_test, inc_x, beta, data_cgemv_t.y_test, inc_y); } +#endif // Find the differences between output vector caculated by cgemv and reference funcs for (i = 0; i < m * inc_y * 2; i++) data_cgemv_t.y_test[i] -= data_cgemv_t.y_verify[i]; // Find the norm of differences - return cblas_scnrm2(m, data_cgemv_t.y_test, inc_y); + return BLASFUNC(scnrm2)(&m, data_cgemv_t.y_test, &inc_y); } /** @@ -213,6 +216,7 @@ static int check_badargs(char order, char trans, blasint m, blasint n, return check_error(); } +#ifndef NO_CBLAS /** * C API specific function * Check if error function was called with expected function name @@ -1130,3 +1134,4 @@ CTEST(cgemv, c_api_xerbla_invalid_order_col_major) ASSERT_EQUAL(TRUE, passed); } #endif +#endif diff --git a/utest/test_extensions/test_cimatcopy.c b/utest/test_extensions/test_cimatcopy.c index a4b1e30ac0..0c96a3b17c 100644 --- a/utest/test_extensions/test_cimatcopy.c +++ b/utest/test_extensions/test_cimatcopy.c @@ -98,6 +98,7 @@ static float check_cimatcopy(char api, char order, char trans, blasint rows, bla BLASFUNC(cimatcopy)(&order, &trans, &rows, &cols, alpha, data_cimatcopy.a_test, &lda_src, &lda_dst); } +#ifndef NO_CBLAS else { if (order == 'C') corder = CblasColMajor; if (order == 'R') corder = CblasRowMajor; @@ -108,6 +109,7 @@ static float check_cimatcopy(char api, char order, char trans, blasint rows, bla cblas_cimatcopy(corder, ctrans, rows, cols, alpha, data_cimatcopy.a_test, lda_src, lda_dst); } +#endif // Find the differences between output matrix computed by cimatcopy and reference func return smatrix_difference(data_cimatcopy.a_test, data_cimatcopy.a_verify, cols_out, rows_out, 2*lda_dst); @@ -502,6 +504,7 @@ CTEST(cimatcopy, rowmajor_conjtrans_col_50_row_100) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test cimatcopy by comparing it against reference @@ -681,6 +684,7 @@ CTEST(cimatcopy, c_api_rowmajor_conjtrans_col_100_row_100) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } +#endif /** * Test error function for an invalid param order. @@ -815,4 +819,4 @@ CTEST(cimatcopy, xerbla_colmajor_trans_invalid_ldb) int passed = check_badargs(order, trans, m, n, lda_src, lda_dst, expected_info); ASSERT_EQUAL(TRUE, passed); } -#endif \ No newline at end of file +#endif diff --git a/utest/test_extensions/test_comatcopy.c b/utest/test_extensions/test_comatcopy.c index 71663406af..b493c93a6f 100644 --- a/utest/test_extensions/test_comatcopy.c +++ b/utest/test_extensions/test_comatcopy.c @@ -99,6 +99,7 @@ static float check_comatcopy(char api, char order, char trans, blasint rows, bla BLASFUNC(comatcopy)(&order, &trans, &rows, &cols, alpha, data_comatcopy.a_test, &lda, data_comatcopy.b_test, &ldb); } +#ifndef NO_CBLAS else { if (order == 'C') corder = CblasColMajor; if (order == 'R') corder = CblasRowMajor; @@ -109,6 +110,7 @@ static float check_comatcopy(char api, char order, char trans, blasint rows, bla cblas_comatcopy(corder, ctrans, rows, cols, alpha, data_comatcopy.a_test, lda, data_comatcopy.b_test, ldb); } +#endif return smatrix_difference(data_comatcopy.b_test, data_comatcopy.b_verify, b_cols, b_rows, ldb*2); } @@ -316,6 +318,7 @@ CTEST(comatcopy, rowmajor_conjtrans_col_100_row_100) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test comatcopy by comparing it against refernce @@ -491,6 +494,7 @@ CTEST(comatcopy, c_api_rowmajor_conjtrans_col_100_row_100) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } +#endif /** * Test error function for an invalid param order. diff --git a/utest/test_extensions/test_crot.c b/utest/test_extensions/test_crot.c index 1c55216d9c..1ff4568132 100644 --- a/utest/test_extensions/test_crot.c +++ b/utest/test_extensions/test_crot.c @@ -107,6 +107,7 @@ static float check_csrot(blasint n, blasint inc_x, blasint inc_y, float *c, floa return (norm / 2); } +#ifndef NO_CBLAS /** * C API specific function * Comapare results computed by csrot and caxpby @@ -789,4 +790,5 @@ CTEST(crot, c_api_check_n_zero) float norm = c_api_check_csrot(n, inc_x, inc_y, c, s); ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } -#endif \ No newline at end of file +#endif +#endif diff --git a/utest/test_extensions/test_crotg.c b/utest/test_extensions/test_crotg.c index 84875ccf73..bb23a5a08e 100644 --- a/utest/test_extensions/test_crotg.c +++ b/utest/test_extensions/test_crotg.c @@ -161,7 +161,7 @@ CTEST(crotg, negative_real_negative_img) ASSERT_DBL_NEAR_TOL(-5.26498f, sa[0], SINGLE_EPS); ASSERT_DBL_NEAR_TOL(-7.01997f, sa[1], SINGLE_EPS); } - +#ifndef NO_CBLAS /** * C API specific test * Test crotg by comparing it against pre-calculated values @@ -287,4 +287,5 @@ CTEST(crotg, c_api_negative_real_negative_img) ASSERT_DBL_NEAR_TOL(-5.26498f, sa[0], SINGLE_EPS); ASSERT_DBL_NEAR_TOL(-7.01997f, sa[1], SINGLE_EPS); } -#endif \ No newline at end of file +#endif +#endif diff --git a/utest/test_extensions/test_cscal.c b/utest/test_extensions/test_cscal.c index 009c600ad4..cf8b3559e8 100644 --- a/utest/test_extensions/test_cscal.c +++ b/utest/test_extensions/test_cscal.c @@ -91,8 +91,10 @@ static float check_cscal(char api, blasint n, float *alpha, blasint inc) if(api == 'F') BLASFUNC(cscal)(&n, alpha, data_cscal.x_test, &inc); +#ifndef NO_CBLAS else cblas_cscal(n, alpha, data_cscal.x_test, inc); +#endif // Find the differences between output vector computed by cscal and cscal_trusted for (i = 0; i < n * 2 * inc; i++) @@ -132,6 +134,7 @@ CTEST(cscal, alpha_r_zero_alpha_i_zero_inc_2) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test cscal by comparing it against reference @@ -161,4 +164,5 @@ CTEST(cscal, c_api_alpha_r_zero_alpha_i_zero_inc_2) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } -#endif \ No newline at end of file +#endif +#endif diff --git a/utest/test_extensions/test_ctrmv.c b/utest/test_extensions/test_ctrmv.c index 2a3f274168..4c61c31c8a 100644 --- a/utest/test_extensions/test_ctrmv.c +++ b/utest/test_extensions/test_ctrmv.c @@ -65,6 +65,7 @@ static float check_ctrmv(char uplo, char trans, char diag, blasint n, blasint ld blasint i; float alpha_conj[] = {1.0f, 0.0f}; char trans_verify = trans; + char cc[2]="C", cr[2]="R"; srand_generate(data_ctrmv.a_test, n * lda * 2); srand_generate(data_ctrmv.x_test, n * incx * 2); @@ -76,7 +77,7 @@ static float check_ctrmv(char uplo, char trans, char diag, blasint n, blasint ld data_ctrmv.x_verify[i] = data_ctrmv.x_test[i]; if (trans == 'R'){ - cblas_cimatcopy(CblasColMajor, CblasConjNoTrans, n, n, alpha_conj, data_ctrmv.a_verify, lda, lda); + BLASFUNC(cimatcopy)(cc, cr, &n, &n, alpha_conj, data_ctrmv.a_verify, &lda, &lda); trans_verify = 'N'; } @@ -263,4 +264,4 @@ CTEST(ctrmv, conj_notrans_lower_unit_triangular_incx_2) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } -#endif \ No newline at end of file +#endif diff --git a/utest/test_extensions/test_ctrsv.c b/utest/test_extensions/test_ctrsv.c index 0e639bb2ab..7298ba42e0 100644 --- a/utest/test_extensions/test_ctrsv.c +++ b/utest/test_extensions/test_ctrsv.c @@ -65,6 +65,7 @@ static float check_ctrsv(char uplo, char trans, char diag, blasint n, blasint ld blasint i; float alpha_conj[] = {1.0f, 0.0f}; char trans_verify = trans; + char cc[2]="C", cr[2]="R"; srand_generate(data_ctrsv.a_test, n * lda * 2); srand_generate(data_ctrsv.x_test, n * incx * 2); @@ -76,8 +77,8 @@ static float check_ctrsv(char uplo, char trans, char diag, blasint n, blasint ld data_ctrsv.x_verify[i] = data_ctrsv.x_test[i]; if (trans == 'R'){ - cblas_cimatcopy(CblasColMajor, CblasConjNoTrans, n, n, - alpha_conj, data_ctrsv.a_verify, lda, lda); + BLASFUNC(cimatcopy)(cc, cr, &n, &n, + alpha_conj, data_ctrsv.a_verify, &lda, &lda); trans_verify = 'N'; } @@ -264,4 +265,4 @@ CTEST(ctrsv, conj_notrans_lower_unit_triangular_incx_2) ASSERT_DBL_NEAR_TOL(0.0f, norm, DOUBLE_EPS); } -#endif \ No newline at end of file +#endif diff --git a/utest/test_extensions/test_daxpby.c b/utest/test_extensions/test_daxpby.c index 6e77c7c7cf..93b26810d8 100644 --- a/utest/test_extensions/test_daxpby.c +++ b/utest/test_extensions/test_daxpby.c @@ -97,6 +97,7 @@ static double check_daxpby(blasint n, double alpha, blasint incx, double beta, b return BLASFUNC(dnrm2)(&n, data_daxpby.y_test, &incy_abs); } +#ifndef NO_CBLAS /** * C API specific function * Test daxpby by comparing it with dscal and daxpy. @@ -142,7 +143,7 @@ static double c_api_check_daxpby(blasint n, double alpha, blasint incx, double b // Find the norm of differences return cblas_dnrm2(n, data_daxpby.y_test, incy_abs); } - +#endif /** * Fortran API specific test * Test daxpby by comparing it with dscal and daxpy. @@ -468,6 +469,7 @@ CTEST(daxpby, check_n_zero) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test daxpby by comparing it with dscal and daxpy. @@ -796,4 +798,5 @@ CTEST(daxpby, c_api_check_n_zero) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } -#endif \ No newline at end of file +#endif +#endif diff --git a/utest/test_extensions/test_dgeadd.c b/utest/test_extensions/test_dgeadd.c index 8f93a842e3..20e8d966bf 100644 --- a/utest/test_extensions/test_dgeadd.c +++ b/utest/test_extensions/test_dgeadd.c @@ -62,10 +62,11 @@ static void dgeadd_trusted(blasint m, blasint n, double alpha, double *aptr, blasint lda, double beta, double *cptr, blasint ldc) { blasint i; + blasint one=1; for (i = 0; i < n; i++) { - cblas_daxpby(m, alpha, aptr, 1, beta, cptr, 1); + BLASFUNC(daxpby)(&m, &alpha, aptr, &one, &beta, cptr, &one); aptr += lda; cptr += ldc; } @@ -113,9 +114,11 @@ static double check_dgeadd(char api, OPENBLAS_CONST enum CBLAS_ORDER order, if (api == 'F') BLASFUNC(dgeadd)(&m, &n, &alpha, data_dgeadd.a_test, &lda, &beta, data_dgeadd.c_test, &ldc); +#ifndef NO_CBLAS else cblas_dgeadd(order, m, n, alpha, data_dgeadd.a_test, lda, beta, data_dgeadd.c_test, ldc); +#endif // Find the differences between output matrix caculated by dgeadd and sgemm return dmatrix_difference(data_dgeadd.c_test, data_dgeadd.c_verify, cols, rows, ldc); @@ -147,9 +150,11 @@ static int check_badargs(char api, OPENBLAS_CONST enum CBLAS_ORDER order, if (api == 'F') BLASFUNC(dgeadd)(&m, &n, &alpha, data_dgeadd.a_test, &lda, &beta, data_dgeadd.c_test, &ldc); +#ifndef NO_CBLAS else cblas_dgeadd(order, m, n, alpha, data_dgeadd.a_test, lda, beta, data_dgeadd.c_test, ldc); +#endif return check_error(); } @@ -417,6 +422,7 @@ CTEST(dgeadd, m_zero) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test dgeadd by comparing it against reference @@ -875,4 +881,5 @@ CTEST(dgeadd, c_api_m_zero) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } -#endif \ No newline at end of file +#endif +#endif diff --git a/utest/test_extensions/test_dgemmt.c b/utest/test_extensions/test_dgemmt.c index 22dcaf2aa1..fd8f5f6661 100644 --- a/utest/test_extensions/test_dgemmt.c +++ b/utest/test_extensions/test_dgemmt.c @@ -73,9 +73,11 @@ static void dgemmt_trusted(char api, enum CBLAS_ORDER order, char uplo, char tra if(api == 'F') BLASFUNC(dgemm)(&transa, &transb, &m, &m, &k, &alpha, data_dgemmt.a_test, &lda, data_dgemmt.b_test, &ldb, &beta, data_dgemmt.c_gemm, &ldc); +#ifndef NO_CBLAS else cblas_dgemm(order, transa, transb, m, m, k, alpha, data_dgemmt.a_test, lda, data_dgemmt.b_test, ldb, beta, data_dgemmt.c_gemm, ldc); +#endif if (uplo == 'L' || uplo == CblasLower) { @@ -152,9 +154,11 @@ static double check_dgemmt(char api, enum CBLAS_ORDER order, char uplo, char tra if (api == 'F') BLASFUNC(dgemmt)(&uplo, &transa, &transb, &m, &k, &alpha, data_dgemmt.a_test, &lda, data_dgemmt.b_test, &ldb, &beta, data_dgemmt.c_test, &ldc); +#ifndef NO_CBLAS else cblas_dgemmt(order, uplo, transa, transb, m, k, alpha, data_dgemmt.a_test, lda, data_dgemmt.b_test, ldb, beta, data_dgemmt.c_test, ldc); +#endif for (i = 0; i < m * ldc; i++) data_dgemmt.c_verify[i] -= data_dgemmt.c_test[i]; @@ -189,9 +193,11 @@ static int check_badargs(char api, enum CBLAS_ORDER order, char uplo, char trans if (api == 'F') BLASFUNC(dgemmt)(&uplo, &transa, &transb, &m, &k, &alpha, data_dgemmt.a_test, &lda, data_dgemmt.b_test, &ldb, &beta, data_dgemmt.c_test, &ldc); +#ifndef NO_CBLAS else cblas_dgemmt(order, uplo, transa, transb, m, k, alpha, data_dgemmt.a_test, lda, data_dgemmt.b_test, ldb, beta, data_dgemmt.c_test, ldc); +#endif return check_error(); } @@ -480,6 +486,7 @@ CTEST(dgemmt, lower_beta_one) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test dgemmt by comparing it against dgemm @@ -1023,6 +1030,7 @@ CTEST(dgemmt, c_api_rowmajor_lower_beta_one) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } +#endif /** * Fortran API specific test @@ -1168,6 +1176,7 @@ CTEST(dgemmt, xerbla_ldc_invalid) ASSERT_EQUAL(TRUE, passed); } +#ifndef NO_CBLAS /** * C API specific test. * Test error function for an invalid param order. @@ -1439,4 +1448,5 @@ CTEST(dgemmt, xerbla_c_api_rowmajor_ldc_invalid) M, K, lda, ldb, ldc, expected_info); ASSERT_EQUAL(TRUE, passed); } -#endif \ No newline at end of file +#endif +#endif diff --git a/utest/test_extensions/test_dimatcopy.c b/utest/test_extensions/test_dimatcopy.c index 811c356b3f..eebb7669eb 100644 --- a/utest/test_extensions/test_dimatcopy.c +++ b/utest/test_extensions/test_dimatcopy.c @@ -93,6 +93,7 @@ static double check_dimatcopy(char api, char order, char trans, blasint rows, bl BLASFUNC(dimatcopy)(&order, &trans, &rows, &cols, &alpha, data_dimatcopy.a_test, &lda_src, &lda_dst); } +#ifndef NO_CBLAS else { if (order == 'C') corder = CblasColMajor; if (order == 'R') corder = CblasRowMajor; @@ -103,6 +104,7 @@ static double check_dimatcopy(char api, char order, char trans, blasint rows, bl cblas_dimatcopy(corder, ctrans, rows, cols, alpha, data_dimatcopy.a_test, lda_src, lda_dst); } +#endif // Find the differences between output matrix computed by dimatcopy and reference func return dmatrix_difference(data_dimatcopy.a_test, data_dimatcopy.a_verify, cols_out, rows_out, lda_dst); @@ -687,6 +689,7 @@ CTEST(dimatcopy, rowmajor_notrans_col_100_row_50) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test dimatcopy by comparing it against reference @@ -778,6 +781,7 @@ CTEST(dimatcopy, c_api_rowmajor_notrans_col_100_row_100) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } +#endif /** * Test error function for an invalid param order. @@ -912,4 +916,4 @@ CTEST(dimatcopy, xerbla_colmajor_trans_invalid_ldb) int passed = check_badargs(order, trans, m, n, lda_src, lda_dst, expected_info); ASSERT_EQUAL(TRUE, passed); } -#endif \ No newline at end of file +#endif diff --git a/utest/test_extensions/test_domatcopy.c b/utest/test_extensions/test_domatcopy.c index e60b9c83d8..e892271d2d 100644 --- a/utest/test_extensions/test_domatcopy.c +++ b/utest/test_extensions/test_domatcopy.c @@ -94,6 +94,7 @@ static double check_domatcopy(char api, char order, char trans, blasint rows, bl BLASFUNC(domatcopy)(&order, &trans, &rows, &cols, &alpha, data_domatcopy.a_test, &lda, data_domatcopy.b_test, &ldb); } +#ifndef NO_CBLAS else { if (order == 'C') corder = CblasColMajor; if (order == 'R') corder = CblasRowMajor; @@ -104,6 +105,7 @@ static double check_domatcopy(char api, char order, char trans, blasint rows, bl cblas_domatcopy(corder, ctrans, rows, cols, alpha, data_domatcopy.a_test, lda, data_domatcopy.b_test, ldb); } +#endif return dmatrix_difference(data_domatcopy.b_test, data_domatcopy.b_verify, b_cols, b_rows, ldb); } @@ -412,6 +414,7 @@ CTEST(domatcopy, rowmajor_notrans_col_100_row_50) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test domatcopy by comparing it against refernce @@ -503,6 +506,7 @@ CTEST(domatcopy, c_api_rowmajor_notrans_col_100_row_100) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } +#endif /** * Test error function for an invalid param order. diff --git a/utest/test_extensions/test_drotmg.c b/utest/test_extensions/test_drotmg.c index 3073c8e3e9..3755776c9e 100644 --- a/utest/test_extensions/test_drotmg.c +++ b/utest/test_extensions/test_drotmg.c @@ -224,6 +224,7 @@ CTEST(drotmg, scaled_y_greater_than_scaled_x) } } +#ifndef NO_CBLAS /** * C API specific test * Test drotmg by comparing it against pre-calculated values @@ -411,4 +412,5 @@ CTEST(drotmg, c_api_scaled_y_greater_than_scaled_x) ASSERT_DBL_NEAR_TOL(tr_param[i], te_param[i], DOUBLE_EPS); } } -#endif \ No newline at end of file +#endif +#endif diff --git a/utest/test_extensions/test_dsum.c b/utest/test_extensions/test_dsum.c index 304619d0a9..c7e6d95614 100644 --- a/utest/test_extensions/test_dsum.c +++ b/utest/test_extensions/test_dsum.c @@ -221,6 +221,7 @@ CTEST(dsum, step_2_N_50){ ASSERT_DBL_NEAR_TOL(50.0, sum, DOUBLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test dsum by comparing it against pre-calculated values @@ -403,3 +404,4 @@ CTEST(dsum, c_api_step_2_N_50){ ASSERT_DBL_NEAR_TOL(50.0, sum, DOUBLE_EPS); } #endif +#endif diff --git a/utest/test_extensions/test_dzsum.c b/utest/test_extensions/test_dzsum.c index a184ad9475..318d7fbe3e 100644 --- a/utest/test_extensions/test_dzsum.c +++ b/utest/test_extensions/test_dzsum.c @@ -221,6 +221,7 @@ CTEST(dzsum, step_2_N_50){ ASSERT_DBL_NEAR_TOL(0.0, sum, DOUBLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test dzsum by comparing it against pre-calculated values @@ -403,3 +404,4 @@ CTEST(dzsum, c_api_step_2_N_50){ ASSERT_DBL_NEAR_TOL(0.0, sum, DOUBLE_EPS); } #endif +#endif diff --git a/utest/test_extensions/test_icamin.c b/utest/test_extensions/test_icamin.c index cca464eac6..8ac0844a95 100644 --- a/utest/test_extensions/test_icamin.c +++ b/utest/test_extensions/test_icamin.c @@ -331,6 +331,7 @@ CTEST(icamin, min_idx_in_vec_tail){ ASSERT_EQUAL(N, index); } +#ifndef NO_CBLAS /** * C API specific test * Test icamin by comparing it against pre-calculated values @@ -622,4 +623,5 @@ CTEST(icamin, c_api_min_idx_in_vec_tail){ blasint index = cblas_icamin(N, x, inc); ASSERT_EQUAL(N - 1, index); } -#endif \ No newline at end of file +#endif +#endif diff --git a/utest/test_extensions/test_idamin.c b/utest/test_extensions/test_idamin.c index bebe76dbae..4bee258a0a 100644 --- a/utest/test_extensions/test_idamin.c +++ b/utest/test_extensions/test_idamin.c @@ -413,6 +413,7 @@ CTEST(idamin, min_idx_in_vec_tail_inc_1){ ASSERT_EQUAL(N, index); } +#ifndef NO_CBLAS /** * C API specific test * Test idamin by comparing it against pre-calculated values @@ -787,3 +788,4 @@ CTEST(idamin, c_api_min_idx_in_vec_tail_inc_1){ ASSERT_EQUAL(N - 1, index); } #endif +#endif diff --git a/utest/test_extensions/test_isamin.c b/utest/test_extensions/test_isamin.c index d93813e6fb..a4a418473d 100644 --- a/utest/test_extensions/test_isamin.c +++ b/utest/test_extensions/test_isamin.c @@ -412,7 +412,7 @@ CTEST(isamin, min_idx_in_vec_tail_inc_1){ free(x); ASSERT_EQUAL(N, index); } - +#ifndef NO_CBLAS /** * C API specific test * Test isamin by comparing it against pre-calculated values @@ -787,3 +787,4 @@ CTEST(isamin, c_api_min_idx_in_vec_tail_inc_1){ ASSERT_EQUAL(N - 1, index); } #endif +#endif diff --git a/utest/test_extensions/test_izamin.c b/utest/test_extensions/test_izamin.c index a0bdae8e23..8c923c6090 100644 --- a/utest/test_extensions/test_izamin.c +++ b/utest/test_extensions/test_izamin.c @@ -331,6 +331,7 @@ CTEST(izamin, min_idx_in_vec_tail){ ASSERT_EQUAL(N, index); } +#ifndef NO_CBLAS /** * C API specific test * Test izamin by comparing it against pre-calculated values @@ -623,3 +624,4 @@ CTEST(izamin, c_api_min_idx_in_vec_tail){ ASSERT_EQUAL(N - 1, index); } #endif +#endif diff --git a/utest/test_extensions/test_saxpby.c b/utest/test_extensions/test_saxpby.c index b4bd5cf0bd..44f89240fb 100644 --- a/utest/test_extensions/test_saxpby.c +++ b/utest/test_extensions/test_saxpby.c @@ -96,6 +96,7 @@ static float check_saxpby(blasint n, float alpha, blasint incx, float beta, blas return BLASFUNC(snrm2)(&n, data_saxpby.y_test, &incy_abs); } +#ifndef NO_CBLAS /** * C API specific function * Test saxpby by comparing it with sscal and saxpy. @@ -141,6 +142,7 @@ static float c_api_check_saxpby(blasint n, float alpha, blasint incx, float beta // Find the norm of differences return cblas_snrm2(n, data_saxpby.y_test, incy_abs); } +#endif /** * Fortran API specific test @@ -467,6 +469,7 @@ CTEST(saxpby, check_n_zero) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test saxpby by comparing it with sscal and saxpy. @@ -791,4 +794,5 @@ CTEST(saxpby, c_api_check_n_zero) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } -#endif \ No newline at end of file +#endif +#endif diff --git a/utest/test_extensions/test_scsum.c b/utest/test_extensions/test_scsum.c index 8e943de229..7162808453 100644 --- a/utest/test_extensions/test_scsum.c +++ b/utest/test_extensions/test_scsum.c @@ -221,6 +221,7 @@ CTEST(scsum, step_2_N_50){ ASSERT_DBL_NEAR_TOL(0.0f, sum, SINGLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test scsum by comparing it against pre-calculated values @@ -403,3 +404,4 @@ CTEST(scsum, c_api_step_2_N_50){ ASSERT_DBL_NEAR_TOL(0.0f, sum, SINGLE_EPS); } #endif +#endif diff --git a/utest/test_extensions/test_sgeadd.c b/utest/test_extensions/test_sgeadd.c index 171132b9da..009066a9f0 100644 --- a/utest/test_extensions/test_sgeadd.c +++ b/utest/test_extensions/test_sgeadd.c @@ -63,10 +63,10 @@ static void sgeadd_trusted(blasint m, blasint n, float alpha, float *aptr, blasint lda, float beta, float *cptr, blasint ldc) { blasint i; - + blasint one=1; for (i = 0; i < n; i++) { - cblas_saxpby(m, alpha, aptr, 1, beta, cptr, 1); + BLASFUNC(saxpby)(&m, &alpha, aptr, &one, &beta, cptr, &one); aptr += lda; cptr += ldc; } @@ -115,9 +115,11 @@ static float check_sgeadd(char api, OPENBLAS_CONST enum CBLAS_ORDER order, BLASFUNC(sgeadd) (&m, &n, &alpha, data_sgeadd.a_test, &lda, &beta, data_sgeadd.c_test, &ldc); +#ifndef NO_CBLAS else cblas_sgeadd(order, m, n, alpha, data_sgeadd.a_test, lda, beta, data_sgeadd.c_test, ldc); +#endif // Find the differences between output matrix caculated by sgeadd and sgemm return smatrix_difference(data_sgeadd.c_test, data_sgeadd.c_verify, cols, rows, ldc); @@ -150,9 +152,11 @@ static int check_badargs(char api, OPENBLAS_CONST enum CBLAS_ORDER order, BLASFUNC(sgeadd) (&m, &n, &alpha, data_sgeadd.a_test, &lda, &beta, data_sgeadd.c_test, &ldc); +#ifndef NO_CBLAS else cblas_sgeadd(order, m, n, alpha, data_sgeadd.a_test, lda, beta, data_sgeadd.c_test, ldc); +#endif return check_error(); } @@ -420,6 +424,7 @@ CTEST(sgeadd, m_zero) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test sgeadd by comparing it against reference @@ -877,4 +882,5 @@ CTEST(sgeadd, c_api_m_zero) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } -#endif \ No newline at end of file +#endif +#endif diff --git a/utest/test_extensions/test_sgemmt.c b/utest/test_extensions/test_sgemmt.c index 5b51e3579b..177ce0d73b 100644 --- a/utest/test_extensions/test_sgemmt.c +++ b/utest/test_extensions/test_sgemmt.c @@ -73,9 +73,11 @@ static void sgemmt_trusted(char api, enum CBLAS_ORDER order, char uplo, char tra if(api == 'F') BLASFUNC(sgemm)(&transa, &transb, &m, &m, &k, &alpha, data_sgemmt.a_test, &lda, data_sgemmt.b_test, &ldb, &beta, data_sgemmt.c_gemm, &ldc); +#ifndef NO_CBLAS else cblas_sgemm(order, transa, transb, m, m, k, alpha, data_sgemmt.a_test, lda, data_sgemmt.b_test, ldb, beta, data_sgemmt.c_gemm, ldc); +#endif if (uplo == 'L' || uplo == CblasLower) { @@ -152,9 +154,11 @@ static float check_sgemmt(char api, enum CBLAS_ORDER order, char uplo, char tran if (api == 'F') BLASFUNC(sgemmt)(&uplo, &transa, &transb, &m, &k, &alpha, data_sgemmt.a_test, &lda, data_sgemmt.b_test, &ldb, &beta, data_sgemmt.c_test, &ldc); +#ifndef NO_CBLAS else cblas_sgemmt(order, uplo, transa, transb, m, k, alpha, data_sgemmt.a_test, lda, data_sgemmt.b_test, ldb, beta, data_sgemmt.c_test, ldc); +#endif for (i = 0; i < m * ldc; i++) data_sgemmt.c_verify[i] -= data_sgemmt.c_test[i]; @@ -189,9 +193,11 @@ static int check_badargs(char api, enum CBLAS_ORDER order, char uplo, char trans if (api == 'F') BLASFUNC(sgemmt)(&uplo, &transa, &transb, &m, &k, &alpha, data_sgemmt.a_test, &lda, data_sgemmt.b_test, &ldb, &beta, data_sgemmt.c_test, &ldc); +#ifndef NO_CBLAS else cblas_sgemmt(order, uplo, transa, transb, m, k, alpha, data_sgemmt.a_test, lda, data_sgemmt.b_test, ldb, beta, data_sgemmt.c_test, ldc); +#endif return check_error(); } @@ -480,6 +486,7 @@ CTEST(sgemmt, lower_beta_one) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test sgemmt by comparing it against sgemm @@ -1023,6 +1030,7 @@ CTEST(sgemmt, c_api_rowmajor_lower_beta_one) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } +#endif /** * Fortran API specific test @@ -1168,6 +1176,7 @@ CTEST(sgemmt, xerbla_ldc_invalid) ASSERT_EQUAL(TRUE, passed); } +#ifndef NO_CBLAS /** * C API specific test. * Test error function for an invalid param order. @@ -1439,4 +1448,5 @@ CTEST(sgemmt, xerbla_c_api_rowmajor_ldc_invalid) M, K, lda, ldb, ldc, expected_info); ASSERT_EQUAL(TRUE, passed); } -#endif \ No newline at end of file +#endif +#endif diff --git a/utest/test_extensions/test_simatcopy.c b/utest/test_extensions/test_simatcopy.c index ba388596db..c00ea0c8f0 100644 --- a/utest/test_extensions/test_simatcopy.c +++ b/utest/test_extensions/test_simatcopy.c @@ -93,6 +93,7 @@ static float check_simatcopy(char api, char order, char trans, blasint rows, bla BLASFUNC(simatcopy)(&order, &trans, &rows, &cols, &alpha, data_simatcopy.a_test, &lda_src, &lda_dst); } +#ifndef NO_CBLAS else { if (order == 'C') corder = CblasColMajor; if (order == 'R') corder = CblasRowMajor; @@ -103,6 +104,7 @@ static float check_simatcopy(char api, char order, char trans, blasint rows, bla cblas_simatcopy(corder, ctrans, rows, cols, alpha, data_simatcopy.a_test, lda_src, lda_dst); } +#endif // Find the differences between output matrix computed by simatcopy and reference func return smatrix_difference(data_simatcopy.a_test, data_simatcopy.a_verify, cols_out, rows_out, lda_dst); @@ -687,6 +689,7 @@ CTEST(simatcopy, rowmajor_notrans_col_100_row_50) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test simatcopy by comparing it against reference @@ -778,6 +781,7 @@ CTEST(simatcopy, c_api_rowmajor_notrans_col_100_row_100) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } +#endif /** * Test error function for an invalid param order. @@ -912,4 +916,4 @@ CTEST(simatcopy, xerbla_colmajor_trans_invalid_ldb) int passed = check_badargs(order, trans, m, n, lda_src, lda_dst, expected_info); ASSERT_EQUAL(TRUE, passed); } -#endif \ No newline at end of file +#endif diff --git a/utest/test_extensions/test_somatcopy.c b/utest/test_extensions/test_somatcopy.c index b53c7cae50..62a6056d92 100644 --- a/utest/test_extensions/test_somatcopy.c +++ b/utest/test_extensions/test_somatcopy.c @@ -94,6 +94,7 @@ static float check_somatcopy(char api, char order, char trans, blasint rows, bla BLASFUNC(somatcopy)(&order, &trans, &rows, &cols, &alpha, data_somatcopy.a_test, &lda, data_somatcopy.b_test, &ldb); } +#ifndef NO_CBLAS else { if (order == 'C') corder = CblasColMajor; if (order == 'R') corder = CblasRowMajor; @@ -104,7 +105,8 @@ static float check_somatcopy(char api, char order, char trans, blasint rows, bla cblas_somatcopy(corder, ctrans, rows, cols, alpha, data_somatcopy.a_test, lda, data_somatcopy.b_test, ldb); } - +#endif + return smatrix_difference(data_somatcopy.b_test, data_somatcopy.b_verify, b_cols, b_rows, ldb); } @@ -412,6 +414,7 @@ CTEST(somatcopy, rowmajor_notrans_col_100_row_50) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test somatcopy by comparing it against refernce @@ -503,6 +506,7 @@ CTEST(somatcopy, c_api_rowmajor_notrans_col_100_row_100) ASSERT_DBL_NEAR_TOL(0.0f, norm, SINGLE_EPS); } +#endif /** * Test error function for an invalid param order. diff --git a/utest/test_extensions/test_srotmg.c b/utest/test_extensions/test_srotmg.c index 3c97e3b4de..f0422d2b2b 100644 --- a/utest/test_extensions/test_srotmg.c +++ b/utest/test_extensions/test_srotmg.c @@ -224,6 +224,7 @@ CTEST(srotmg, scaled_y_greater_than_scaled_x) } } +#ifndef NO_CBLAS /** * C API specific test * Test srotmg by comparing it against pre-calculated values @@ -411,4 +412,5 @@ CTEST(srotmg, c_api_scaled_y_greater_than_scaled_x) ASSERT_DBL_NEAR_TOL(tr_param[i], te_param[i], SINGLE_EPS); } } -#endif \ No newline at end of file +#endif +#endif diff --git a/utest/test_extensions/test_ssum.c b/utest/test_extensions/test_ssum.c index 9c5e297286..5b20c0b031 100644 --- a/utest/test_extensions/test_ssum.c +++ b/utest/test_extensions/test_ssum.c @@ -221,6 +221,7 @@ CTEST(ssum, step_2_N_50){ ASSERT_DBL_NEAR_TOL(50.0f, sum, SINGLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test ssum by comparing it against pre-calculated values @@ -403,3 +404,4 @@ CTEST(ssum, c_api_step_2_N_50){ ASSERT_DBL_NEAR_TOL(50.0f, sum, SINGLE_EPS); } #endif +#endif diff --git a/utest/test_extensions/test_zaxpby.c b/utest/test_extensions/test_zaxpby.c index 6148f44c5f..d6ca999432 100644 --- a/utest/test_extensions/test_zaxpby.c +++ b/utest/test_extensions/test_zaxpby.c @@ -96,6 +96,7 @@ static double check_zaxpby(blasint n, double *alpha, blasint incx, double *beta, return BLASFUNC(dznrm2)(&n, data_zaxpby.y_test, &incy_abs); } +#ifndef NO_CBLAS /** * C API specific function * Test zaxpby by comparing it with zscal and zaxpy. @@ -145,6 +146,7 @@ static double c_api_check_zaxpby(blasint n, double *alpha, blasint incx, double // Find the norm of differences return cblas_dznrm2(n, data_zaxpby.y_test, incy_abs); } +#endif /** * Fortran API specific test @@ -387,6 +389,7 @@ CTEST(zaxpby, check_n_zero) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test zaxpby by comparing it with zscal and zaxpy. @@ -628,3 +631,4 @@ CTEST(zaxpby, c_api_check_n_zero) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } #endif +#endif diff --git a/utest/test_extensions/test_zgeadd.c b/utest/test_extensions/test_zgeadd.c index 7496ccf88c..466b94a52b 100644 --- a/utest/test_extensions/test_zgeadd.c +++ b/utest/test_extensions/test_zgeadd.c @@ -62,13 +62,14 @@ static void zgeadd_trusted(blasint m, blasint n, double *alpha, double *aptr, blasint lda, double *beta, double *cptr, blasint ldc) { blasint i; + blasint one=1; lda *= 2; ldc *= 2; for (i = 0; i < n; i++) { - cblas_zaxpby(m, alpha, aptr, 1, beta, cptr, 1); + BLASFUNC(zaxpby)(&m, alpha, aptr, &one, beta, cptr, &one); aptr += lda; cptr += ldc; } @@ -116,9 +117,11 @@ static double check_zgeadd(char api, OPENBLAS_CONST enum CBLAS_ORDER order, if (api == 'F') BLASFUNC(zgeadd)(&m, &n, alpha, data_zgeadd.a_test, &lda, beta, data_zgeadd.c_test, &ldc); +#ifndef NO_CBLAS else cblas_zgeadd(order, m, n, alpha, data_zgeadd.a_test, lda, beta, data_zgeadd.c_test, ldc); +#endif // Find the differences between output matrix caculated by zgeadd and sgemm return dmatrix_difference(data_zgeadd.c_test, data_zgeadd.c_verify, cols, rows, ldc * 2); @@ -150,9 +153,11 @@ static int check_badargs(char api, OPENBLAS_CONST enum CBLAS_ORDER order, if (api == 'F') BLASFUNC(zgeadd)(&m, &n, alpha, data_zgeadd.a_test, &lda, beta, data_zgeadd.c_test, &ldc); +#ifndef NO_CBLAS else cblas_zgeadd(order, m, n, alpha, data_zgeadd.a_test, lda, beta, data_zgeadd.c_test, ldc); +#endif return check_error(); } @@ -420,6 +425,7 @@ CTEST(zgeadd, m_zero) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test zgeadd by comparing it against reference @@ -877,4 +883,5 @@ CTEST(zgeadd, c_api_m_zero) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } -#endif \ No newline at end of file +#endif +#endif diff --git a/utest/test_extensions/test_zgemm.c b/utest/test_extensions/test_zgemm.c index 4160a50866..6dcc12724f 100644 --- a/utest/test_extensions/test_zgemm.c +++ b/utest/test_extensions/test_zgemm.c @@ -73,6 +73,7 @@ static double check_zgemm(char transa, char transb, blasint m, blasint n, blasin double alpha_conj[] = {1.0, 0.0}; char transa_verify = transa; char transb_verify = transb; + char cc[2]="C", cr[2]="R"; int arows = k, acols = m; int brows = n, bcols = k; @@ -99,12 +100,12 @@ static double check_zgemm(char transa, char transb, blasint m, blasint n, blasin data_zgemm.c_verify[i] = data_zgemm.c_test[i]; if (transa == 'R'){ - cblas_zimatcopy(CblasColMajor, CblasConjNoTrans, arows, acols, alpha_conj, data_zgemm.a_verify, lda, lda); + BLASFUNC(zimatcopy)(cc, cr, &arows, &acols, alpha_conj, data_zgemm.a_verify, &lda, &lda); transa_verify = 'N'; } if (transb == 'R'){ - cblas_zimatcopy(CblasColMajor, CblasConjNoTrans, brows, bcols, alpha_conj, data_zgemm.b_verify, ldb, ldb); + BLASFUNC(zimatcopy)(cc, cr, &brows, &bcols, alpha_conj, data_zgemm.b_verify, &ldb, &ldb); transb_verify = 'N'; } @@ -270,4 +271,4 @@ CTEST(zgemm, transa_conjnotransb) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } -#endif \ No newline at end of file +#endif diff --git a/utest/test_extensions/test_zgemmt.c b/utest/test_extensions/test_zgemmt.c index c553810084..34b8b61867 100644 --- a/utest/test_extensions/test_zgemmt.c +++ b/utest/test_extensions/test_zgemmt.c @@ -73,9 +73,11 @@ static void zgemmt_trusted(char api, enum CBLAS_ORDER order, char uplo, char tra if(api == 'F') BLASFUNC(zgemm)(&transa, &transb, &m, &m, &k, alpha, data_zgemmt.a_test, &lda, data_zgemmt.b_test, &ldb, beta, data_zgemmt.c_gemm, &ldc); +#ifndef NO_CBLAS else cblas_zgemm(order, transa, transb, m, m, k, alpha, data_zgemmt.a_test, lda, data_zgemmt.b_test, ldb, beta, data_zgemmt.c_gemm, ldc); +#endif ldc *= 2; @@ -160,9 +162,11 @@ static double check_zgemmt(char api, enum CBLAS_ORDER order, char uplo, char tra if (api == 'F') BLASFUNC(zgemmt)(&uplo, &transa, &transb, &m, &k, alpha, data_zgemmt.a_test, &lda, data_zgemmt.b_test, &ldb, beta, data_zgemmt.c_test, &ldc); +#ifndef NO_CBLAS else cblas_zgemmt(order, uplo, transa, transb, m, k, alpha, data_zgemmt.a_test, lda, data_zgemmt.b_test, ldb, beta, data_zgemmt.c_test, ldc); +#endif for (i = 0; i < m * ldc * 2; i++) data_zgemmt.c_verify[i] -= data_zgemmt.c_test[i]; @@ -197,9 +201,11 @@ static int check_badargs(char api, enum CBLAS_ORDER order, char uplo, char trans if (api == 'F') BLASFUNC(zgemmt)(&uplo, &transa, &transb, &m, &k, alpha, data_zgemmt.a_test, &lda, data_zgemmt.b_test, &ldb, beta, data_zgemmt.c_test, &ldc); +#ifndef NO_CBLAS else cblas_zgemmt(order, uplo, transa, transb, m, k, alpha, data_zgemmt.a_test, lda, data_zgemmt.b_test, ldb, beta, data_zgemmt.c_test, ldc); +#endif return check_error(); } @@ -680,6 +686,7 @@ CTEST(zgemmt, lower_beta_one) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test zgemmt by comparing it against sgemm @@ -1591,6 +1598,7 @@ CTEST(zgemmt, c_api_rowmajor_lower_beta_one) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } +#endif /** * Fortran API specific test @@ -1735,7 +1743,7 @@ CTEST(zgemmt, xerbla_ldc_invalid) M, K, lda, ldb, ldc, expected_info); ASSERT_EQUAL(TRUE, passed); } - +#ifndef NO_CBLAS /** * C API specific test. * Test error function for an invalid param order. @@ -2007,4 +2015,5 @@ CTEST(zgemmt, xerbla_c_api_rowmajor_ldc_invalid) M, K, lda, ldb, ldc, expected_info); ASSERT_EQUAL(TRUE, passed); } -#endif \ No newline at end of file +#endif +#endif diff --git a/utest/test_extensions/test_zgemv_t.c b/utest/test_extensions/test_zgemv_t.c index b2d0b27139..a575a0e4a8 100644 --- a/utest/test_extensions/test_zgemv_t.c +++ b/utest/test_extensions/test_zgemv_t.c @@ -65,6 +65,7 @@ static struct DATA_ZGEMV_T data_zgemv_t; static void matrix_vector_product(blasint n, blasint m, blasint lda, blasint inc_x) { blasint i; + blasint one=1; double *a_ptr = data_zgemv_t.a_verify; double *x_ptr = data_zgemv_t.x_test; double *x_res = data_zgemv_t.x_verify; @@ -73,7 +74,7 @@ static void matrix_vector_product(blasint n, blasint m, blasint lda, blasint inc for (i = 0; i < n * inc_x; i += inc_x) { - result = cblas_zdotu(lda, a_ptr, 1, x_ptr, inc_x); + result = BLASFUNC(zdotu)(&lda, a_ptr, &one, x_ptr, &inc_x); x_res[0] = CREAL(result); x_res[1] = CIMAG(result); a_ptr += lda * 2; @@ -157,6 +158,7 @@ static double check_zgemv(char api, char order, char trans, blasint m, blasint n BLASFUNC(zgemv)(&trans, &m, &n, alpha, data_zgemv_t.a_test, &lda, data_zgemv_t.x_test, &inc_x, beta, data_zgemv_t.y_test, &inc_y); } +#ifndef NO_CBLAS else { if (order == 'C') corder = CblasColMajor; if (order == 'R') corder = CblasRowMajor; @@ -177,13 +179,14 @@ static double check_zgemv(char api, char order, char trans, blasint m, blasint n cblas_zgemv(corder, ctrans, m, n, alpha, data_zgemv_t.a_test, lda, data_zgemv_t.x_test, inc_x, beta, data_zgemv_t.y_test, inc_y); } +#endif // Find the differences between output vector caculated by zgemv and reference funcs for (i = 0; i < m * inc_y * 2; i++) data_zgemv_t.y_test[i] -= data_zgemv_t.y_verify[i]; // Find the norm of differences - return cblas_dznrm2(m, data_zgemv_t.y_test, inc_y); + return BLASFUNC(dznrm2)(&m, data_zgemv_t.y_test, &inc_y); } /** @@ -217,7 +220,7 @@ static int check_badargs(char order, char trans, blasint m, blasint n, return check_error(); } - +#ifndef NO_CBLAS /** * C API specific function * Check if error function was called with expected function name @@ -1134,3 +1137,4 @@ CTEST(zgemv, c_api_xerbla_invalid_order_col_major) ASSERT_EQUAL(TRUE, passed); } #endif +#endif diff --git a/utest/test_extensions/test_zimatcopy.c b/utest/test_extensions/test_zimatcopy.c index 8376bc493a..86bc4670f2 100644 --- a/utest/test_extensions/test_zimatcopy.c +++ b/utest/test_extensions/test_zimatcopy.c @@ -98,6 +98,7 @@ static double check_zimatcopy(char api, char order, char trans, blasint rows, bl BLASFUNC(zimatcopy)(&order, &trans, &rows, &cols, alpha, data_zimatcopy.a_test, &lda_src, &lda_dst); } +#ifndef NO_CBLAS else { if (order == 'C') corder = CblasColMajor; if (order == 'R') corder = CblasRowMajor; @@ -108,6 +109,7 @@ static double check_zimatcopy(char api, char order, char trans, blasint rows, bl cblas_zimatcopy(corder, ctrans, rows, cols, alpha, data_zimatcopy.a_test, lda_src, lda_dst); } +#endif // Find the differences between output matrix computed by zimatcopy and reference func return dmatrix_difference(data_zimatcopy.a_test, data_zimatcopy.a_verify, cols_out, rows_out, lda_dst*2); @@ -502,6 +504,7 @@ CTEST(zimatcopy, rowmajor_conjtrans_col_50_row_100) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test zimatcopy by comparing it against reference @@ -681,6 +684,7 @@ CTEST(zimatcopy, c_api_rowmajor_conjtrans_col_100_row_100) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } +#endif /** * Test error function for an invalid param order. @@ -815,4 +819,4 @@ CTEST(zimatcopy, xerbla_colmajor_trans_invalid_ldb) int passed = check_badargs(order, trans, m, n, lda_src, lda_dst, expected_info); ASSERT_EQUAL(TRUE, passed); } -#endif \ No newline at end of file +#endif diff --git a/utest/test_extensions/test_zomatcopy.c b/utest/test_extensions/test_zomatcopy.c index 495831c56c..208cfd981c 100644 --- a/utest/test_extensions/test_zomatcopy.c +++ b/utest/test_extensions/test_zomatcopy.c @@ -99,6 +99,7 @@ static double check_zomatcopy(char api, char order, char trans, blasint rows, bl BLASFUNC(zomatcopy)(&order, &trans, &rows, &cols, alpha, data_zomatcopy.a_test, &lda, data_zomatcopy.b_test, &ldb); } +#ifndef NO_CBLAS else { if (order == 'C') corder = CblasColMajor; if (order == 'R') corder = CblasRowMajor; @@ -109,7 +110,8 @@ static double check_zomatcopy(char api, char order, char trans, blasint rows, bl cblas_zomatcopy(corder, ctrans, rows, cols, alpha, data_zomatcopy.a_test, lda, data_zomatcopy.b_test, ldb); } - +#endif + return dmatrix_difference(data_zomatcopy.b_test, data_zomatcopy.b_verify, b_cols, b_rows, ldb*2); } @@ -325,6 +327,7 @@ CTEST(zomatcopy, rowmajor_conjtrans_col_100_row_100) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test zomatcopy by comparing it against refernce @@ -508,6 +511,7 @@ CTEST(zomatcopy, c_api_rowmajor_conjtrans_col_100_row_100) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } +#endif /** * Test error function for an invalid param order. diff --git a/utest/test_extensions/test_zrot.c b/utest/test_extensions/test_zrot.c index 5471e051a6..c5ae22fc57 100644 --- a/utest/test_extensions/test_zrot.c +++ b/utest/test_extensions/test_zrot.c @@ -105,6 +105,7 @@ static double check_zdrot(blasint n, blasint inc_x, blasint inc_y, double *c, do return (norm / 2); } +#ifndef NO_CBLAS /** * C API specific function * Comapare results computed by zdrot and zaxpby @@ -787,4 +788,5 @@ CTEST(zrot, c_api_check_n_zero) double norm = c_api_check_zdrot(n, inc_x, inc_y, c, s); ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } -#endif \ No newline at end of file +#endif +#endif diff --git a/utest/test_extensions/test_zrotg.c b/utest/test_extensions/test_zrotg.c index 1de95447d8..c834bed6e6 100644 --- a/utest/test_extensions/test_zrotg.c +++ b/utest/test_extensions/test_zrotg.c @@ -162,6 +162,7 @@ CTEST(zrotg, negative_real_negative_img) ASSERT_DBL_NEAR_TOL(-7.01997150991369, sa[1], DOUBLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test zrotg by comparing it against pre-calculated values @@ -287,4 +288,5 @@ CTEST(zrotg, c_api_negative_real_negative_img) ASSERT_DBL_NEAR_TOL(-5.26497863243527, sa[0], DOUBLE_EPS); ASSERT_DBL_NEAR_TOL(-7.01997150991369, sa[1], DOUBLE_EPS); } -#endif \ No newline at end of file +#endif +#endif diff --git a/utest/test_extensions/test_zscal.c b/utest/test_extensions/test_zscal.c index 132f4ee5b6..63cf355ae4 100644 --- a/utest/test_extensions/test_zscal.c +++ b/utest/test_extensions/test_zscal.c @@ -92,8 +92,10 @@ static double check_zscal(char api, blasint n, double *alpha, blasint inc) if(api == 'F') BLASFUNC(zscal)(&n, alpha, data_zscal.x_test, &inc); +#ifndef NO_CBLAS else cblas_zscal(n, alpha, data_zscal.x_test, inc); +#endif // Find the differences between output vector computed by zscal and zscal_trusted for (i = 0; i < n * 2 * inc; i++) @@ -133,6 +135,7 @@ CTEST(zscal, alpha_r_zero_alpha_i_zero_inc_2) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } +#ifndef NO_CBLAS /** * C API specific test * Test zscal by comparing it against reference @@ -162,4 +165,5 @@ CTEST(zscal, c_api_alpha_r_zero_alpha_i_zero_inc_2) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } -#endif \ No newline at end of file +#endif +#endif diff --git a/utest/test_extensions/test_ztrmv.c b/utest/test_extensions/test_ztrmv.c index 5668ec2966..5819877bf2 100644 --- a/utest/test_extensions/test_ztrmv.c +++ b/utest/test_extensions/test_ztrmv.c @@ -65,7 +65,7 @@ static double check_ztrmv(char uplo, char trans, char diag, blasint n, blasint l blasint i; double alpha_conj[] = {1.0, 0.0}; char trans_verify = trans; - + char cc[2]="C", cr[2]="R"; drand_generate(data_ztrmv.a_test, n * lda * 2); drand_generate(data_ztrmv.x_test, n * incx * 2); @@ -76,7 +76,7 @@ static double check_ztrmv(char uplo, char trans, char diag, blasint n, blasint l data_ztrmv.x_verify[i] = data_ztrmv.x_test[i]; if (trans == 'R'){ - cblas_zimatcopy(CblasColMajor, CblasConjNoTrans, n, n, alpha_conj, data_ztrmv.a_verify, lda, lda); + BLASFUNC(zimatcopy)(cc, cr, &n, &n, alpha_conj, data_ztrmv.a_verify, &lda, &lda); trans_verify = 'N'; } @@ -263,4 +263,4 @@ CTEST(ztrmv, conj_notrans_lower_unit_triangular_incx_2) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } -#endif \ No newline at end of file +#endif diff --git a/utest/test_extensions/test_ztrsv.c b/utest/test_extensions/test_ztrsv.c index 4b7ec6aaf2..5db7963eea 100644 --- a/utest/test_extensions/test_ztrsv.c +++ b/utest/test_extensions/test_ztrsv.c @@ -65,6 +65,7 @@ static double check_ztrsv(char uplo, char trans, char diag, blasint n, blasint l blasint i; double alpha_conj[] = {1.0, 0.0}; char trans_verify = trans; + char cc[2]="C", cr[2]="R"; drand_generate(data_ztrsv.a_test, n * lda * 2); drand_generate(data_ztrsv.x_test, n * incx * 2); @@ -76,8 +77,8 @@ static double check_ztrsv(char uplo, char trans, char diag, blasint n, blasint l data_ztrsv.x_verify[i] = data_ztrsv.x_test[i]; if (trans == 'R'){ - cblas_zimatcopy(CblasColMajor, CblasConjNoTrans, n, n, - alpha_conj, data_ztrsv.a_verify, lda, lda); + BLASFUNC(zimatcopy)(cc, cr, &n, &n, + alpha_conj, data_ztrsv.a_verify, &lda, &lda); trans_verify = 'N'; } @@ -264,4 +265,4 @@ CTEST(ztrsv, conj_notrans_lower_unit_triangular_incx_2) ASSERT_DBL_NEAR_TOL(0.0, norm, DOUBLE_EPS); } -#endif \ No newline at end of file +#endif From 243640c3546b48af07bdb62265efa8c8b13f3208 Mon Sep 17 00:00:00 2001 From: Matti Picus Date: Sun, 12 May 2024 10:24:16 +0300 Subject: [PATCH 072/267] use blasint instead of int to quiet warnings --- utest/test_extensions/test_cgemm.c | 4 ++-- utest/test_extensions/test_zgemm.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/utest/test_extensions/test_cgemm.c b/utest/test_extensions/test_cgemm.c index 08a26554a0..15d64e372e 100644 --- a/utest/test_extensions/test_cgemm.c +++ b/utest/test_extensions/test_cgemm.c @@ -75,8 +75,8 @@ static float check_cgemm(char transa, char transb, blasint m, blasint n, blasint char transb_verify = transb; char cc[2]="C", cr[2]="R"; - int arows = k, acols = m; - int brows = n, bcols = k; + blasint arows = k, acols = m; + blasint brows = n, bcols = k; if (transa == 'T' || transa == 'C'){ arows = m; acols = k; diff --git a/utest/test_extensions/test_zgemm.c b/utest/test_extensions/test_zgemm.c index 6dcc12724f..bd23ebca42 100644 --- a/utest/test_extensions/test_zgemm.c +++ b/utest/test_extensions/test_zgemm.c @@ -75,8 +75,8 @@ static double check_zgemm(char transa, char transb, blasint m, blasint n, blasin char transb_verify = transb; char cc[2]="C", cr[2]="R"; - int arows = k, acols = m; - int brows = n, bcols = k; + blasint arows = k, acols = m; + blasint brows = n, bcols = k; if (transa == 'T' || transa == 'C'){ arows = m; acols = k; From 87b3d9054f8a615457fa5795bbab983c24ebe029 Mon Sep 17 00:00:00 2001 From: Amrita H S Date: Tue, 7 May 2024 11:31:36 -0500 Subject: [PATCH 073/267] Fix regression SAXPY when compiler with OpenXL compiler. SAXPY built with OpenXL regresses when compared to SAXPY built with gcc. OpenXL compiler doesn't know that the SAXPY inner kernel assembly is a 64 element loop and to it the remainder loop is the main loop. It vectorizes and interleaves the remainder to be a 48 elements per iteration loop. With a max of 63 iterations, a 48 element loop is mostly not going to get executed, so the 1 element scalar loop that is the remainder after that is probably mostly what gets executed. This can be fixed by adding a pragma, loop interleave_count(2) which will result in 8 element loop. Signed-off-by: Amrita H S --- kernel/power/saxpy_power10.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernel/power/saxpy_power10.c b/kernel/power/saxpy_power10.c index 302b2418e3..a01e1b53da 100644 --- a/kernel/power/saxpy_power10.c +++ b/kernel/power/saxpy_power10.c @@ -76,6 +76,9 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS saxpy_kernel_64(n1, &x[i], &y[i], da); i += n1; +#if defined(__clang__) +#pragma clang loop interleave_count(2) +#endif while(i < n) { From b45a78c6e9eae6ea46f25478f5aab257a53a8e21 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Mon, 13 May 2024 14:50:50 +0200 Subject: [PATCH 074/267] fix zdotu argument passing in utest_ext on windows (#4691) * fix passing of results on windows --- utest/test_extensions/test_cgemv_t.c | 4 ++++ utest/test_extensions/test_zgemv_t.c | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/utest/test_extensions/test_cgemv_t.c b/utest/test_extensions/test_cgemv_t.c index ceb0b4bc59..dd95d32bad 100644 --- a/utest/test_extensions/test_cgemv_t.c +++ b/utest/test_extensions/test_cgemv_t.c @@ -74,7 +74,11 @@ static void matrix_vector_product(blasint n, blasint m, blasint lda, blasint inc for (i = 0; i < n * inc_x; i+= inc_x) { +#ifdef RETURN_BY_STACK + BLASFUNC(cdotu)(&result, &lda, a_ptr, &one, x_ptr, &inc_x); +#else result = BLASFUNC(cdotu)(&lda, a_ptr, &one, x_ptr, &inc_x); +#endif x_res[0] = CREAL(result); x_res[1] = CIMAG(result); a_ptr += lda * 2; diff --git a/utest/test_extensions/test_zgemv_t.c b/utest/test_extensions/test_zgemv_t.c index a575a0e4a8..4e419ad1ba 100644 --- a/utest/test_extensions/test_zgemv_t.c +++ b/utest/test_extensions/test_zgemv_t.c @@ -74,8 +74,12 @@ static void matrix_vector_product(blasint n, blasint m, blasint lda, blasint inc for (i = 0; i < n * inc_x; i += inc_x) { - result = BLASFUNC(zdotu)(&lda, a_ptr, &one, x_ptr, &inc_x); - x_res[0] = CREAL(result); +#ifdef RETURN_BY_STACK + BLASFUNC(zdotu)(&result, &lda, a_ptr, &one, x_ptr, &inc_x); +#else + result = BLASFUNC(zdotu)(&lda, a_ptr, &one, x_ptr, &inc_x); +#endif + x_res[0] = CREAL(result); x_res[1] = CIMAG(result); a_ptr += lda * 2; x_res += 2 * inc_x; From 0dc80a5c8dfc43d1550f489b1f4abba36eee675a Mon Sep 17 00:00:00 2001 From: Deeksha Goplani Date: Mon, 13 May 2024 22:17:23 +0530 Subject: [PATCH 075/267] locks improvement --- driver/others/memory.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/driver/others/memory.c b/driver/others/memory.c index 784a1f0282..53e49b452d 100644 --- a/driver/others/memory.c +++ b/driver/others/memory.c @@ -2746,7 +2746,7 @@ struct newmemstruct }; static volatile struct newmemstruct *newmemory; -static int memory_initialized = 0; +static volatile int memory_initialized = 0; static int memory_overflowed = 0; /* Memory allocation routine */ /* procpos ... indicates where it comes from */ @@ -2791,14 +2791,12 @@ void *blas_memory_alloc(int procpos){ }; void *(**func)(void *address); -#if defined(USE_OPENMP) if (!memory_initialized) { +#if defined(SMP) && !defined(USE_OPENMP) + LOCK_COMMAND(&alloc_lock); + if (!memory_initialized) { #endif - LOCK_COMMAND(&alloc_lock); - - if (!memory_initialized) { - #if defined(WHEREAMI) && !defined(USE_OPENMP) for (position = 0; position < NUM_BUFFERS; position ++){ memory[position].addr = (void *)0; @@ -2827,12 +2825,12 @@ void *blas_memory_alloc(int procpos){ #endif memory_initialized = 1; - + WMB; +#if defined(SMP) && !defined(USE_OPENMP) } UNLOCK_COMMAND(&alloc_lock); -#if defined(USE_OPENMP) - } #endif +} #ifdef DEBUG printf("Alloc Start ...\n"); From 5fd871d7eabdb5e33f5d2ef81455ecac4ac098ab Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 14 May 2024 18:48:03 +0200 Subject: [PATCH 076/267] Introduce a lower limit for multithreading --- interface/gbmv.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/interface/gbmv.c b/interface/gbmv.c index 1d58ba807f..7a65813685 100644 --- a/interface/gbmv.c +++ b/interface/gbmv.c @@ -227,7 +227,10 @@ void CNAME(enum CBLAS_ORDER order, buffer = (FLOAT *)blas_memory_alloc(1); #ifdef SMP - nthreads = num_cpu_avail(2); + if (m * n < 250000 || kl+ku < 15 ) + nthreads = 1; + else + nthreads = num_cpu_avail(2); if (nthreads == 1) { #endif From 2957281275810ab8bc87304134552cdd046216f2 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 14 May 2024 18:59:21 +0200 Subject: [PATCH 077/267] Introduce a lower limit for multithreading --- interface/zgbmv.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/interface/zgbmv.c b/interface/zgbmv.c index 5e275a8edc..5128b22e15 100644 --- a/interface/zgbmv.c +++ b/interface/zgbmv.c @@ -251,7 +251,10 @@ void CNAME(enum CBLAS_ORDER order, buffer = (FLOAT *)blas_memory_alloc(1); #ifdef SMP - nthreads = num_cpu_avail(2); + if (m * n < 125000 || ku + kl < 15) + nthreads = 1; + else + nthreads = num_cpu_avail(2); if (nthreads == 1) { #endif From 56d3d1039c945d630da3187a7e86d8c25b325340 Mon Sep 17 00:00:00 2001 From: frjohnst Date: Wed, 15 May 2024 09:44:29 -0400 Subject: [PATCH 078/267] Revert "resolve second_ conflict which breaks xlf timef" This reverts commit 9b24b3141985de115e6191b376b435117a32404a. It turns out that PRs 4515 and 4520 break the tests under lapack-netlib/TESTING which require SECOND and DSECND. IBM has decided this is a bigger biger problem than the conflict between lapack second_ and the xlf run time. --- lapack-netlib/SRC/Makefile | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/lapack-netlib/SRC/Makefile b/lapack-netlib/SRC/Makefile index 205a32d31b..de22427016 100644 --- a/lapack-netlib/SRC/Makefile +++ b/lapack-netlib/SRC/Makefile @@ -101,10 +101,8 @@ SCLAUX = la_constants.o \ slaset.o slasq1.o slasq2.o slasq3.o slasq4.o slasq5.o slasq6.o \ slasr.o slasrt.o slassq.o slasv2.o spttrf.o sstebz.o sstedc.o \ ssteqr.o ssterf.o slaisnan.o sisnan.o \ - slartgp.o slartgs.o scombssq.o ../INSTALL/sroundup_lwork.o -ifneq ($(F_COMPILER), IBM) -SCLAUX += ../INSTALL/second_$(TIMER).o -endif + slartgp.o slartgs.o scombssq.o ../INSTALL/sroundup_lwork.o \ + ../INSTALL/second_$(TIMER).o endif ifneq "$(or $(BUILD_DOUBLE),$(BUILD_COMPLEX16))" "" @@ -126,10 +124,7 @@ DZLAUX = la_constants.o\ dlasr.o dlasrt.o dlassq.o dlasv2.o dpttrf.o dstebz.o dstedc.o \ dsteqr.o dsterf.o dlaisnan.o disnan.o \ dlartgp.o dlartgs.o ../INSTALL/droundup_lwork.o \ - ../INSTALL/dlamch.o -ifneq ($(F_COMPILER), IBM) -DZLAUX += ../INSTALL/dsecnd_$(TIMER).o -endif + ../INSTALL/dlamch.o ../INSTALL/dsecnd_$(TIMER).o endif #ifeq ($(BUILD_SINGLE),1) From 87026ac1b17491aa16eb9cda53945980cfe20bb6 Mon Sep 17 00:00:00 2001 From: frjohnst Date: Wed, 15 May 2024 09:45:17 -0400 Subject: [PATCH 079/267] Revert "fix conlict between PR 4515 and AIX shared obj support" This reverts commit bdaa6705ca7b0031223417402e3f87e85fef86ac. It turns out that PRs 4515 and 4520 break the tests under lapack-netlib/TESTING which require SECOND and DSECND. IBM has decided this is a bigger biger problem than the conflict between lapack second_ and the xlf run time. --- exports/Makefile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/exports/Makefile b/exports/Makefile index 457e59b2c8..668a4866e8 100644 --- a/exports/Makefile +++ b/exports/Makefile @@ -315,11 +315,6 @@ test : linktest.c linktest.c : $(GENSYM) ../Makefile.system ../getarch.c ./$(GENSYM) linktest $(ARCH) "$(BU)" $(EXPRECISION) $(NO_CBLAS) $(NO_LAPACK) $(NO_LAPACKE) $(NEED2UNDERSCORES) $(ONLY_CBLAS) "$(SYMBOLPREFIX)" "$(SYMBOLSUFFIX)" $(BUILD_LAPACK_DEPRECATED) $(BUILD_BFLOAT16) $(BUILD_SINGLE) $(BUILD_DOUBLE) $(BUILD_COMPLEX) $(BUILD_COMPLEX16) > linktest.c -ifeq ($(F_COMPILER), IBM) - mv linktest.c linktest.c.FIRST - egrep -v 'second_|dsecnd_' linktest.c.FIRST > linktest.c - rm linktest.c.FIRST -endif clean :: @rm -f *.def *.dylib __.SYMDEF* *.renamed From f5c080f0831a1fdf3ddad1106be920ac7c1987db Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 15 May 2024 20:58:31 +0200 Subject: [PATCH 080/267] Fix CMAKE syntax in kernel file parsing of IFNEQ conditionals (#4695) * Fix syntax in parsing of IFNEQ --- cmake/utils.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/utils.cmake b/cmake/utils.cmake index 5e8ba866be..9befc9a3c4 100644 --- a/cmake/utils.cmake +++ b/cmake/utils.cmake @@ -187,8 +187,8 @@ macro(ParseMakefileVars MAKEFILE_IN) set (HasValidGroup 1) set (STR ${CMAKE_MATCH_4}) endif () - if (DEFINED ${CMAKE_MATCH_1} AND ${HasValidGroup} EQUAL 1) - if (NOT (${${CMAKE_MATCH_1}} STREQUAL ${STR})) + if (DEFINED CMAKE_MATCH_1 AND ${HasValidGroup} EQUAL 1) + if (NOT (CMAKE_MATCH_1 STREQUAL ${STR})) #message (STATUS "condition is true") set (IfElse 1) continue () From 668f48f4fc80db2d886576f20b7d4ddb6defd4c1 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 15 May 2024 23:58:14 +0200 Subject: [PATCH 081/267] Use CMAKE_C_COMPILER_VERSION instead of dumpversion calls (#4698) * Use CMAKE_C_COMPILER_VERSION throughout --- cmake/cc.cmake | 15 +++------------ cmake/os.cmake | 2 +- cmake/system.cmake | 26 +++++++++----------------- 3 files changed, 13 insertions(+), 30 deletions(-) diff --git a/cmake/cc.cmake b/cmake/cc.cmake index 0a498510aa..a162986f85 100644 --- a/cmake/cc.cmake +++ b/cmake/cc.cmake @@ -5,10 +5,10 @@ include(CheckCCompilerFlag) if (${CMAKE_C_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_C_COMPILER_ID} STREQUAL "LSB" OR ${CMAKE_C_COMPILER_ID} MATCHES "Clang") - set(CCOMMON_OPT "${CCOMMON_OPT} -Wall") set(COMMON_PROF "${COMMON_PROF} -fno-inline") set(NO_UNINITIALIZED_WARN "-Wno-uninitialized") + set(GCC_VERSION ${CMAKE_C_COMPILER_VERSION}) if (QUIET_MAKE) set(CCOMMON_OPT "${CCOMMON_OPT} ${NO_UNINITIALIZED_WARN} -Wno-unused") @@ -140,7 +140,6 @@ endif () if (${CORE} STREQUAL COOPERLAKE) if (NOT DYNAMIC_ARCH) if (NOT NO_AVX512) - execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) if (${GCC_VERSION} VERSION_GREATER 10.1 OR ${GCC_VERSION} VERSION_EQUAL 10.1) set (CCOMMON_OPT "${CCOMMON_OPT} -march=cooperlake") else () @@ -153,7 +152,6 @@ endif () if (${CORE} STREQUAL SAPPHIRERAPIDS) if (NOT DYNAMIC_ARCH) if (NOT NO_AVX512) - execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) if (${GCC_VERSION} VERSION_GREATER 11.0 OR ${GCC_VERSION} VERSION_EQUAL 11.0) set (CCOMMON_OPT "${CCOMMON_OPT} -march=sapphirerapids") else () @@ -167,7 +165,6 @@ if (${CORE} STREQUAL ZEN) if (HAVE_AVX512VL) if (NOT DYNAMIC_ARCH) if (NOT NO_AVX512) - execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) if (${GCC_VERSION} VERSION_GREATER 13.0 OR ${GCC_VERSION} VERSION_EQUAL 13.0) set (CCOMMON_OPT "${CCOMMON_OPT} -march=znver4") else () @@ -180,7 +177,6 @@ endif () if (${CORE} STREQUAL A64FX) if (NOT DYNAMIC_ARCH) - execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) if (${GCC_VERSION} VERSION_GREATER 11.0 OR ${GCC_VERSION} VERSION_EQUAL 11.0) set (CCOMMON_OPT "${CCOMMON_OPT} -march=armv8.2-a+sve -mtune=a64fx") else () @@ -194,7 +190,6 @@ if (${CORE} STREQUAL NEOVERSEN2) if (${CMAKE_C_COMPILER_ID} STREQUAL "PGI" AND NOT NO_SVE) set (CCOMMON_OPT "${CCOMMON_OPT} -Msve_intrinsics -march=armv8.5-a+sve+sve2+bf16 -mtune=neoverse-n2") else () - execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) if (${GCC_VERSION} VERSION_GREATER 10.4 OR ${GCC_VERSION} VERSION_EQUAL 10.4) set (CCOMMON_OPT "${CCOMMON_OPT} -march=armv8.5-a+sve+sve2+bf16 -mtune=neoverse-n2") else () @@ -209,7 +204,6 @@ if (${CORE} STREQUAL NEOVERSEV1) if (${CMAKE_C_COMPILER_ID} STREQUAL "PGI" AND NOT NO_SVE) set (CCOMMON_OPT "${CCOMMON_OPT} -Msve_intrinsics -march=armv8.4-a+sve -mtune=neoverse-v1") else () - execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) if (${GCC_VERSION} VERSION_GREATER 10.4 OR ${GCC_VERSION} VERSION_EQUAL 10.4) set (CCOMMON_OPT "${CCOMMON_OPT} -march=armv8.4-a+sve -mtune=neoverse-v1") else () @@ -221,7 +215,6 @@ endif () if (${CORE} STREQUAL NEOVERSEN1) if (NOT DYNAMIC_ARCH) - execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) if (${GCC_VERSION} VERSION_GREATER 9.4 OR ${GCC_VERSION} VERSION_EQUAL 9.4) set (CCOMMON_OPT "${CCOMMON_OPT} -march=armv8.2-a+sve -mtune=neoverse-n1") else () @@ -266,23 +259,21 @@ endif () if (${CORE} STREQUAL POWER10) if (NOT DYNAMIC_ARCH) - execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) if (${GCC_VERSION} VERSION_GREATER 10.2 OR ${GCC_VERSION} VERSION_EQUAL 10.2) set (CCOMMON_OPT "${CCOMMON_OPT} -mcpu=power10 -mtune=power10 -mvsx -fno-fast-math") else () - message(FATAL_ERROR "Compiler GCC.${GCC_VERSION} does not support Power10." ) + message(FATAL_ERROR "Compiler GCC ${GCC_VERSION} does not support Power10." ) endif() endif () endif () if (${CORE} STREQUAL POWER9) if (NOT DYNAMIC_ARCH) - execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) if (${GCC_VERSION} VERSION_GREATER 5.0 OR ${GCC_VERSION} VERSION_EQUAL 5.0) set (CCOMMON_OPT "${CCOMMON_OPT} -mcpu=power9 -mtune=power9 -mvsx -fno-fast-math") else () set (CCOMMON_OPT "${CCOMMON_OPT} -mcpu=power8 -mtune=power8 -mvsx -fno-fast-math") - message(WARNING "Compiler GCC.${GCC_VERSION} does not fully support Power9.") + message(WARNING "Compiler GCC ${GCC_VERSION} does not fully support Power9.") endif () endif () endif () diff --git a/cmake/os.cmake b/cmake/os.cmake index e24059dd5b..2effbe0e5b 100644 --- a/cmake/os.cmake +++ b/cmake/os.cmake @@ -38,7 +38,7 @@ if (${CMAKE_SYSTEM_NAME} STREQUAL "Windows") # Test for supporting MS_ABI # removed string parsing in favor of CMake's version comparison -hpa - execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) + set(GCC_VERSION ${CMAKE_C_COMPILER_VERSION}) if (${GCC_VERSION} VERSION_GREATER 4.7 OR ${GCC_VERSION} VERSION_EQUAL 4.7) # GCC Version >=4.7 # It is compatible with MSVC ABI. diff --git a/cmake/system.cmake b/cmake/system.cmake index 6fec04bfcf..3c60619584 100644 --- a/cmake/system.cmake +++ b/cmake/system.cmake @@ -170,7 +170,6 @@ include("${PROJECT_SOURCE_DIR}/cmake/prebuild.cmake") if (DEFINED TARGET) if (${TARGET} STREQUAL COOPERLAKE AND NOT NO_AVX512) if (${CMAKE_C_COMPILER_ID} STREQUAL "GNU") - execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) if (${CMAKE_C_COMPILER_VERSION} VERSION_GREATER 10.09) set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=cooperlake") else() @@ -186,7 +185,6 @@ if (DEFINED TARGET) endif() if (${TARGET} STREQUAL SAPPHIRERAPIDS AND NOT NO_AVX512) if (${CMAKE_C_COMPILER_ID} STREQUAL "GNU") - execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) if (${CMAKE_C_COMPILER_VERSION} VERSION_GREATER 11.0) set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=sapphirerapids") else() @@ -209,7 +207,6 @@ if (DEFINED TARGET) if (((${TARGET} STREQUAL ZEN) AND HAVE_AVX512VL) AND NOT NO_AVX512) if (${CMAKE_C_COMPILER_ID} STREQUAL "GNU") - execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) if (${CMAKE_C_COMPILER_VERSION} VERSION_GREATER 12.99) set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=znver4") else() @@ -227,8 +224,7 @@ if (DEFINED TARGET) if ((${TARGET} STREQUAL HASWELL OR (${TARGET} STREQUAL ZEN AND NOT HAVE_AVX512VL)) AND NOT NO_AVX2) if (${CMAKE_C_COMPILER_ID} STREQUAL "GNU") - execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) - if (${GCC_VERSION} VERSION_GREATER 4.7 OR ${GCC_VERSION} VERSION_EQUAL 4.7) + if (CMAKE_C_COMPILER_VERSION} VERSION_GREATER 4.7 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 4.7) set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -mavx2") endif() elseif (${CMAKE_C_COMPILER_ID} STREQUAL "CLANG") @@ -267,20 +263,18 @@ if (DEFINED TARGET) endif() if (${TARGET} STREQUAL POWER10) - execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) - if (${GCC_VERSION} VERSION_GREATER 10.2 OR ${GCC_VERSION} VERSION_EQUAL 10.2) + if (CMAKE_C_COMPILER VERSION VERSION_GREATER 10.2 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 10.2) set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -mcpu=power10 -mtune=power10 -mvsx -fno-fast-math") else () - message(FATAL_ERROR "Compiler GCC.${GCC_VERSION} does not support Power10.") + message(FATAL_ERROR "Compiler GCC ${CMAKE_C_COMPILER_VERSION} does not support Power10.") endif() endif() if (${TARGET} STREQUAL POWER9) - execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) - if (${GCC_VERSION} VERSION_GREATER 5.0 OR ${GCC_VERSION} VERSION_EQUAL 5.0) + if (CMAKE_C_COMPILER_VERSION VERSION_GREATER 5.0 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 5.0) set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -mcpu=power9 -mtune=power9 -mvsx -fno-fast-math") else () set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -mcpu=power8 -mtune=power8 -mvsx -fno-fast-math") - message(WARNING "Compiler GCC.${GCC_VERSION} does not support fully Power9.") + message(WARNING "Compiler GCC ${CMAKE_C_COMPILER_VERSION} does not support fully Power9.") endif() endif() if (${TARGET} STREQUAL POWER8) @@ -291,11 +285,10 @@ if (${TARGET} STREQUAL NEOVERSEV1) if (${CMAKE_C_COMPILER_ID} STREQUAL "PGI" AND NOT NO_SVE) set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -Msve_intrinsics -march=armv8.4-a+sve -mtune=neoverse-v1") else () - execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) - if (${GCC_VERSION} VERSION_GREATER 10.4 OR ${GCC_VERSION} VERSION_EQUAL 10.4) + if (CMAKE_C_COMPILER_VERSION VERSION_GREATER 10.4 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 10.4) set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=armv8.4-a+sve -mtune=neoverse-v1") else () - message(FATAL_ERROR "Compiler ${CMAKE_C_COMPILER} ${GCC_VERSION} does not support Neoverse V1.") + message(FATAL_ERROR "Compiler ${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_VERSION} does not support Neoverse V1.") endif() endif() endif() @@ -303,11 +296,10 @@ if (${TARGET} STREQUAL NEOVERSEV1) if (${CMAKE_C_COMPILER_ID} STREQUAL "PGI" AND NOT NO_SVE) set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -Msve-intrinsics -march=armv8.5-a+sve+sve2+bf16 -mtune=neoverse-n2") else () - execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) - if (${GCC_VERSION} VERSION_GREATER 10.4 OR ${GCC_VERSION} VERSION_EQUAL 10.4) + if (CMAKE_C_COMPILER_VERSION VERSION_GREATER 10.4 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 10.4) set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=armv8.5-a+sve+sve2+bf16 -mtune=neoverse-n2") else () - message(FATAL_ERROR "Compiler $${CMAKE_C_COMPILER} {GCC_VERSION} does not support Neoverse N2.") + message(FATAL_ERROR "Compiler $${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_VERSION} does not support Neoverse N2.") endif() endif() endif() From 3a1417671a702e1925b1fff6b78163267823314c Mon Sep 17 00:00:00 2001 From: Chip Kerchner Date: Wed, 15 May 2024 19:36:46 -0500 Subject: [PATCH 082/267] POWER: Fixing endianness issue in cswap/zswap kernel for AIX --- kernel/power/cswap_microk_power10.c | 44 +++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/kernel/power/cswap_microk_power10.c b/kernel/power/cswap_microk_power10.c index 2a44a9e30e..f71b6f98c4 100644 --- a/kernel/power/cswap_microk_power10.c +++ b/kernel/power/cswap_microk_power10.c @@ -58,6 +58,16 @@ static void cswap_kernel_32 (long n, float *x, float *y) "lxvp 62, 224(%3) \n\t" +#if (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + "stxv 32, 0(%3) \n\t" + "stxv 33, 16(%3) \n\t" + "stxv 34, 32(%3) \n\t" + "stxv 35, 48(%3) \n\t" + "stxv 36, 64(%3) \n\t" + "stxv 37, 80(%3) \n\t" + "stxv 38, 96(%3) \n\t" + "stxv 39, 112(%3) \n\t" +#else "stxv 33, 0(%3) \n\t" "stxv 32, 16(%3) \n\t" "stxv 35, 32(%3) \n\t" @@ -66,9 +76,20 @@ static void cswap_kernel_32 (long n, float *x, float *y) "stxv 36, 80(%3) \n\t" "stxv 39, 96(%3) \n\t" "stxv 38, 112(%3) \n\t" +#endif "addi %3, %3, 128 \n\t" +#if (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + "stxv 40, 0(%3) \n\t" + "stxv 41, 16(%3) \n\t" + "stxv 42, 32(%3) \n\t" + "stxv 43, 48(%3) \n\t" + "stxv 44, 64(%3) \n\t" + "stxv 45, 80(%3) \n\t" + "stxv 46, 96(%3) \n\t" + "stxv 47, 112(%3) \n\t" +#else "stxv 41, 0(%3) \n\t" "stxv 40, 16(%3) \n\t" "stxv 43, 32(%3) \n\t" @@ -77,9 +98,20 @@ static void cswap_kernel_32 (long n, float *x, float *y) "stxv 44, 80(%3) \n\t" "stxv 47, 96(%3) \n\t" "stxv 46, 112(%3) \n\t" +#endif "addi %3, %3, 128 \n\t" +#if (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + "stxv 48, 0(%4) \n\t" + "stxv 49, 16(%4) \n\t" + "stxv 50, 32(%4) \n\t" + "stxv 51, 48(%4) \n\t" + "stxv 52, 64(%4) \n\t" + "stxv 53, 80(%4) \n\t" + "stxv 54, 96(%4) \n\t" + "stxv 55, 112(%4) \n\t" +#else "stxv 49, 0(%4) \n\t" "stxv 48, 16(%4) \n\t" "stxv 51, 32(%4) \n\t" @@ -88,9 +120,20 @@ static void cswap_kernel_32 (long n, float *x, float *y) "stxv 52, 80(%4) \n\t" "stxv 55, 96(%4) \n\t" "stxv 54, 112(%4) \n\t" +#endif "addi %4, %4, 128 \n\t" +#if (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + "stxv 56, 0(%4) \n\t" + "stxv 57, 16(%4) \n\t" + "stxv 58, 32(%4) \n\t" + "stxv 59, 48(%4) \n\t" + "stxv 60, 64(%4) \n\t" + "stxv 61, 80(%4) \n\t" + "stxv 62, 96(%4) \n\t" + "stxv 63, 112(%4) \n\t" +#else "stxv 57, 0(%4) \n\t" "stxv 56, 16(%4) \n\t" "stxv 59, 32(%4) \n\t" @@ -99,6 +142,7 @@ static void cswap_kernel_32 (long n, float *x, float *y) "stxv 60, 80(%4) \n\t" "stxv 63, 96(%4) \n\t" "stxv 62, 112(%4) \n\t" +#endif "addi %4, %4, 128 \n\t" From a3f6b13bc9bfa5773b0f115fe1b134393507337c Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 16 May 2024 09:25:53 +0200 Subject: [PATCH 083/267] remove spurious brace --- cmake/system.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/system.cmake b/cmake/system.cmake index 3c60619584..479f50538f 100644 --- a/cmake/system.cmake +++ b/cmake/system.cmake @@ -224,7 +224,7 @@ if (DEFINED TARGET) if ((${TARGET} STREQUAL HASWELL OR (${TARGET} STREQUAL ZEN AND NOT HAVE_AVX512VL)) AND NOT NO_AVX2) if (${CMAKE_C_COMPILER_ID} STREQUAL "GNU") - if (CMAKE_C_COMPILER_VERSION} VERSION_GREATER 4.7 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 4.7) + if (CMAKE_C_COMPILER_VERSION VERSION_GREATER 4.7 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 4.7) set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -mavx2") endif() elseif (${CMAKE_C_COMPILER_ID} STREQUAL "CLANG") From 0050a9660bd0604a470a75f51870d6e470287a84 Mon Sep 17 00:00:00 2001 From: Matthias Langer <7855770+bashimao@users.noreply.github.com> Date: Thu, 16 May 2024 09:59:52 +0000 Subject: [PATCH 084/267] Correctly detect ARM Neoverse V2 CPUs. --- Makefile.arm64 | 10 ++++++++++ cpuid_arm64.c | 26 ++++++++++++++++++++++++-- kernel/arm64/KERNEL.NEOVERSEV2 | 1 + 3 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 kernel/arm64/KERNEL.NEOVERSEV2 diff --git a/Makefile.arm64 b/Makefile.arm64 index 4feb545239..6ba63e94e0 100644 --- a/Makefile.arm64 +++ b/Makefile.arm64 @@ -176,6 +176,16 @@ endif endif endif +# Detect ARM Neoverse V2. +ifeq ($(CORE), NEOVERSEV2) +ifeq (1, $(filter 1,$(GCCVERSIONGTEQ12) $(ISCLANG))) +CCOMMON_OPT += -march=armv9-a -mtune=neoverse-v2 +ifneq ($(F_COMPILER), NAG) +FCOMMON_OPT += -march=armv9-a -mtune=neoverse-v2 +endif +endif +endif + # Use a53 tunings because a55 is only available in GCC>=8.1 ifeq ($(CORE), CORTEXA55) ifeq (1, $(filter 1,$(GCCVERSIONGTEQ7) $(ISCLANG))) diff --git a/cpuid_arm64.c b/cpuid_arm64.c index b23edc4e7d..2cfa96ea60 100644 --- a/cpuid_arm64.c +++ b/cpuid_arm64.c @@ -46,6 +46,7 @@ size_t length64=sizeof(value64); #define CPU_NEOVERSEN1 11 #define CPU_NEOVERSEV1 16 #define CPU_NEOVERSEN2 17 +#define CPU_NEOVERSEV2 24 #define CPU_CORTEXX1 18 #define CPU_CORTEXX2 19 #define CPU_CORTEXA510 20 @@ -91,7 +92,8 @@ static char *cpuname[] = { "CORTEXA510", "CORTEXA710", "FT2000", - "CORTEXA76" + "CORTEXA76", + "NEOVERSEV2" }; static char *cpuname_lower[] = { @@ -118,7 +120,8 @@ static char *cpuname_lower[] = { "cortexa510", "cortexa710", "ft2000", - "cortexa76" + "cortexa76", + "neoversev2" }; int get_feature(char *search) @@ -213,6 +216,8 @@ int detect(void) return CPU_CORTEXX2; else if (strstr(cpu_part, "0xd4e")) //X3 return CPU_CORTEXX2; + else if (strstr(cpu_part, "0xd4f")) //NVIDIA Grace et al. + return CPU_NEOVERSEV2; else if (strstr(cpu_part, "0xd0b")) return CPU_CORTEXA76; } @@ -425,6 +430,23 @@ void get_cpuconfig(void) printf("#define DTB_DEFAULT_ENTRIES 48\n"); printf("#define DTB_SIZE 4096\n"); break; + case CPU_NEOVERSEV2: + printf("#define ARMV9\n"); + printf("#define %s\n", cpuname[d]); + printf("#define L1_CODE_SIZE 65536\n"); + printf("#define L1_CODE_LINESIZE 64\n"); + printf("#define L1_CODE_ASSOCIATIVE 4\n"); + printf("#define L1_DATA_SIZE 65536\n"); + printf("#define L1_DATA_LINESIZE 64\n"); + printf("#define L1_DATA_ASSOCIATIVE 4\n"); + printf("#define L2_SIZE 1048576\n"); + printf("#define L2_LINESIZE 64\n"); + printf("#define L2_ASSOCIATIVE 8\n"); + // L1 Data TLB = 48 entries + // L2 Data TLB = 2048 entries + printf("#define DTB_DEFAULT_ENTRIES 48\n"); + printf("#define DTB_SIZE 4096\n"); // Set to 4096 for symmetry with other configs. + break; case CPU_CORTEXA510: case CPU_CORTEXA710: case CPU_CORTEXX1: diff --git a/kernel/arm64/KERNEL.NEOVERSEV2 b/kernel/arm64/KERNEL.NEOVERSEV2 new file mode 100644 index 0000000000..bc59990979 --- /dev/null +++ b/kernel/arm64/KERNEL.NEOVERSEV2 @@ -0,0 +1 @@ +include $(KERNELDIR)/KERNEL.ARMV8SVE From 9f2816183701c931256c1bfd25978c39729382d0 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Mon, 6 May 2024 11:36:30 +0300 Subject: [PATCH 085/267] BENCH: add benchmarks using codspeed.io --- .github/workflows/codspeed-bench.yml | 150 ++++++++ .gitignore | 1 + benchmark/pybench/benchmarks/bench_blas.py | 185 ++++++++++ benchmark/pybench/meson.build | 48 +++ benchmark/pybench/openblas_wrap/__init__.py | 28 ++ .../pybench/openblas_wrap/blas_lapack.pyf.src | 326 ++++++++++++++++++ .../pybench/openblas_wrap/generate_f2pymod.py | 299 ++++++++++++++++ benchmark/pybench/openblas_wrap/meson.build | 50 +++ benchmark/pybench/scipy_openblas.pc | 12 + 9 files changed, 1099 insertions(+) create mode 100644 .github/workflows/codspeed-bench.yml create mode 100644 benchmark/pybench/benchmarks/bench_blas.py create mode 100644 benchmark/pybench/meson.build create mode 100644 benchmark/pybench/openblas_wrap/__init__.py create mode 100644 benchmark/pybench/openblas_wrap/blas_lapack.pyf.src create mode 100644 benchmark/pybench/openblas_wrap/generate_f2pymod.py create mode 100644 benchmark/pybench/openblas_wrap/meson.build create mode 100644 benchmark/pybench/scipy_openblas.pc diff --git a/.github/workflows/codspeed-bench.yml b/.github/workflows/codspeed-bench.yml new file mode 100644 index 0000000000..04befefa9e --- /dev/null +++ b/.github/workflows/codspeed-bench.yml @@ -0,0 +1,150 @@ +name: Run codspeed benchmarks + +on: [push, pull_request] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +jobs: + benchmarks: + if: "github.repository == 'OpenMathLib/OpenBLAS'" + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + fortran: [gfortran] + build: [make] + pyver: ["3.12"] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.pyver }} + + - name: Print system information + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + cat /proc/cpuinfo + fi + + - name: Install Dependencies + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + sudo apt-get update + sudo apt-get install -y gfortran cmake ccache libtinfo5 + else + echo "::error::$RUNNER_OS not supported" + exit 1 + fi + + - name: Compilation cache + uses: actions/cache@v3 + with: + path: ~/.ccache + # We include the commit sha in the cache key, as new cache entries are + # only created if there is no existing entry for the key yet. + # GNU make and cmake call the compilers differently. It looks like + # that causes the cache to mismatch. Keep the ccache for both build + # tools separate to avoid polluting each other. + key: ccache-${{ runner.os }}-${{ matrix.build }}-${{ matrix.fortran }}-${{ github.ref }}-${{ github.sha }} + # Restore a matching ccache cache entry. Prefer same branch and same Fortran compiler. + restore-keys: | + ccache-${{ runner.os }}-${{ matrix.build }}-${{ matrix.fortran }}-${{ github.ref }} + ccache-${{ runner.os }}-${{ matrix.build }}-${{ matrix.fortran }} + ccache-${{ runner.os }}-${{ matrix.build }} + + - name: Write out the .pc + run: | + cd benchmark/pybench + cat > openblas.pc << EOF + libdir=${{ github.workspace }} + includedir= ${{ github.workspace }} + openblas_config= OpenBLAS 0.3.27 DYNAMIC_ARCH NO_AFFINITY Haswell MAX_THREADS=64 + version=0.0.99 + extralib=-lm -lpthread -lgfortran -lquadmath -L${{ github.workspace }} -lopenblas + Name: openblas + Description: OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version + Version: ${version} + URL: https://github.com/xianyi/OpenBLAS + Libs: ${{ github.workspace }}/libopenblas.so -Wl,-rpath,${{ github.workspace }} + Libs.private: -lm -lpthread -lgfortran -lquadmath -L${{ github.workspace }} -lopenblas + Cflags: -I${{ github.workspace}} + EOF + cat openblas.pc + + - name: Configure ccache + run: | + if [ "${{ matrix.build }}" = "make" ]; then + # Add ccache to path + if [ "$RUNNER_OS" = "Linux" ]; then + echo "/usr/lib/ccache" >> $GITHUB_PATH + elif [ "$RUNNER_OS" = "macOS" ]; then + echo "$(brew --prefix)/opt/ccache/libexec" >> $GITHUB_PATH + else + echo "::error::$RUNNER_OS not supported" + exit 1 + fi + fi + # Limit the maximum size and switch on compression to avoid exceeding the total disk or cache quota (5 GB). + test -d ~/.ccache || mkdir -p ~/.ccache + echo "max_size = 300M" > ~/.ccache/ccache.conf + echo "compression = true" >> ~/.ccache/ccache.conf + ccache -s + + - name: Build OpenBLAS + run: | + case "${{ matrix.build }}" in + "make") + make -j$(nproc) DYNAMIC_ARCH=1 USE_OPENMP=0 FC="ccache ${{ matrix.fortran }}" + ;; + "cmake") + mkdir build && cd build + cmake -DDYNAMIC_ARCH=1 \ + -DNOFORTRAN=0 \ + -DBUILD_WITHOUT_LAPACK=0 \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_Fortran_COMPILER=${{ matrix.fortran }} \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_Fortran_COMPILER_LAUNCHER=ccache \ + .. + cmake --build . + ;; + *) + echo "::error::Configuration not supported" + exit 1 + ;; + esac + + - name: Show ccache status + continue-on-error: true + run: ccache -s + + - name: Install benchmark dependencies + run: pip install meson ninja numpy pytest pytest-codspeed --user + + - name: Build the wrapper + run: | + cd benchmark/pybench + export PKG_CONFIG_PATH=$PWD + meson setup build --prefix=$PWD/build-install + meson install -C build + # + # sanity check + cd build/openblas_wrap + python -c'import _flapack; print(dir(_flapack))' + + - name: Run benchmarks + uses: CodSpeedHQ/action@v2 + with: + token: ${{ secrets.CODSPEED_TOKEN }} + run: | + cd benchmark/pybench + export PYTHONPATH=$PWD/build-install/lib/python${{matrix.pyver}}/site-packages/ + OPENBLAS_NUM_THREADS=1 pytest benchmarks/bench_blas.py --codspeed + diff --git a/.gitignore b/.gitignore index dc6804f1ef..8294da4d44 100644 --- a/.gitignore +++ b/.gitignore @@ -109,3 +109,4 @@ benchmark/smallscaling CMakeCache.txt CMakeFiles/* .vscode +**/__pycache__ diff --git a/benchmark/pybench/benchmarks/bench_blas.py b/benchmark/pybench/benchmarks/bench_blas.py new file mode 100644 index 0000000000..064be1ead9 --- /dev/null +++ b/benchmark/pybench/benchmarks/bench_blas.py @@ -0,0 +1,185 @@ +import pytest +import numpy as np +from openblas_wrap import ( + # level 1 + dnrm2, ddot, daxpy, + # level 3 + dgemm, dsyrk, + # lapack + dgesv, # linalg.solve + dgesdd, dgesdd_lwork, # linalg.svd + dsyev, dsyev_lwork, # linalg.eigh +) + +# ### BLAS level 1 ### + +# dnrm2 + +dnrm2_sizes = [100, 1000] + +def run_dnrm2(n, x, incx): + res = dnrm2(x, n, incx=incx) + return res + + +@pytest.mark.parametrize('n', dnrm2_sizes) +def test_nrm2(benchmark, n): + rndm = np.random.RandomState(1234) + x = np.array(rndm.uniform(size=(n,)), dtype=float) + result = benchmark(run_dnrm2, n, x, 1) + + +# ddot + +ddot_sizes = [100, 1000] + +def run_ddot(x, y,): + res = ddot(x, y) + return res + + +@pytest.mark.parametrize('n', ddot_sizes) +def test_dot(benchmark, n): + rndm = np.random.RandomState(1234) + x = np.array(rndm.uniform(size=(n,)), dtype=float) + y = np.array(rndm.uniform(size=(n,)), dtype=float) + result = benchmark(run_ddot, x, y) + + +# daxpy + +daxpy_sizes = [100, 1000] + +def run_daxpy(x, y,): + res = daxpy(x, y, a=2.0) + return res + + +@pytest.mark.parametrize('n', daxpy_sizes) +def test_daxpy(benchmark, n): + rndm = np.random.RandomState(1234) + x = np.array(rndm.uniform(size=(n,)), dtype=float) + y = np.array(rndm.uniform(size=(n,)), dtype=float) + result = benchmark(run_daxpy, x, y) + + + + +# ### BLAS level 3 ### + +# dgemm + +gemm_sizes = [100, 1000] + +def run_gemm(a, b, c): + alpha = 1.0 + res = dgemm(alpha, a, b, c=c, overwrite_c=True) + return res + + +@pytest.mark.parametrize('n', gemm_sizes) +def test_gemm(benchmark, n): + rndm = np.random.RandomState(1234) + a = np.array(rndm.uniform(size=(n, n)), dtype=float, order='F') + b = np.array(rndm.uniform(size=(n, n)), dtype=float, order='F') + c = np.empty((n, n), dtype=float, order='F') + result = benchmark(run_gemm, a, b, c) + assert result is c + + +# dsyrk + +syrk_sizes = [100, 1000] + + +def run_syrk(a, c): + res = dsyrk(1.0, a, c=c, overwrite_c=True) + return res + + +@pytest.mark.parametrize('n', syrk_sizes) +def test_syrk(benchmark, n): + rndm = np.random.RandomState(1234) + a = np.array(rndm.uniform(size=(n, n)), dtype=float, order='F') + c = np.empty((n, n), dtype=float, order='F') + result = benchmark(run_syrk, a, c) + assert result is c + + +# ### LAPACK ### + +# linalg.solve + +gesv_sizes = [100, 1000] + + +def run_gesv(a, b): + res = dgesv(a, b, overwrite_a=True, overwrite_b=True) + return res + + +@pytest.mark.parametrize('n', gesv_sizes) +def test_gesv(benchmark, n): + rndm = np.random.RandomState(1234) + a = (np.array(rndm.uniform(size=(n, n)), dtype=float, order='F') + + np.eye(n, order='F')) + b = np.array(rndm.uniform(size=(n, 1)), order='F') + lu, piv, x, info = benchmark(run_gesv, a, b) + assert lu is a + assert x is b + assert info == 0 + + +# linalg.svd + +gesdd_sizes = [(100, 5), (1000, 222)] + + +def run_gesdd(a, lwork): + res = dgesdd(a, lwork=lwork, full_matrices=False, overwrite_a=False) + return res + + +@pytest.mark.parametrize('mn', gesdd_sizes) +def test_gesdd(benchmark, mn): + m, n = mn + rndm = np.random.RandomState(1234) + a = np.array(rndm.uniform(size=(m, n)), dtype=float, order='F') + + lwork, info = dgesdd_lwork(m, n) + lwork = int(lwork) + assert info == 0 + + u, s, vt, info = benchmark(run_gesdd, a, lwork) + + assert info == 0 + np.testing.assert_allclose(u @ np.diag(s) @ vt, a, atol=1e-13) + + +# linalg.eigh + +syev_sizes = [50, 200] + + +def run_syev(a, lwork): + res = dsyev(a, lwork=lwork, overwrite_a=True) + return res + + +@pytest.mark.parametrize('n', syev_sizes) +def test_syev(benchmark, n): + rndm = np.random.RandomState(1234) + a = rndm.uniform(size=(n, n)) + a = np.asarray(a + a.T, dtype=float, order='F') + a_ = a.copy() + + lwork, info = dsyev_lwork(n) + lwork = int(lwork) + assert info == 0 + + w, v, info = benchmark(run_syev, a, lwork) + + assert info == 0 + assert a is v # overwrite_a=True + + diff --git a/benchmark/pybench/meson.build b/benchmark/pybench/meson.build new file mode 100644 index 0000000000..5d921c9ed5 --- /dev/null +++ b/benchmark/pybench/meson.build @@ -0,0 +1,48 @@ +# +# Taken from SciPy (of course) +# +project( + 'openblas-wrap', + 'c', 'fortran', + version: '0.1', + license: 'BSD-3', + meson_version: '>= 1.1.0', + default_options: [ + 'buildtype=debugoptimized', + 'b_ndebug=if-release', + 'c_std=c17', + 'fortran_std=legacy', + ], +) + +py3 = import('python').find_installation(pure: false) +py3_dep = py3.dependency() + +cc = meson.get_compiler('c') + +_global_c_args = cc.get_supported_arguments( + '-Wno-unused-but-set-variable', + '-Wno-unused-function', + '-Wno-conversion', + '-Wno-misleading-indentation', +) +add_project_arguments(_global_c_args, language : 'c') + +# We need -lm for all C code (assuming it uses math functions, which is safe to +# assume for SciPy). For C++ it isn't needed, because libstdc++/libc++ is +# guaranteed to depend on it. For Fortran code, Meson already adds `-lm`. +m_dep = cc.find_library('m', required : false) +if m_dep.found() + add_project_link_arguments('-lm', language : 'c') +endif + +generate_f2pymod = find_program('openblas_wrap/generate_f2pymod.py') + +openblas = dependency('openblas', method: 'pkg-config', required: true) +openblas_dep = declare_dependency( + dependencies: openblas, + compile_args: [] +) + + +subdir('openblas_wrap') diff --git a/benchmark/pybench/openblas_wrap/__init__.py b/benchmark/pybench/openblas_wrap/__init__.py new file mode 100644 index 0000000000..06e16a665a --- /dev/null +++ b/benchmark/pybench/openblas_wrap/__init__.py @@ -0,0 +1,28 @@ +""" +Trampoline to hide the LAPACK details (scipy.lapack.linalg or scipy_openblas32 or...) +from benchmarking. +""" + +__version__ = "0.1" + + +#from scipy.linalg.blas import ( +from ._flapack import ( + # level 1 + dnrm2 as dnrm2, + ddot as ddot, + daxpy as daxpy, + # level 3 + dgemm as dgemm, + dsyrk as dsyrk, +) + +#from scipy.linalg.lapack import ( +from openblas_wrap._flapack import ( + # linalg.solve + dgesv as dgesv, + # linalg.svd + dgesdd as dgesdd, dgesdd_lwork as dgesdd_lwork, + # linalg.eigh + dsyev as dsyev, dsyev_lwork as dsyev_lwork +) diff --git a/benchmark/pybench/openblas_wrap/blas_lapack.pyf.src b/benchmark/pybench/openblas_wrap/blas_lapack.pyf.src new file mode 100644 index 0000000000..d2d94baa0e --- /dev/null +++ b/benchmark/pybench/openblas_wrap/blas_lapack.pyf.src @@ -0,0 +1,326 @@ +! +! Taken from scipy/linalg +! +! Shorthand notations +! +! +! +! +! +! +! +! +! +! +! +! +! +! +! +! +! +! +! +! +! +! +! +! +! +! +! +! +! +! Level 1 BLAS +! + + +python module _flapack + usercode ''' +#define F_INT int +''' + +interface + + +subroutine axpy(n,a,x,offx,incx,y,offy,incy) + ! Calculate z = a*x+y, where a is scalar. + + callstatement (*f2py_func)(&n,&a,x+offx,&incx,y+offy,&incy) + callprotoargument F_INT*,*,*,F_INT*,*,F_INT* + + dimension(*), intent(in) :: x + dimension(*), intent(in,out,out=z) :: y + optional, intent(in):: a=<1.0,\0,(1.0\,0.0),\2> + integer optional, intent(in),check(incx>0||incx<0) :: incx = 1 + integer optional, intent(in),check(incy>0||incy<0) :: incy = 1 + integer optional, intent(in),depend(x) :: offx=0 + integer optional, intent(in),depend(y) :: offy=0 + check(offx>=0 && offx=0 && offy(n-1)*abs(incx)) :: n + check(len(y)-offy>(n-1)*abs(incy)) :: n + +end subroutine axpy + +function ddot(n,x,offx,incx,y,offy,incy) result (xy) + ! Computes a vector-vector dot product. + + callstatement ddot_return_value = (*f2py_func)(&n,x+offx,&incx,y+offy,&incy) + callprotoargument F_INT*,double*,F_INT*,double*,F_INT* + intent(c) ddot + fortranname F_FUNC(ddot,DDOT) + + double precision dimension(*), intent(in) :: x + double precision dimension(*), intent(in) :: y + double precision ddot,xy + integer optional, intent(in),check(incx>0||incx<0) :: incx = 1 + integer optional, intent(in),check(incy>0||incy<0) :: incy = 1 + integer optional, intent(in),depend(x) :: offx=0 + integer optional, intent(in),depend(y) :: offy=0 + check(offx>=0 && offx=0 && offy(n-1)*abs(incx)) :: n + check(len(y)-offy>(n-1)*abs(incy)) :: n + +end function ddot + + +function nrm2(n,x,offx,incx) result(n2) + + nrm2, n2 + + callstatement nrm2_return_value = (*f2py_func)(&n,x+offx,&incx) + callprotoargument F_INT*,*,F_INT* + intent(c) nrm2 + fortranname F_FUNC(nrm2,NRM2) + + dimension(*),intent(in) :: x + + integer optional, intent(in),check(incx>0) :: incx = 1 + + integer optional,intent(in),depend(x) :: offx=0 + check(offx>=0 && offx(n-1)*abs(incx)) :: n + +end function nrm2 + +! +! Level 3 BLAS +! + + +subroutine gemm(m,n,k,alpha,a,b,beta,c,trans_a,trans_b,lda,ka,ldb,kb) + ! Computes a scalar-matrix-matrix product and adds the result to a + ! scalar-matrix product. + ! + ! c = gemm(alpha,a,b,beta=0,c=0,trans_a=0,trans_b=0,overwrite_c=0) + ! Calculate C <- alpha * op(A) * op(B) + beta * C + + callstatement (*f2py_func)((trans_a?(trans_a==2?"C":"T"):"N"), & + (trans_b?(trans_b==2?"C":"T"):"N"),&m,&n,&k,&alpha,a,&lda,b,&ldb,&beta,c,&m) + callprotoargument char*,char*,F_INT*,F_INT*,F_INT*,*,*,F_INT*,*, & + F_INT*,*,*,F_INT* + + integer optional,intent(in),check(trans_a>=0 && trans_a <=2) :: trans_a = 0 + integer optional,intent(in),check(trans_b>=0 && trans_b <=2) :: trans_b = 0 + intent(in) :: alpha + intent(in),optional :: beta = <0.0,\0,(0.0\,0.0),\2> + + dimension(lda,ka),intent(in) :: a + dimension(ldb,kb),intent(in) :: b + dimension(m,n),intent(in,out,copy),depend(m,n),optional :: c + check(shape(c,0)==m && shape(c,1)==n) :: c + + integer depend(a),intent(hide) :: lda = shape(a,0) + integer depend(a),intent(hide) :: ka = shape(a,1) + integer depend(b),intent(hide) :: ldb = shape(b,0) + integer depend(b),intent(hide) :: kb = shape(b,1) + + integer depend(a,trans_a,ka,lda),intent(hide):: m = (trans_a?ka:lda) + integer depend(a,trans_a,ka,lda),intent(hide):: k = (trans_a?lda:ka) + integer depend(b,trans_b,kb,ldb,k),intent(hide),check(trans_b?kb==k:ldb==k) :: & + n = (trans_b?ldb:kb) + +end subroutine gemm + + +subroutine rk(n,k,alpha,a,beta,c,trans,lower,lda,ka) + ! performs one of the symmetric rank k operations + ! C := alpha*A*A**T + beta*C, or C := alpha*A**T*A + beta*C, + ! + ! c = syrk(alpha,a,beta=0,c=0,trans=0,lower=0,overwrite_c=0) + ! + callstatement (*f2py_func)((lower?"L":"U"), & + (trans?(trans==2?"C":"T"):"N"), &n,&k,&alpha,a,&lda,&beta,c,&n) + callprotoargument char*,char*,F_INT*,F_INT*,*,*,F_INT*,*, & + *,F_INT* + + integer optional, intent(in),check(lower==0||lower==1) :: lower = 0 + integer optional,intent(in),check(trans>=0 && trans <=2) :: trans = 0 + + intent(in) :: alpha + intent(in),optional :: beta = <0.0,\0,(0.0\,0.0),\2,\2,\2> + + dimension(lda,ka),intent(in) :: a + dimension(n,n),intent(in,out,copy),depend(n),optional :: c + check(shape(c,0)==n && shape(c,1)==n) :: c + + integer depend(a),intent(hide) :: lda = shape(a,0) + integer depend(a),intent(hide) :: ka = shape(a,1) + + integer depend(a, trans, ka, lda), intent(hide) :: n = (trans ? ka : lda) + integer depend(a, trans, ka, lda), intent(hide) :: k = (trans ? lda : ka) + +end subroutine rk + + +! +! LAPACK +! + +subroutine gesv(n,nrhs,a,piv,b,info) + ! lu,piv,x,info = gesv(a,b,overwrite_a=0,overwrite_b=0) + ! Solve A * X = B. + ! A = P * L * U + ! U is upper diagonal triangular, L is unit lower triangular, + ! piv pivots columns. + + callstatement {F_INT i;(*f2py_func)(&n,&nrhs,a,&n,piv,b,&n,&info);for(i=0;i\*,F_INT*,F_INT*,*,F_INT*,F_INT* + + integer depend(a),intent(hide):: n = shape(a,0) + integer depend(b),intent(hide):: nrhs = shape(b,1) + dimension(n,n),check(shape(a,0)==shape(a,1)) :: a + integer dimension(n),depend(n),intent(out) :: piv + dimension(n,nrhs),check(shape(a,0)==shape(b,0)),depend(n) :: b + integer intent(out)::info + intent(in,out,copy,out=x) b + intent(in,out,copy,out=lu) a +end subroutine gesv + + +subroutine gesdd(m,n,minmn,u0,u1,vt0,vt1,a,compute_uv,full_matrices,u,s,vt,work,lwork,iwork,info) + ! u,s,vt,info = gesdd(a,compute_uv=1,lwork=..,overwrite_a=0) + ! Compute the singular value decomposition (SVD) using divide and conquer: + ! A = U * SIGMA * transpose(V) + ! A - M x N matrix + ! U - M x M matrix or min(M,N) x N if full_matrices=False + ! SIGMA - M x N zero matrix with a main diagonal filled with min(M,N) + ! singular values + ! transpose(V) - N x N matrix or N x min(M,N) if full_matrices=False + + callstatement (*f2py_func)((compute_uv?(full_matrices?"A":"S"):"N"),&m,&n,a,&m,s,u,&u0,vt,&vt0,work,&lwork,iwork,&info) + callprotoargument char*,F_INT*,F_INT*,*,F_INT*,*,*,F_INT*,*,F_INT*,*,F_INT*,F_INT*,F_INT* + + integer intent(in),optional,check(compute_uv==0||compute_uv==1):: compute_uv = 1 + integer intent(in),optional,check(full_matrices==0||full_matrices==1):: full_matrices = 1 + integer intent(hide),depend(a):: m = shape(a,0) + integer intent(hide),depend(a):: n = shape(a,1) + integer intent(hide),depend(m,n):: minmn = MIN(m,n) + integer intent(hide),depend(compute_uv,minmn) :: u0 = (compute_uv?m:1) + integer intent(hide),depend(compute_uv,minmn, full_matrices) :: u1 = (compute_uv?(full_matrices?m:minmn):1) + integer intent(hide),depend(compute_uv,minmn, full_matrices) :: vt0 = (compute_uv?(full_matrices?n:minmn):1) + integer intent(hide),depend(compute_uv,minmn) :: vt1 = (compute_uv?n:1) + dimension(m,n),intent(in,copy,aligned8) :: a + dimension(minmn),intent(out),depend(minmn) :: s + dimension(u0,u1),intent(out),depend(u0, u1) :: u + dimension(vt0,vt1),intent(out),depend(vt0, vt1) :: vt + dimension(lwork),intent(hide,cache),depend(lwork) :: work + integer optional,intent(in),depend(minmn,compute_uv) & + :: lwork = max((compute_uv?4*minmn*minmn+MAX(m,n)+9*minmn:MAX(14*minmn+4,10*minmn+2+25*(25+8))+MAX(m,n)),1) + integer intent(hide,cache),dimension(8*minmn),depend(minmn) :: iwork + integer intent(out)::info + +end subroutine gesdd + +subroutine gesdd_lwork(m,n,minmn,u0,vt0,a,compute_uv,full_matrices,u,s,vt,work,lwork,iwork,info) + ! LWORK computation for (S/D)GESDD + + fortranname gesdd + callstatement (*f2py_func)((compute_uv?(full_matrices?"A":"S"):"N"),&m,&n,&a,&m,&s,&u,&u0,&vt,&vt0,&work,&lwork,&iwork,&info) + callprotoargument char*,F_INT*,F_INT*,*,F_INT*,*,*,F_INT*,*,F_INT*,*,F_INT*,F_INT*,F_INT* + + integer intent(in),optional,check(compute_uv==0||compute_uv==1):: compute_uv = 1 + integer intent(in),optional,check(full_matrices==0||full_matrices==1):: full_matrices = 1 + integer intent(in) :: m + integer intent(in) :: n + integer intent(hide),depend(m,n):: minmn = MIN(m,n) + integer intent(hide),depend(compute_uv,minmn) :: u0 = (compute_uv?m:1) + integer intent(hide),depend(compute_uv,minmn, full_matrices) :: vt0 = (compute_uv?(full_matrices?n:minmn):1) + intent(hide) :: a + intent(hide) :: s + intent(hide) :: u + intent(hide) :: vt + intent(out) :: work + integer intent(hide) :: lwork = -1 + integer intent(hide) :: iwork + integer intent(out) :: info + +end subroutine gesdd_lwork + + +subroutine syev(compute_v,lower,n,w,a,lda,work,lwork,info) + ! w,v,info = syev(a,compute_v=1,lower=0,lwork=3*n-1,overwrite_a=0) + ! Compute all eigenvalues and, optionally, eigenvectors of a + ! real symmetric matrix A. + ! + ! Performance tip: + ! If compute_v=0 then set also overwrite_a=1. + + callstatement (*f2py_func)((compute_v?"V":"N"),(lower?"L":"U"),&n,a,&lda,w,work,&lwork,&info) + callprotoargument char*,char*,F_INT*,*,F_INT*,*,*,F_INT*,F_INT* + + integer optional,intent(in):: compute_v = 1 + check(compute_v==1||compute_v==0) compute_v + integer optional,intent(in),check(lower==0||lower==1) :: lower = 0 + + integer intent(hide),depend(a):: n = shape(a,0) + integer intent(hide),depend(a):: lda = MAX(1,shape(a,0)) + dimension(n,n),check(shape(a,0)==shape(a,1)) :: a + intent(in,copy,out,out=v) :: a + + dimension(n),intent(out),depend(n) :: w + + integer optional,intent(in),depend(n) :: lwork=max(3*n-1,1) + check(lwork>=3*n-1) :: lwork + dimension(lwork),intent(hide),depend(lwork) :: work + + integer intent(out) :: info + +end subroutine syev + + +subroutine syev_lwork(lower,n,w,a,lda,work,lwork,info) + ! LWORK routines for syev + + fortranname syev + + callstatement (*f2py_func)("N",(lower?"L":"U"),&n,&a,&lda,&w,&work,&lwork,&info) + callprotoargument char*,char*,F_INT*,*,F_INT*,*,*,F_INT*,F_INT* + + integer intent(in):: n + integer optional,intent(in),check(lower==0||lower==1) :: lower = 0 + + integer intent(hide),depend(n):: lda = MAX(1, n) + intent(hide):: a + intent(hide):: w + integer intent(hide):: lwork = -1 + + intent(out):: work + integer intent(out):: info + +end subroutine syev_lwork + +end interface + +end python module _flapack + + + diff --git a/benchmark/pybench/openblas_wrap/generate_f2pymod.py b/benchmark/pybench/openblas_wrap/generate_f2pymod.py new file mode 100644 index 0000000000..5a8ba13895 --- /dev/null +++ b/benchmark/pybench/openblas_wrap/generate_f2pymod.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +""" +Process f2py template files (`filename.pyf.src` -> `filename.pyf`) + +Usage: python generate_pyf.py filename.pyf.src -o filename.pyf +""" + +import os +import sys +import re +import subprocess +import argparse + + +# START OF CODE VENDORED FROM `numpy.distutils.from_template` +############################################################# +""" +process_file(filename) + + takes templated file .xxx.src and produces .xxx file where .xxx + is .pyf .f90 or .f using the following template rules: + + '<..>' denotes a template. + + All function and subroutine blocks in a source file with names that + contain '<..>' will be replicated according to the rules in '<..>'. + + The number of comma-separated words in '<..>' will determine the number of + replicates. + + '<..>' may have two different forms, named and short. For example, + + named: + where anywhere inside a block '

' will be replaced with + 'd', 's', 'z', and 'c' for each replicate of the block. + + <_c> is already defined: <_c=s,d,c,z> + <_t> is already defined: <_t=real,double precision,complex,double complex> + + short: + , a short form of the named, useful when no

appears inside + a block. + + In general, '<..>' contains a comma separated list of arbitrary + expressions. If these expression must contain a comma|leftarrow|rightarrow, + then prepend the comma|leftarrow|rightarrow with a backslash. + + If an expression matches '\\' then it will be replaced + by -th expression. + + Note that all '<..>' forms in a block must have the same number of + comma-separated entries. + + Predefined named template rules: + + + + + +""" + +routine_start_re = re.compile( + r'(\n|\A)(( (\$|\*))|)\s*(subroutine|function)\b', + re.I +) +routine_end_re = re.compile(r'\n\s*end\s*(subroutine|function)\b.*(\n|\Z)', re.I) +function_start_re = re.compile(r'\n (\$|\*)\s*function\b', re.I) + +def parse_structure(astr): + """ Return a list of tuples for each function or subroutine each + tuple is the start and end of a subroutine or function to be + expanded. + """ + + spanlist = [] + ind = 0 + while True: + m = routine_start_re.search(astr, ind) + if m is None: + break + start = m.start() + if function_start_re.match(astr, start, m.end()): + while True: + i = astr.rfind('\n', ind, start) + if i==-1: + break + start = i + if astr[i:i+7]!='\n $': + break + start += 1 + m = routine_end_re.search(astr, m.end()) + ind = end = m and m.end()-1 or len(astr) + spanlist.append((start, end)) + return spanlist + +template_re = re.compile(r"<\s*(\w[\w\d]*)\s*>") +named_re = re.compile(r"<\s*(\w[\w\d]*)\s*=\s*(.*?)\s*>") +list_re = re.compile(r"<\s*((.*?))\s*>") + +def find_repl_patterns(astr): + reps = named_re.findall(astr) + names = {} + for rep in reps: + name = rep[0].strip() or unique_key(names) + repl = rep[1].replace(r'\,', '@comma@') + thelist = conv(repl) + names[name] = thelist + return names + +def find_and_remove_repl_patterns(astr): + names = find_repl_patterns(astr) + astr = re.subn(named_re, '', astr)[0] + return astr, names + +item_re = re.compile(r"\A\\(?P\d+)\Z") +def conv(astr): + b = astr.split(',') + l = [x.strip() for x in b] + for i in range(len(l)): + m = item_re.match(l[i]) + if m: + j = int(m.group('index')) + l[i] = l[j] + return ','.join(l) + +def unique_key(adict): + """ Obtain a unique key given a dictionary.""" + allkeys = list(adict.keys()) + done = False + n = 1 + while not done: + newkey = '__l%s' % (n) + if newkey in allkeys: + n += 1 + else: + done = True + return newkey + + +template_name_re = re.compile(r'\A\s*(\w[\w\d]*)\s*\Z') +def expand_sub(substr, names): + substr = substr.replace(r'\>', '@rightarrow@') + substr = substr.replace(r'\<', '@leftarrow@') + lnames = find_repl_patterns(substr) + substr = named_re.sub(r"<\1>", substr) # get rid of definition templates + + def listrepl(mobj): + thelist = conv(mobj.group(1).replace(r'\,', '@comma@')) + if template_name_re.match(thelist): + return "<%s>" % (thelist) + name = None + for key in lnames.keys(): # see if list is already in dictionary + if lnames[key] == thelist: + name = key + if name is None: # this list is not in the dictionary yet + name = unique_key(lnames) + lnames[name] = thelist + return "<%s>" % name + + substr = list_re.sub(listrepl, substr) # convert all lists to named templates + # newnames are constructed as needed + + numsubs = None + base_rule = None + rules = {} + for r in template_re.findall(substr): + if r not in rules: + thelist = lnames.get(r, names.get(r, None)) + if thelist is None: + raise ValueError('No replicates found for <%s>' % (r)) + if r not in names and not thelist.startswith('_'): + names[r] = thelist + rule = [i.replace('@comma@', ',') for i in thelist.split(',')] + num = len(rule) + + if numsubs is None: + numsubs = num + rules[r] = rule + base_rule = r + elif num == numsubs: + rules[r] = rule + else: + print("Mismatch in number of replacements (base <{}={}>) " + "for <{}={}>. Ignoring." + .format(base_rule, ','.join(rules[base_rule]), r, thelist)) + if not rules: + return substr + + def namerepl(mobj): + name = mobj.group(1) + return rules.get(name, (k+1)*[name])[k] + + newstr = '' + for k in range(numsubs): + newstr += template_re.sub(namerepl, substr) + '\n\n' + + newstr = newstr.replace('@rightarrow@', '>') + newstr = newstr.replace('@leftarrow@', '<') + return newstr + +def process_str(allstr): + newstr = allstr + writestr = '' + + struct = parse_structure(newstr) + + oldend = 0 + names = {} + names.update(_special_names) + for sub in struct: + cleanedstr, defs = find_and_remove_repl_patterns(newstr[oldend:sub[0]]) + writestr += cleanedstr + names.update(defs) + writestr += expand_sub(newstr[sub[0]:sub[1]], names) + oldend = sub[1] + writestr += newstr[oldend:] + + return writestr + +include_src_re = re.compile( + r"(\n|\A)\s*include\s*['\"](?P[\w\d./\\]+\.src)['\"]", + re.I +) + +def resolve_includes(source): + d = os.path.dirname(source) + with open(source) as fid: + lines = [] + for line in fid: + m = include_src_re.match(line) + if m: + fn = m.group('name') + if not os.path.isabs(fn): + fn = os.path.join(d, fn) + if os.path.isfile(fn): + lines.extend(resolve_includes(fn)) + else: + lines.append(line) + else: + lines.append(line) + return lines + +def process_file(source): + lines = resolve_includes(source) + return process_str(''.join(lines)) + +_special_names = find_repl_patterns(''' +<_c=s,d,c,z> +<_t=real,double precision,complex,double complex> + + + + + +''') + +# END OF CODE VENDORED FROM `numpy.distutils.from_template` +########################################################### + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("infile", type=str, + help="Path to the input file") + parser.add_argument("-o", "--outdir", type=str, + help="Path to the output directory") + args = parser.parse_args() + + if not args.infile.endswith(('.pyf', '.pyf.src', '.f.src')): + raise ValueError(f"Input file has unknown extension: {args.infile}") + + outdir_abs = os.path.join(os.getcwd(), args.outdir) + + # Write out the .pyf/.f file + if args.infile.endswith(('.pyf.src', '.f.src')): + code = process_file(args.infile) + fname_pyf = os.path.join(args.outdir, + os.path.splitext(os.path.split(args.infile)[1])[0]) + + with open(fname_pyf, 'w') as f: + f.write(code) + else: + fname_pyf = args.infile + + # Now invoke f2py to generate the C API module file + if args.infile.endswith(('.pyf.src', '.pyf')): + p = subprocess.Popen([sys.executable, '-m', 'numpy.f2py', fname_pyf, + '--build-dir', outdir_abs], #'--quiet'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + cwd=os.getcwd()) + out, err = p.communicate() + if not (p.returncode == 0): + raise RuntimeError(f"Writing {args.outfile} with f2py failed!\n" + f"{out}\n" + r"{err}") + + +if __name__ == "__main__": + main() diff --git a/benchmark/pybench/openblas_wrap/meson.build b/benchmark/pybench/openblas_wrap/meson.build new file mode 100644 index 0000000000..9f1b717876 --- /dev/null +++ b/benchmark/pybench/openblas_wrap/meson.build @@ -0,0 +1,50 @@ +# find numpy & f2py includes +inc_numpy = run_command(py3, + ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], + check : true +).stdout().strip() + +inc_f2py = run_command(py3, + ['-c', 'import os; os.chdir(".."); import numpy.f2py; print(numpy.f2py.get_include())'], + check : true +).stdout().strip() + + +inc_np = include_directories(inc_numpy, inc_f2py) +fortranobject_c = inc_f2py / 'fortranobject.c' + + +fortranobject_lib = static_library('_fortranobject', + fortranobject_c, +# c_args: numpy_nodepr_api, + dependencies: py3_dep, + include_directories: [inc_np, inc_f2py], + gnu_symbol_visibility: 'hidden', +) +fortranobject_dep = declare_dependency( + link_with: fortranobject_lib, + include_directories: [inc_np, inc_f2py], +) + + +# f2py generated wrappers + +flapack_module = custom_target('flapack_module', + output: ['_flapackmodule.c'], + input: 'blas_lapack.pyf.src', + command: [generate_f2pymod, '@INPUT@', '-o', '@OUTDIR@'], +) + +py3.extension_module('_flapack', + flapack_module, + link_args: [], # version_link_args, + dependencies: [openblas_dep, fortranobject_dep], + install: true, + subdir: 'openblas_wrap' +) + + +py3.install_sources( + ['__init__.py'], + subdir: 'openblas_wrap' +) diff --git a/benchmark/pybench/scipy_openblas.pc b/benchmark/pybench/scipy_openblas.pc new file mode 100644 index 0000000000..2348fac626 --- /dev/null +++ b/benchmark/pybench/scipy_openblas.pc @@ -0,0 +1,12 @@ +libdir=/home/br/repos/OpenBLAS/ +includedir=/home/br/repos/OpenBLAS/ +openblas_config= OpenBLAS 0.3.27 DYNAMIC_ARCH NO_AFFINITY Haswell MAX_THREADS=64 +version=0.3.27 +extralib=-lm -lpthread -lgfortran -lquadmath -L${libdir} -lopenblas +Name: openblas +Description: OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version +Version: ${version} +URL: https://github.com/xianyi/OpenBLAS +Libs: -L${libdir} -lopenblas +Libs.private: ${extralib} +Cflags: -I${includedir} From 81cf0db047552e00f60e0267fd62f1ffd2a0c560 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Sat, 18 May 2024 15:30:00 +0300 Subject: [PATCH 086/267] DOC: add a readme for benchmarks/pybench --- benchmark/pybench/README.md | 49 +++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 benchmark/pybench/README.md diff --git a/benchmark/pybench/README.md b/benchmark/pybench/README.md new file mode 100644 index 0000000000..7523ca75ab --- /dev/null +++ b/benchmark/pybench/README.md @@ -0,0 +1,49 @@ +# Continuous benchmarking of OpenBLAS performance + +We run a set of benchmarks of subset of OpenBLAS functionality. + +## Benchmark runner + +[![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/OpenMathLib/OpenBLAS/) + +Click on [benchmarks](https://codspeed.io/OpenMathLib/OpenBLAS/benchmarks) to see the performance of a particular benchmark over time; +Click on [branches](https://codspeed.io/OpenMathLib/OpenBLAS/branches/) and then on the last PR link to see the flamegraphs. + +## What are the benchmarks + +We run raw BLAS/LAPACK subroutines, via f2py-generated python wrappers. The wrappers themselves are equivalent to [those from SciPy](https://docs.scipy.org/doc/scipy/reference/linalg.lapack.html). +In fact, the wrappers _are_ from SciPy, we take a small subset simply to avoid having to build the whole SciPy for each CI run. + + +## Adding a new benchmark + +`.github/workflows/codspeed-bench.yml` does all the orchestration on CI. + +Benchmarks live in the `benchmark/pybench` directory. It is organized as follows: + +- benchmarks themselves live in the `benchmarks` folder. Note that the LAPACK routines are imported from the `openblas_wrap` package. +- the `openblas_wrap` package is a simple trampoline: it contains an f2py extension, `_flapack`, which talks to OpenBLAS, and exports the python names in its `__init__.py`. +This way, the `openblas_wrap` package shields the benchmarks from the details of where a particular LAPACK function comes from. If wanted, you may for instance swap the `_flapack` extension to +`scipy.linalg.blas` and `scipy.linalg.lapack`. + +To change parameters of an existing benchmark, edit python files in the `benchmark/pybench/benchmarks` directory. + +To add a benchmark for a new BLAS or LAPACK function, you need to: + +- add an f2py wrapper for the bare LAPACK function. You can simply copy a wrapper from SciPy (look for `*.pyf.src` files in https://github.com/scipy/scipy/tree/main/scipy/linalg) +- add an import to `benchmark/pybench/openblas_wrap/__init__.py` + + +## Running benchmarks locally + +This benchmarking layer is orchestrated from python, therefore you'll need to +have all what it takes to build OpenBLAS from source, plus `python` and + +``` +$ python -mpip install numpy meson ninja pytest pytest-benchmark +``` + +The benchmark syntax is consistent with that of `pytest-benchmark` framework. The incantation to run the suite locally is `$ pytest benchmark/pybench/benchmarks/test_blas.py`. + +An ASV compatible benchmark suite is planned but currently not implemented. + From 6494f432df77565519590cff746c9333329dcd24 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 18 May 2024 16:49:03 +0200 Subject: [PATCH 087/267] Fix INTERFACE64 builds on Loongarch64 --- cmake/fc.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmake/fc.cmake b/cmake/fc.cmake index 176d3d4388..69246385c4 100644 --- a/cmake/fc.cmake +++ b/cmake/fc.cmake @@ -67,6 +67,9 @@ if (${F_COMPILER} STREQUAL "GFORTRAN" OR ${F_COMPILER} STREQUAL "F95" OR CMAKE_F else() set(FCOMMON_OPT "${FCOMMON_OPT} -mabi=lp64") endif () + if (INTERFACE64) + set(FCOMMON_OPT "${FCOMMON_OPT} -fdefault-integer-8") + endif () else () CHECK_C_COMPILER_FLAG("-mabi=ilp32d" COMPILER_SUPPORT_ILP32D_ABI) if(COMPILER_SUPPORT_ILP32D_ABI) From 172d91846f241e4f603eced526ab1da06c399ea8 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Mon, 20 May 2024 22:53:43 +0200 Subject: [PATCH 088/267] Don't try to deploy docs in a fork --- .github/workflows/docs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 359281d4e8..203097812f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,6 +6,7 @@ on: jobs: build: name: Deploy docs + if: "github.repository == 'OpenMathLib/OpenBLAS'" runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 From e112191b549afa98062acc292ac62e1b766ae53d Mon Sep 17 00:00:00 2001 From: Rajalakshmi Srinivasaraghavan Date: Wed, 22 May 2024 08:00:06 -0500 Subject: [PATCH 089/267] POWER: Fix issues in zscal to address lapack failures This patch fixes following lapack failures with clang compiler on POWER. zed.out: ZVX: 18 out of 5190 tests failed to pass the threshold zgd.out: ZGV drivers: 25 out of 1092 tests failed to pass the threshold zgd.out: ZGV drivers: 6 out of 1092 tests failed to pass the threshold --- kernel/power/zscal.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/power/zscal.c b/kernel/power/zscal.c index 0068138e88..6b7392d0ca 100644 --- a/kernel/power/zscal.c +++ b/kernel/power/zscal.c @@ -38,6 +38,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma GCC optimize "O1" +#if defined(__clang__) +#pragma clang fp contract(off) +#endif + #if defined(__VEC__) || defined(__ALTIVEC__) #if defined(POWER8) || defined(POWER9) #if defined(DOUBLE) From e2c1a1e26959b772db419383746a32836e25600e Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 23 May 2024 23:08:27 +0200 Subject: [PATCH 090/267] Fix potential bounds error (Reference-LAPACK PR 1018) --- lapack-netlib/SRC/cunhr_col.f | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lapack-netlib/SRC/cunhr_col.f b/lapack-netlib/SRC/cunhr_col.f index faa37d3ed3..b86bcd5059 100644 --- a/lapack-netlib/SRC/cunhr_col.f +++ b/lapack-netlib/SRC/cunhr_col.f @@ -420,7 +420,7 @@ SUBROUTINE CUNHR_COL( M, N, NB, A, LDA, T, LDT, D, INFO ) * JBTEMP2 = JB - 2 DO J = JB, JB+JNB-2 - DO I = J-JBTEMP2, NB + DO I = J-JBTEMP2, MIN( NB, N ) T( I, J ) = CZERO END DO END DO @@ -436,4 +436,4 @@ SUBROUTINE CUNHR_COL( M, N, NB, A, LDA, T, LDT, D, INFO ) * * End of CUNHR_COL * - END \ No newline at end of file + END From 4dd715d2207aaff765f623c41387b58f5ff9795e Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 23 May 2024 23:09:55 +0200 Subject: [PATCH 091/267] Fix potential bounds error (Reference-LAPACK PR 1018) --- lapack-netlib/SRC/dorhr_col.f | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lapack-netlib/SRC/dorhr_col.f b/lapack-netlib/SRC/dorhr_col.f index a9363c1a5a..064f9cb01d 100644 --- a/lapack-netlib/SRC/dorhr_col.f +++ b/lapack-netlib/SRC/dorhr_col.f @@ -419,7 +419,7 @@ SUBROUTINE DORHR_COL( M, N, NB, A, LDA, T, LDT, D, INFO ) * JBTEMP2 = JB - 2 DO J = JB, JB+JNB-2 - DO I = J-JBTEMP2, NB + DO I = J-JBTEMP2, MIN( NB, N ) T( I, J ) = ZERO END DO END DO @@ -435,4 +435,4 @@ SUBROUTINE DORHR_COL( M, N, NB, A, LDA, T, LDT, D, INFO ) * * End of DORHR_COL * - END \ No newline at end of file + END From 791e015024824a02e5cdc94c020d8c590a97d65c Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 23 May 2024 23:11:14 +0200 Subject: [PATCH 092/267] Fix potential bounds error (Reference-LAPACK PR 1018) --- lapack-netlib/SRC/sorhr_col.f | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lapack-netlib/SRC/sorhr_col.f b/lapack-netlib/SRC/sorhr_col.f index 5d91881482..ab6df2bc8d 100644 --- a/lapack-netlib/SRC/sorhr_col.f +++ b/lapack-netlib/SRC/sorhr_col.f @@ -419,7 +419,7 @@ SUBROUTINE SORHR_COL( M, N, NB, A, LDA, T, LDT, D, INFO ) * JBTEMP2 = JB - 2 DO J = JB, JB+JNB-2 - DO I = J-JBTEMP2, NB + DO I = J-JBTEMP2, MIN( NB, N ) T( I, J ) = ZERO END DO END DO From 7ebbe3cc72f48482379ed1f9892fc557275165a2 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 23 May 2024 23:12:19 +0200 Subject: [PATCH 093/267] Fix potential bounds error (Reference-LAPACK PR 1018) --- lapack-netlib/SRC/zunhr_col.f | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lapack-netlib/SRC/zunhr_col.f b/lapack-netlib/SRC/zunhr_col.f index 3c7b2deb18..ce5c09e068 100644 --- a/lapack-netlib/SRC/zunhr_col.f +++ b/lapack-netlib/SRC/zunhr_col.f @@ -420,7 +420,7 @@ SUBROUTINE ZUNHR_COL( M, N, NB, A, LDA, T, LDT, D, INFO ) * JBTEMP2 = JB - 2 DO J = JB, JB+JNB-2 - DO I = J-JBTEMP2, NB + DO I = J-JBTEMP2, MIN( NB, N ) T( I, J ) = CZERO END DO END DO @@ -436,4 +436,4 @@ SUBROUTINE ZUNHR_COL( M, N, NB, A, LDA, T, LDT, D, INFO ) * * End of ZUNHR_COL * - END \ No newline at end of file + END From 62f7b244ff5c268f311e5f080d1f44c3b432f3d0 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Fri, 24 May 2024 17:20:27 +0000 Subject: [PATCH 094/267] Replace use of FLT_MAX in x86_64 zscal.c by isinf() Commit def4996 fixed issues with inf and nan values in zscal, but used FLT_MAX, where DBL_MAX or isinf() is more appropriate, as FLT_MAX is for single precision only. Using FLT_MAX caused test case failures in the LAPACK tests. isinf() is consistent with the later fix 969601a1 --- kernel/x86_64/zscal.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/x86_64/zscal.c b/kernel/x86_64/zscal.c index bc79c0cafd..075b6091fe 100644 --- a/kernel/x86_64/zscal.c +++ b/kernel/x86_64/zscal.c @@ -394,7 +394,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i, } } - else if (da_r < -FLT_MAX || da_r > FLT_MAX) { + else if (isinf(da_r)) { while(j < n) { x[i]= NAN; @@ -410,7 +410,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i, while(j < n) { temp0 = -da_i * x[i+1]; - if (x[i] < -FLT_MAX || x[i] > FLT_MAX) + if (isinf(x[i])) temp0 = NAN; x[i+1] = da_i * x[i]; if ( x[i] == x[i]) //preserve NaN From 616cc28d82da8981b853cccf365439905f8efdba Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sun, 26 May 2024 12:59:11 +0200 Subject: [PATCH 095/267] Override icx's default fast math mode to ensure correct NaN handling --- Makefile.x86_64 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile.x86_64 b/Makefile.x86_64 index c0dbe84c8b..e9831327fe 100644 --- a/Makefile.x86_64 +++ b/Makefile.x86_64 @@ -8,6 +8,11 @@ endif endif endif +ifeq ($(C_COMPILER), CLANG) +ifeq ($(findstring icx,$(CC)),icx) +CCOMMON_OPT += -fp-model=consistent +endif +endif ifneq ($(DYNAMIC_ARCH),1) ADD_CPUFLAGS = 1 From 8b4996a2d59f554d5aafe14332aa0582c203a6d6 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sun, 26 May 2024 13:16:03 +0200 Subject: [PATCH 096/267] Override icx's default fast math mode to ensure correct NaN handling --- cmake/cc.cmake | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmake/cc.cmake b/cmake/cc.cmake index a162986f85..9dce9ea35a 100644 --- a/cmake/cc.cmake +++ b/cmake/cc.cmake @@ -4,6 +4,10 @@ ## Sets C related variables. include(CheckCCompilerFlag) +if (${CMAKE_C_COMPILER_ID} MATCHES "IntelLLVM") + set(CCOMMON_OPT "${CCOMMON_OPT} -fp-model=consistent") +endif () + if (${CMAKE_C_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_C_COMPILER_ID} STREQUAL "LSB" OR ${CMAKE_C_COMPILER_ID} MATCHES "Clang") set(CCOMMON_OPT "${CCOMMON_OPT} -Wall") set(COMMON_PROF "${COMMON_PROF} -fno-inline") From 103637887e35062aae8e795805c13ac9dfb6ca31 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 29 May 2024 15:46:10 +0200 Subject: [PATCH 097/267] add cblas_?gemm_batch --- cblas.h | 15 +++++++++++++++ common_level3.h | 7 ++++++- common_macro.h | 11 +++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/cblas.h b/cblas.h index ab8b7dcde4..097b4303d6 100644 --- a/cblas.h +++ b/cblas.h @@ -416,6 +416,18 @@ void cblas_cgeadd(OPENBLAS_CONST enum CBLAS_ORDER CORDER,OPENBLAS_CONST blasint void cblas_zgeadd(OPENBLAS_CONST enum CBLAS_ORDER CORDER,OPENBLAS_CONST blasint crows, OPENBLAS_CONST blasint ccols, OPENBLAS_CONST double *calpha, double *a, OPENBLAS_CONST blasint clda, OPENBLAS_CONST double *cbeta, double *c, OPENBLAS_CONST blasint cldc); +void cblas_sgemm_batch(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE * TransA_array, OPENBLAS_CONST enum CBLAS_TRANSPOSE * TransB_array, OPENBLAS_CONST blasint * M_array, OPENBLAS_CONST blasint * N_array, OPENBLAS_CONST blasint * K_array, + OPENBLAS_CONST float * alpha_array, OPENBLAS_CONST float ** A_array, OPENBLAS_CONST blasint * lda_array, OPENBLAS_CONST float ** B_array, OPENBLAS_CONST blasint * ldb_array, OPENBLAS_CONST float * beta_array, float ** C_array, OPENBLAS_CONST blasint * ldc_array, OPENBLAS_CONST blasint group_count, OPENBLAS_CONST blasint * group_size); + +void cblas_dgemm_batch(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE * TransA_array, OPENBLAS_CONST enum CBLAS_TRANSPOSE * TransB_array, OPENBLAS_CONST blasint * M_array, OPENBLAS_CONST blasint * N_array, OPENBLAS_CONST blasint * K_array, + OPENBLAS_CONST double * alpha_array, OPENBLAS_CONST double ** A_array, OPENBLAS_CONST blasint * lda_array, OPENBLAS_CONST double ** B_array, OPENBLAS_CONST blasint * ldb_array, OPENBLAS_CONST double * beta_array, double ** C_array, OPENBLAS_CONST blasint * ldc_array, OPENBLAS_CONST blasint group_count, OPENBLAS_CONST blasint * group_size); + +void cblas_cgemm_batch(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE * TransA_array, OPENBLAS_CONST enum CBLAS_TRANSPOSE * TransB_array, OPENBLAS_CONST blasint * M_array, OPENBLAS_CONST blasint * N_array, OPENBLAS_CONST blasint * K_array, + OPENBLAS_CONST void * alpha_array, OPENBLAS_CONST void ** A_array, OPENBLAS_CONST blasint * lda_array, OPENBLAS_CONST void ** B_array, OPENBLAS_CONST blasint * ldb_array, OPENBLAS_CONST void * beta_array, void ** C_array, OPENBLAS_CONST blasint * ldc_array, OPENBLAS_CONST blasint group_count, OPENBLAS_CONST blasint * group_size); + +void cblas_zgemm_batch(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE * TransA_array, OPENBLAS_CONST enum CBLAS_TRANSPOSE * TransB_array, OPENBLAS_CONST blasint * M_array, OPENBLAS_CONST blasint * N_array, OPENBLAS_CONST blasint * K_array, + OPENBLAS_CONST void * alpha_array, OPENBLAS_CONST void ** A_array, OPENBLAS_CONST blasint * lda_array, OPENBLAS_CONST void ** B_array, OPENBLAS_CONST blasint * ldb_array, OPENBLAS_CONST void * beta_array, void ** C_array, OPENBLAS_CONST blasint * ldc_array, OPENBLAS_CONST blasint group_count, OPENBLAS_CONST blasint * group_size); + /*** BFLOAT16 and INT8 extensions ***/ /* convert float array to BFLOAT16 array by rounding */ void cblas_sbstobf16(OPENBLAS_CONST blasint n, OPENBLAS_CONST float *in, OPENBLAS_CONST blasint incin, bfloat16 *out, OPENBLAS_CONST blasint incout); @@ -431,6 +443,9 @@ void cblas_sbgemv(OPENBLAS_CONST enum CBLAS_ORDER order, OPENBLAS_CONST enum void cblas_sbgemm(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K, OPENBLAS_CONST float alpha, OPENBLAS_CONST bfloat16 *A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST bfloat16 *B, OPENBLAS_CONST blasint ldb, OPENBLAS_CONST float beta, float *C, OPENBLAS_CONST blasint ldc); +void cblas_sbgemm_batch(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE * TransA_array, OPENBLAS_CONST enum CBLAS_TRANSPOSE * TransB_array, OPENBLAS_CONST blasint * M_array, OPENBLAS_CONST blasint * N_array, OPENBLAS_CONST blasint * K_array, + OPENBLAS_CONST float * alpha_array, OPENBLAS_CONST bfloat16 ** A_array, OPENBLAS_CONST blasint * lda_array, OPENBLAS_CONST bfloat16 ** B_array, OPENBLAS_CONST blasint * ldb_array, OPENBLAS_CONST float * beta_array, float ** C_array, OPENBLAS_CONST blasint * ldc_array, OPENBLAS_CONST blasint group_count, OPENBLAS_CONST blasint * group_size); + #ifdef __cplusplus } #endif /* __cplusplus */ diff --git a/common_level3.h b/common_level3.h index 5080ada10d..d370c1f96a 100644 --- a/common_level3.h +++ b/common_level3.h @@ -1937,8 +1937,13 @@ int zimatcopy_k_rtc(BLASLONG, BLASLONG, double, double, double *, BLASLONG); int sgeadd_k(BLASLONG, BLASLONG, float, float*, BLASLONG, float, float *, BLASLONG); int dgeadd_k(BLASLONG, BLASLONG, double, double*, BLASLONG, double, double *, BLASLONG); int cgeadd_k(BLASLONG, BLASLONG, float, float, float*, BLASLONG, float, float, float *, BLASLONG); -int zgeadd_k(BLASLONG, BLASLONG, double,double, double*, BLASLONG, double, double, double *, BLASLONG); +int zgeadd_k(BLASLONG, BLASLONG, double,double, double*, BLASLONG, double, double, double *, BLASLONG); +int sgemm_batch_thread(blas_arg_t * queue, BLASLONG nums); +int dgemm_batch_thread(blas_arg_t * queue, BLASLONG nums); +int cgemm_batch_thread(blas_arg_t * queue, BLASLONG nums); +int zgemm_batch_thread(blas_arg_t * queue, BLASLONG nums); +int sbgemm_batch_thread(blas_arg_t * queue, BLASLONG nums); #ifdef __CUDACC__ } diff --git a/common_macro.h b/common_macro.h index 3226d0f11b..a924651de2 100644 --- a/common_macro.h +++ b/common_macro.h @@ -2655,9 +2655,20 @@ typedef struct { BLASLONG prea, preb, prec, pred; #endif + + //for gemm_batch + void * routine; + int routine_mode; + } blas_arg_t; #endif +#ifdef SMALL_MATRIX_OPT +#define BLAS_SMALL_OPT 0x10000U +#define BLAS_SMALL_B0_OPT 0x30000U +#endif + + #ifdef XDOUBLE #define TRSV_NUU qtrsv_NUU From 89c7bbcba65cc4dc639bf5030910b307d2861df8 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 29 May 2024 15:47:02 +0200 Subject: [PATCH 098/267] add cblas_?gemm_batch --- interface/Makefile | 24 ++- interface/gemm_batch.c | 372 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 391 insertions(+), 5 deletions(-) create mode 100644 interface/gemm_batch.c diff --git a/interface/Makefile b/interface/Makefile index 048d679d6b..97439d87f8 100644 --- a/interface/Makefile +++ b/interface/Makefile @@ -282,12 +282,12 @@ CSBLAS2OBJS = \ CSBLAS3OBJS = \ cblas_sgemm.$(SUFFIX) cblas_ssymm.$(SUFFIX) cblas_strmm.$(SUFFIX) cblas_strsm.$(SUFFIX) \ cblas_ssyrk.$(SUFFIX) cblas_ssyr2k.$(SUFFIX) cblas_somatcopy.$(SUFFIX) cblas_simatcopy.$(SUFFIX)\ - cblas_sgeadd.$(SUFFIX) cblas_sgemmt.$(SUFFIX) + cblas_sgeadd.$(SUFFIX) cblas_sgemmt.$(SUFFIX) cblas_sgemm_batch.$(SUFFIX) ifeq ($(BUILD_BFLOAT16),1) CSBBLAS1OBJS = cblas_sbdot.$(SUFFIX) CSBBLAS2OBJS = cblas_sbgemv.$(SUFFIX) -CSBBLAS3OBJS = cblas_sbgemm.$(SUFFIX) cblas_sbgemmt.$(SUFFIX) +CSBBLAS3OBJS = cblas_sbgemm.$(SUFFIX) cblas_sbgemmt.$(SUFFIX) cblas_sbgemm_batch.$(SUFFIX) CSBEXTOBJS = cblas_sbstobf16.$(SUFFIX) cblas_sbdtobf16.$(SUFFIX) cblas_sbf16tos.$(SUFFIX) cblas_dbf16tod.$(SUFFIX) endif @@ -308,7 +308,7 @@ CDBLAS2OBJS = \ CDBLAS3OBJS += \ cblas_dgemm.$(SUFFIX) cblas_dsymm.$(SUFFIX) cblas_dtrmm.$(SUFFIX) cblas_dtrsm.$(SUFFIX) \ cblas_dsyrk.$(SUFFIX) cblas_dsyr2k.$(SUFFIX) cblas_domatcopy.$(SUFFIX) cblas_dimatcopy.$(SUFFIX) \ - cblas_dgeadd.$(SUFFIX) cblas_dgemmt.$(SUFFIX) + cblas_dgeadd.$(SUFFIX) cblas_dgemmt.$(SUFFIX) cblas_dgemm_batch.$(SUFFIX) CCBLAS1OBJS = \ cblas_icamax.$(SUFFIX) cblas_icamin.$(SUFFIX) cblas_scasum.$(SUFFIX) cblas_caxpy.$(SUFFIX) \ @@ -333,7 +333,7 @@ CCBLAS3OBJS = \ cblas_csyrk.$(SUFFIX) cblas_csyr2k.$(SUFFIX) \ cblas_chemm.$(SUFFIX) cblas_cherk.$(SUFFIX) cblas_cher2k.$(SUFFIX) \ cblas_comatcopy.$(SUFFIX) cblas_cimatcopy.$(SUFFIX)\ - cblas_cgeadd.$(SUFFIX) cblas_cgemmt.$(SUFFIX) + cblas_cgeadd.$(SUFFIX) cblas_cgemmt.$(SUFFIX) cblas_cgemm_batch.$(SUFFIX) CXERBLAOBJ = \ cblas_xerbla.$(SUFFIX) @@ -364,7 +364,7 @@ CZBLAS3OBJS = \ cblas_zsyrk.$(SUFFIX) cblas_zsyr2k.$(SUFFIX) \ cblas_zhemm.$(SUFFIX) cblas_zherk.$(SUFFIX) cblas_zher2k.$(SUFFIX)\ cblas_zomatcopy.$(SUFFIX) cblas_zimatcopy.$(SUFFIX) \ - cblas_zgeadd.$(SUFFIX) cblas_zgemmt.$(SUFFIX) + cblas_zgeadd.$(SUFFIX) cblas_zgemmt.$(SUFFIX) cblas_zgemm_batch.$(SUFFIX) ifeq ($(SUPPORT_GEMM3M), 1) @@ -2419,3 +2419,17 @@ cblas_zgeadd.$(SUFFIX) cblas_zgeadd.$(PSUFFIX) : zgeadd.c cblas_xerbla.$(SUFFIX) cblas_xerbla.$(PSUFFIX) : xerbla.c $(CC) -c $(CFLAGS) -DCBLAS $< -o $(@F) +cblas_sbgemm_batch.$(SUFFIX) cblas_sbgemm_batch.$(PSUFFIX) : gemm_batch.c ../param.h + $(CC) -c $(CFLAGS) -DCBLAS $< -o $(@F) + +cblas_sgemm_batch.$(SUFFIX) cblas_sgemm_batch.$(PSUFFIX) : gemm_batch.c ../param.h + $(CC) -c $(CFLAGS) -DCBLAS $< -o $(@F) + +cblas_dgemm_batch.$(SUFFIX) cblas_dgemm_batch.$(PSUFFIX) : gemm_batch.c ../param.h + $(CC) -c $(CFLAGS) -DCBLAS $< -o $(@F) + +cblas_cgemm_batch.$(SUFFIX) cblas_cgemm_batch.$(PSUFFIX) : gemm_batch.c ../param.h + $(CC) -c $(CFLAGS) -DCBLAS $< -o $(@F) + +cblas_zgemm_batch.$(SUFFIX) cblas_zgemm_batch.$(PSUFFIX) : gemm_batch.c ../param.h + $(CC) -c $(CFLAGS) -DCBLAS $< -o $(@F) diff --git a/interface/gemm_batch.c b/interface/gemm_batch.c new file mode 100644 index 0000000000..d674f9615a --- /dev/null +++ b/interface/gemm_batch.c @@ -0,0 +1,372 @@ +/***************************************************************************** +Copyright (c) 2020, The OpenBLAS Project +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. Neither the name of the OpenBLAS project nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +**********************************************************************************/ + +#include +#include +#include "common.h" + +void openblas_warning(int verbose, const char * msg); + +#ifndef COMPLEX +#ifdef XDOUBLE +#define ERROR_NAME "QGEMM_BATCH " +#elif defined(DOUBLE) +#define ERROR_NAME "DGEMM_BATCH " +#define GEMM_BATCH_THREAD dgemm_batch_thread +#else +#define ERROR_NAME "SGEMM_BATCH " +#define GEMM_BATCH_THREAD sgemm_batch_thread +#endif +#else +#ifdef XDOUBLE +#define ERROR_NAME "XGEMM_BATCH " +#elif defined(DOUBLE) +#define ERROR_NAME "ZGEMM_BATCH " +#define GEMM_BATCH_THREAD zgemm_batch_thread +#else +#define ERROR_NAME "CGEMM_BATCH " +#define GEMM_BATCH_THREAD cgemm_batch_thread +#endif +#endif +static int (*gemm[])(blas_arg_t *, BLASLONG *, BLASLONG *, IFLOAT *, IFLOAT *, BLASLONG) = { + GEMM_NN, GEMM_TN, GEMM_RN, GEMM_CN, + GEMM_NT, GEMM_TT, GEMM_RT, GEMM_CT, + GEMM_NR, GEMM_TR, GEMM_RR, GEMM_CR, + GEMM_NC, GEMM_TC, GEMM_RC, GEMM_CC, +}; + +#if defined(SMALL_MATRIX_OPT) && !defined(GEMM3M) && !defined(XDOUBLE) +#define USE_SMALL_MATRIX_OPT 1 +#else +#define USE_SMALL_MATRIX_OPT 0 +#endif + +#if USE_SMALL_MATRIX_OPT +#ifndef DYNAMIC_ARCH +#define SMALL_KERNEL_ADDR(table, idx) ((void *)(table[idx])) +#else +#define SMALL_KERNEL_ADDR(table, idx) ((void *)(*(uintptr_t *)((char *)gotoblas + (size_t)(table[idx])))) +#endif + + +#ifndef COMPLEX +static size_t gemm_small_kernel[] = { + GEMM_SMALL_KERNEL_NN, GEMM_SMALL_KERNEL_TN, 0, 0, + GEMM_SMALL_KERNEL_NT, GEMM_SMALL_KERNEL_TT, 0, 0, +}; + + +static size_t gemm_small_kernel_b0[] = { + GEMM_SMALL_KERNEL_B0_NN, GEMM_SMALL_KERNEL_B0_TN, 0, 0, + GEMM_SMALL_KERNEL_B0_NT, GEMM_SMALL_KERNEL_B0_TT, 0, 0, +}; + +#define GEMM_SMALL_KERNEL_B0(idx) (int (*)(BLASLONG, BLASLONG, BLASLONG, IFLOAT *, BLASLONG, FLOAT, IFLOAT *, BLASLONG, FLOAT *, BLASLONG)) SMALL_KERNEL_ADDR(gemm_small_kernel_b0, (idx)) +#define GEMM_SMALL_KERNEL(idx) (int (*)(BLASLONG, BLASLONG, BLASLONG, IFLOAT *, BLASLONG, FLOAT, IFLOAT *, BLASLONG, FLOAT, FLOAT *, BLASLONG)) SMALL_KERNEL_ADDR(gemm_small_kernel, (idx)) +#else + +static size_t zgemm_small_kernel[] = { + GEMM_SMALL_KERNEL_NN, GEMM_SMALL_KERNEL_TN, GEMM_SMALL_KERNEL_RN, GEMM_SMALL_KERNEL_CN, + GEMM_SMALL_KERNEL_NT, GEMM_SMALL_KERNEL_TT, GEMM_SMALL_KERNEL_RT, GEMM_SMALL_KERNEL_CT, + GEMM_SMALL_KERNEL_NR, GEMM_SMALL_KERNEL_TR, GEMM_SMALL_KERNEL_RR, GEMM_SMALL_KERNEL_CR, + GEMM_SMALL_KERNEL_NC, GEMM_SMALL_KERNEL_TC, GEMM_SMALL_KERNEL_RC, GEMM_SMALL_KERNEL_CC, +}; + +static size_t zgemm_small_kernel_b0[] = { + GEMM_SMALL_KERNEL_B0_NN, GEMM_SMALL_KERNEL_B0_TN, GEMM_SMALL_KERNEL_B0_RN, GEMM_SMALL_KERNEL_B0_CN, + GEMM_SMALL_KERNEL_B0_NT, GEMM_SMALL_KERNEL_B0_TT, GEMM_SMALL_KERNEL_B0_RT, GEMM_SMALL_KERNEL_B0_CT, + GEMM_SMALL_KERNEL_B0_NR, GEMM_SMALL_KERNEL_B0_TR, GEMM_SMALL_KERNEL_B0_RR, GEMM_SMALL_KERNEL_B0_CR, + GEMM_SMALL_KERNEL_B0_NC, GEMM_SMALL_KERNEL_B0_TC, GEMM_SMALL_KERNEL_B0_RC, GEMM_SMALL_KERNEL_B0_CC, +}; + +#define ZGEMM_SMALL_KERNEL(idx) (int (*)(BLASLONG, BLASLONG, BLASLONG, FLOAT *, BLASLONG, FLOAT , FLOAT, FLOAT *, BLASLONG, FLOAT , FLOAT, FLOAT *, BLASLONG)) SMALL_KERNEL_ADDR(zgemm_small_kernel, (idx)) +#define ZGEMM_SMALL_KERNEL_B0(idx) (int (*)(BLASLONG, BLASLONG, BLASLONG, FLOAT *, BLASLONG, FLOAT , FLOAT, FLOAT *, BLASLONG, FLOAT *, BLASLONG)) SMALL_KERNEL_ADDR(zgemm_small_kernel_b0, (idx)) +#endif +#endif + +void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE * transa_array, enum CBLAS_TRANSPOSE * transb_array, + blasint * m_array, blasint * n_array, blasint * k_array, +#ifndef COMPLEX + FLOAT * alpha_array, + FLOAT ** a_array, blasint * lda_array, + FLOAT ** b_array, blasint * ldb_array, + FLOAT * beta_array, + FLOAT ** c_array, blasint * ldc_array, blasint group_count, blasint * group_size) { +#else + void * valpha_array, + void ** va_array, blasint * lda_array, + void ** vb_array, blasint * ldb_array, + void * vbeta_array, + void ** vc_array, blasint * ldc_array, blasint group_count, blasint * group_size) { + + FLOAT * alpha_array=(FLOAT *)valpha_array; + FLOAT * beta_array=(FLOAT *)vbeta_array; + FLOAT ** a_array=(FLOAT**)va_array; + FLOAT ** b_array=(FLOAT**)vb_array; + FLOAT ** c_array=(FLOAT**)vc_array; + +#endif + blas_arg_t * args_array=NULL; + + int mode=0, group_mode=0; + blasint total_num=0; + + blasint i=0, j=0, matrix_idx=0, count=0; + + int group_transa, group_transb; + BLASLONG group_nrowa, group_nrowb; + blasint info; + + void * group_alpha, * group_beta; + BLASLONG group_m, group_n, group_k; + BLASLONG group_lda, group_ldb, group_ldc; + void * group_routine=NULL; +#ifdef SMALL_MATRIX_OPT + void * group_small_matrix_opt_routine=NULL; +#endif + +#if defined (SMP) || defined(SMALL_MATRIX_OPT) + double MNK; +#endif + + PRINT_DEBUG_CNAME; + + for(i=0; i= 0) { + BLASFUNC(xerbla)(ERROR_NAME, &info, sizeof(ERROR_NAME)); + free(args_array); + return; + } + + if (group_m == 0 || group_n == 0) continue; + + group_mode=mode; + +#if defined(SMP) || defined(SMALL_MATRIX_OPT) + MNK = (double) group_m * (double) group_n * (double) group_k; +#endif + +#ifdef SMALL_MATRIX_OPT + if (MNK <= 100.0*100.0*100.0){ + group_routine=NULL; +#if !defined(COMPLEX) + if(*(FLOAT *)(group_beta) == 0.0){ + group_mode=mode | BLAS_SMALL_B0_OPT; + group_small_matrix_opt_routine=(void *)(gemm_small_kernel_b0[(group_transb<<2)|group_transa]); + }else{ + group_mode=mode | BLAS_SMALL_OPT; + group_small_matrix_opt_routine=(void *)(gemm_small_kernel[(group_transb<<2)|group_transa]); + } +#else + if(((FLOAT *)(group_beta))[0] == 0.0 && ((FLOAT *)(group_beta))[1] == 0.0){ + group_mode=mode | BLAS_SMALL_B0_OPT; + group_small_matrix_opt_routine=(void *)(zgemm_small_kernel_b0[(group_transb<<2)|group_transa]); + }else{ + group_mode=mode | BLAS_SMALL_OPT; + group_small_matrix_opt_routine=(void *)(zgemm_small_kernel[(group_transb<<2)|group_transa]); + } + +#endif + + }else{ +#endif + group_routine=(void*)(gemm[(group_transb<<2)|group_transa]); +#ifdef SMALL_MATRIX_OPT + } +#endif + + + for(j=0; j0){ + GEMM_BATCH_THREAD(args_array,count); + } + + free(args_array); +} From 833a8880c608cad1d6e0b515da41611612be71e9 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 29 May 2024 15:47:50 +0200 Subject: [PATCH 099/267] add cblas_?gemm_batch --- exports/gensymbol | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/exports/gensymbol b/exports/gensymbol index 2260358423..28dd883f2b 100755 --- a/exports/gensymbol +++ b/exports/gensymbol @@ -60,7 +60,7 @@ cblasobjsc=" cblas_ctbsv cblas_ctpmv cblas_ctpsv cblas_ctrmm cblas_ctrmv cblas_ctrsm cblas_ctrsv cblas_scnrm2 cblas_scasum cblas_cgemmt cblas_icamax cblas_icamin cblas_icmin cblas_icmax cblas_scsum cblas_cimatcopy cblas_comatcopy - cblas_caxpyc cblas_crotg cblas_csrot cblas_scamax cblas_scamin + cblas_caxpyc cblas_crotg cblas_csrot cblas_scamax cblas_scamin cblas_cgemm_batch " cblasobjsd=" cblas_dasum cblas_daxpy cblas_dcopy cblas_ddot @@ -70,7 +70,7 @@ cblasobjsd=" cblas_dsyr2k cblas_dsyr cblas_dsyrk cblas_dtbmv cblas_dtbsv cblas_dtpmv cblas_dtpsv cblas_dtrmm cblas_dtrmv cblas_dtrsm cblas_dtrsv cblas_daxpby cblas_dgeadd cblas_dgemmt cblas_idamax cblas_idamin cblas_idmin cblas_idmax cblas_dsum cblas_dimatcopy cblas_domatcopy - cblas_damax cblas_damin + cblas_damax cblas_damin cblas_dgemm_batch " cblasobjss=" @@ -82,7 +82,7 @@ cblasobjss=" cblas_stbmv cblas_stbsv cblas_stpmv cblas_stpsv cblas_strmm cblas_strmv cblas_strsm cblas_strsv cblas_sgeadd cblas_sgemmt cblas_isamax cblas_isamin cblas_ismin cblas_ismax cblas_ssum cblas_simatcopy cblas_somatcopy - cblas_samax cblas_samin + cblas_samax cblas_samin cblas_sgemm_batch " cblasobjsz=" @@ -94,12 +94,12 @@ cblasobjsz=" cblas_ztrsv cblas_cdotc_sub cblas_cdotu_sub cblas_zdotc_sub cblas_zdotu_sub cblas_zaxpby cblas_zgeadd cblas_zgemmt cblas_izamax cblas_izamin cblas_izmin cblas_izmax cblas_dzsum cblas_zimatcopy cblas_zomatcopy - cblas_zaxpyc cblas_zdrot cblas_zrotg cblas_dzamax cblas_dzamin + cblas_zaxpyc cblas_zdrot cblas_zrotg cblas_dzamax cblas_dzamin cblas_zgemm_batch " cblasobjs="cblas_xerbla" -bfcblasobjs="cblas_sbgemm cblas_sbgemv cblas_sbdot cblas_sbstobf16 cblas_sbdtobf16 cblas_sbf16tos cblas_dbf16tod" +bfcblasobjs="cblas_sbgemm cblas_sbgemv cblas_sbdot cblas_sbstobf16 cblas_sbdtobf16 cblas_sbf16tos cblas_dbf16tod cblas_sbgemm_batch" exblasobjs=" qamax qamin qasum qaxpy qcabs1 qcopy qdot qgbmv qgemm From d0794f88dcb83fd1a3a9fd317124e9ec984b6255 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 29 May 2024 15:49:20 +0200 Subject: [PATCH 100/267] add gemm_batch driver --- driver/level3/Makefile | 23 ++++- driver/level3/gemm_batch_thread.c | 156 ++++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+), 4 deletions(-) create mode 100644 driver/level3/gemm_batch_thread.c diff --git a/driver/level3/Makefile b/driver/level3/Makefile index b8465d4ed6..c304838423 100644 --- a/driver/level3/Makefile +++ b/driver/level3/Makefile @@ -37,7 +37,7 @@ SBLASOBJS += \ ssyrk_UN.$(SUFFIX) ssyrk_UT.$(SUFFIX) ssyrk_LN.$(SUFFIX) ssyrk_LT.$(SUFFIX) \ ssyr2k_UN.$(SUFFIX) ssyr2k_UT.$(SUFFIX) ssyr2k_LN.$(SUFFIX) ssyr2k_LT.$(SUFFIX) \ ssyrk_kernel_U.$(SUFFIX) ssyrk_kernel_L.$(SUFFIX) \ - ssyr2k_kernel_U.$(SUFFIX) ssyr2k_kernel_L.$(SUFFIX) + ssyr2k_kernel_U.$(SUFFIX) ssyr2k_kernel_L.$(SUFFIX) sgemm_batch_thread.$(SUFFIX) DBLASOBJS += \ dgemm_nn.$(SUFFIX) dgemm_nt.$(SUFFIX) dgemm_tn.$(SUFFIX) dgemm_tt.$(SUFFIX) \ @@ -53,7 +53,7 @@ DBLASOBJS += \ dsyrk_UN.$(SUFFIX) dsyrk_UT.$(SUFFIX) dsyrk_LN.$(SUFFIX) dsyrk_LT.$(SUFFIX) \ dsyr2k_UN.$(SUFFIX) dsyr2k_UT.$(SUFFIX) dsyr2k_LN.$(SUFFIX) dsyr2k_LT.$(SUFFIX) \ dsyrk_kernel_U.$(SUFFIX) dsyrk_kernel_L.$(SUFFIX) \ - dsyr2k_kernel_U.$(SUFFIX) dsyr2k_kernel_L.$(SUFFIX) + dsyr2k_kernel_U.$(SUFFIX) dsyr2k_kernel_L.$(SUFFIX) dgemm_batch_thread.$(SUFFIX) QBLASOBJS += \ qgemm_nn.$(SUFFIX) qgemm_nt.$(SUFFIX) qgemm_tn.$(SUFFIX) qgemm_tt.$(SUFFIX) \ @@ -103,7 +103,7 @@ CBLASOBJS += \ cherk_kernel_LN.$(SUFFIX) cherk_kernel_LC.$(SUFFIX) \ csyr2k_kernel_U.$(SUFFIX) csyr2k_kernel_L.$(SUFFIX) \ cher2k_kernel_UN.$(SUFFIX) cher2k_kernel_UC.$(SUFFIX) \ - cher2k_kernel_LN.$(SUFFIX) cher2k_kernel_LC.$(SUFFIX) + cher2k_kernel_LN.$(SUFFIX) cher2k_kernel_LC.$(SUFFIX) cgemm_batch_thread.$(SUFFIX) ZBLASOBJS += \ zgemm_nn.$(SUFFIX) zgemm_cn.$(SUFFIX) zgemm_tn.$(SUFFIX) zgemm_nc.$(SUFFIX) \ @@ -137,7 +137,7 @@ ZBLASOBJS += \ zherk_kernel_LN.$(SUFFIX) zherk_kernel_LC.$(SUFFIX) \ zsyr2k_kernel_U.$(SUFFIX) zsyr2k_kernel_L.$(SUFFIX) \ zher2k_kernel_UN.$(SUFFIX) zher2k_kernel_UC.$(SUFFIX) \ - zher2k_kernel_LN.$(SUFFIX) zher2k_kernel_LC.$(SUFFIX) + zher2k_kernel_LN.$(SUFFIX) zher2k_kernel_LC.$(SUFFIX) zgemm_batch_thread.$(SUFFIX) XBLASOBJS += \ @@ -2942,6 +2942,21 @@ gemm_thread_variable.$(PSUFFIX) : gemm_thread_variable.c ../../common.h beta_thread.$(PSUFFIX) : beta_thread.c ../../common.h $(CC) -c $(PFLAGS) $< -o $(@F) +sbgemm_batch_thread.$(SUFFIX) : gemm_batch_thread.c ../../common.h + $(CC) -c $(CFLAGS) $< -o $(@F) + +sgemm_batch_thread.$(SUFFIX) : gemm_batch_thread.c ../../common.h + $(CC) -c $(CFLAGS) $< -o $(@F) + +dgemm_batch_thread.$(SUFFIX) : gemm_batch_thread.c ../../common.h + $(CC) -c $(CFLAGS) $< -o $(@F) + +cgemm_batch_thread.$(SUFFIX) : gemm_batch_thread.c ../../common.h + $(CC) -c $(CFLAGS) $< -o $(@F) + +zgemm_batch_thread.$(SUFFIX) : gemm_batch_thread.c ../../common.h + $(CC) -c $(CFLAGS) $< -o $(@F) + sbgemm_thread_nn.$(PSUFFIX) : gemm.c level3_thread.c ../../param.h $(CC) $(PFLAGS) $(BLOCKS) -c -DTHREADED_LEVEL3 -DHALF -UDOUBLE -UCOMPLEX -DNN $< -o $(@F) diff --git a/driver/level3/gemm_batch_thread.c b/driver/level3/gemm_batch_thread.c new file mode 100644 index 0000000000..45d6977ba2 --- /dev/null +++ b/driver/level3/gemm_batch_thread.c @@ -0,0 +1,156 @@ +/***************************************************************************** +Copyright (c) 2020, The OpenBLAS Project +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. Neither the name of the OpenBLAS project nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +**********************************************************************************/ + +#include "common.h" + +void openblas_warning(int verbose, const char * msg); + +#ifdef SMALL_MATRIX_OPT +static int inner_small_matrix_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, IFLOAT *sa, IFLOAT *sb, BLASLONG mypos){ + int routine_mode; +#ifndef COMPLEX + int (*gemm_small_kernel)(BLASLONG, BLASLONG, BLASLONG, FLOAT *, BLASLONG, FLOAT ,FLOAT *, BLASLONG, FLOAT, FLOAT *, BLASLONG); + int (*gemm_small_kernel_b0)(BLASLONG, BLASLONG, BLASLONG, FLOAT *, BLASLONG, FLOAT, FLOAT *, BLASLONG, FLOAT *, BLASLONG); +#else + int (*zgemm_small_kernel)(BLASLONG, BLASLONG, BLASLONG, FLOAT *, BLASLONG, FLOAT , FLOAT, FLOAT *, BLASLONG, FLOAT , FLOAT, FLOAT *, BLASLONG); + int (*zgemm_small_kernel_b0)(BLASLONG, BLASLONG, BLASLONG, FLOAT *, BLASLONG, FLOAT , FLOAT, FLOAT *, BLASLONG, FLOAT *, BLASLONG); + FLOAT alpha[2], beta[2]; +#endif + routine_mode=args->routine_mode; + if((routine_mode & BLAS_SMALL_B0_OPT) == BLAS_SMALL_B0_OPT){ +#ifndef COMPLEX + gemm_small_kernel_b0=args->routine; + gemm_small_kernel_b0(args->m, args->n, args->k, args->a, args->lda, *(FLOAT *)(args->alpha), args->b, args->ldb, args->c, args->ldc); +#else + zgemm_small_kernel_b0=args->routine; + alpha[0] = *((FLOAT *)args -> alpha + 0); + alpha[1] = *((FLOAT *)args -> alpha + 1); + zgemm_small_kernel_b0(args->m, args->n, args->k, args->a, args->lda, alpha[0], alpha[1], args->b, args->ldb, args->c, args->ldc); +#endif + return(0); + }else if(routine_mode & BLAS_SMALL_OPT){ +#ifndef COMPLEX + gemm_small_kernel=args->routine; + gemm_small_kernel(args->m, args->n, args->k, args->a, args->lda, *(FLOAT *)(args->alpha), args->b, args->ldb, *(FLOAT *)(args->beta), args->c, args->ldc); +#else + zgemm_small_kernel=args->routine; + alpha[0] = *((FLOAT *)args -> alpha + 0); + alpha[1] = *((FLOAT *)args -> alpha + 1); + beta[0] = *((FLOAT *)args -> beta + 0); + beta[1] = *((FLOAT *)args -> beta + 1); + zgemm_small_kernel(args->m, args->n, args->k, args->a, args->lda, alpha[0], alpha[1], args->b, args->ldb, beta[0], beta[1], args->c, args->ldc); +#endif + return(0); + } + return(1); +} +#endif + +int CNAME(blas_arg_t * args_array, BLASLONG nums){ + XFLOAT *buffer; + XFLOAT *sa, *sb; + int nthreads=1; + int (*routine)(blas_arg_t *, void *, void *, XFLOAT *, XFLOAT *, BLASLONG); + int i=0, /*j,*/ current_nums; + +#ifdef SMP + blas_queue_t * queue=NULL; +#endif + + if(nums <=0 ) return 0; + + buffer = (XFLOAT *)blas_memory_alloc(0); + sa = (XFLOAT *)((BLASLONG)buffer +GEMM_OFFSET_A); + sb = (XFLOAT *)(((BLASLONG)sa + ((GEMM_P * GEMM_Q * COMPSIZE * SIZE + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); + +#ifdef SMP + nthreads=num_cpu_avail(3); + + if(nthreads==1){ + +#endif + //single thread + for(i=0; inthreads)? nthreads: (nums-i); + + queue[i].sa=sa; + queue[i].sb=sb; + queue[i+current_nums-1].next=NULL; + + exec_blas(current_nums, &queue[i]); + } + free(queue); + } +#endif + blas_memory_free(buffer); + return 0; +} From 362a06339677f1018690ab409f6a8cde664de347 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 29 May 2024 23:16:58 +0200 Subject: [PATCH 101/267] remove return value --- interface/gemm_batch.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/gemm_batch.c b/interface/gemm_batch.c index d674f9615a..846d8e0f4d 100644 --- a/interface/gemm_batch.c +++ b/interface/gemm_batch.c @@ -169,7 +169,7 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE * transa_array, enum CB if(args_array == NULL){ openblas_warning(0, "memory alloc failed!\n"); - return(1); + return; } #ifdef SMP From dd4505c5ddbc0ca4b6fe40b43d27af13086d4bce Mon Sep 17 00:00:00 2001 From: Alexander Neumann <30894796+Neumann-A@users.noreply.github.com> Date: Thu, 30 May 2024 09:04:23 +0200 Subject: [PATCH 102/267] Fix CMake warning --- cmake/prebuild.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/prebuild.cmake b/cmake/prebuild.cmake index 47e9584197..6d98ed32ef 100644 --- a/cmake/prebuild.cmake +++ b/cmake/prebuild.cmake @@ -1342,7 +1342,7 @@ else(NOT CMAKE_CROSSCOMPILING) if ("${CMAKE_C_COMPILER_ID}" STREQUAL "MSVC") #Use generic for MSVC now - message("MSVC") + message(STATUS "MSVC") set(GETARCH_FLAGS ${GETARCH_FLAGS} -DFORCE_GENERIC) else() list(APPEND GETARCH_SRC ${PROJECT_SOURCE_DIR}/cpuid.S) From ff6670cb83ebfb12c199576ba1d2f459b0c9d750 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 30 May 2024 18:26:02 +0200 Subject: [PATCH 103/267] don't generate non-cblas files for gemm_batch --- interface/CMakeLists.txt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index 55374674aa..01e67de6f2 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -97,6 +97,9 @@ foreach (CBLAS_FLAG ${CBLAS_FLAGS}) #sdsdot, dsdot if (BUILD_SINGLE OR BUILD_DOUBLE) GenerateNamedObjects("sdsdot.c" "" "sdsdot" ${CBLAS_FLAG} "" "" true "SINGLE") + if(CBLAS_FLAG EQUAL 1) + GenerateNamedObjects("gemm_batch.c" "" "gemm_batch" ${CBLAS_FLAG} "" "" true) +endif () endif () if (BUILD_DOUBLE) GenerateNamedObjects("dsdot.c" "" "dsdot" ${CBLAS_FLAG} "" "" true "SINGLE") @@ -125,6 +128,9 @@ if (BUILD_BFLOAT16) GenerateNamedObjects("tobf16.c" "DOUBLE_PREC" "sbdtobf16" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("bf16to.c" "SINGLE_PREC" "sbf16tos" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("bf16to.c" "DOUBLE_PREC" "dbf16tod" ${CBLAS_FLAG} "" "" true "BFLOAT16") + if(CBLAS_FLAG EQUAL 1) + GenerateNamedObjects("gemm_batch.c" "" "sbgemm_batch" ${CBLAS_FLAG} "" "" true "BFLOAT16") +endif () endif () # complex-specific sources @@ -154,6 +160,9 @@ foreach (float_type ${FLOAT_TYPES}) GenerateNamedObjects("max.c" "USE_ABS" "scamax" ${CBLAS_FLAG} "" "" true "COMPLEX") GenerateNamedObjects("asum.c" "" "scasum" ${CBLAS_FLAG} "" "" true "COMPLEX") GenerateNamedObjects("sum.c" "" "scsum" ${CBLAS_FLAG} "" "" true "COMPLEX") + if(CBLAS_FLAG EQUAL 1) + GenerateNamedObjects("gemm_batch.c" "" "cgemm_batch" ${CBLAS_FLAG} "" "" true "COMPLEX") + endif () endif () if (${float_type} STREQUAL "ZCOMPLEX") GenerateNamedObjects("zscal.c" "SSCAL" "dscal" ${CBLAS_FLAG} "" "" false "ZCOMPLEX") @@ -163,6 +172,9 @@ foreach (float_type ${FLOAT_TYPES}) GenerateNamedObjects("max.c" "USE_ABS" "dzamax" ${CBLAS_FLAG} "" "" true "ZCOMPLEX") GenerateNamedObjects("asum.c" "" "dzasum" ${CBLAS_FLAG} "" "" true "ZCOMPLEX") GenerateNamedObjects("sum.c" "" "dzsum" ${CBLAS_FLAG} "" "" true "ZCOMPLEX") + if(CBLAS_FLAG EQUAL 1) + GenerateNamedObjects("gemm_batch.c" "" "zgemm_batch" ${CBLAS_FLAG} "" "" true "ZCOMPLEX") + endif () endif () endforeach () @@ -212,6 +224,7 @@ if ( BUILD_COMPLEX AND NOT BUILD_SINGLE) GenerateNamedObjects("nrm2.c" "" "nrm2" 0 "" "" false "SINGLE") GenerateNamedObjects("gemv.c" "" "gemv" 0 "" "" false "SINGLE") GenerateNamedObjects("gemm.c" "" "gemm" 0 "" "" false "SINGLE") + GenerateNamedObjects("gemm_batch.c" "" "gemm_batch" 1 "" "" false "SINGLE") GenerateNamedObjects("asum.c" "" "asum" 0 "" "" false "SINGLE") GenerateNamedObjects("swap.c" "" "swap" 0 "" "" false "SINGLE") GenerateNamedObjects("axpy.c" "" "axpy" 0 "" "" false "SINGLE") @@ -225,6 +238,7 @@ if ( BUILD_COMPLEX16 AND NOT BUILD_DOUBLE) GenerateNamedObjects("nrm2.c" "" "nrm2" 0 "" "" false "DOUBLE") GenerateNamedObjects("gemv.c" "" "gemv" 0 "" "" false "DOUBLE") GenerateNamedObjects("gemm.c" "" "gemm" 0 "" "" false "DOUBLE") + GenerateNamedObjects("gemm_batch.c" "" "gemm_batch" 1 "" "" false "DOUBLE") GenerateNamedObjects("asum.c" "" "asum" 0 "" "" false "DOUBLE") GenerateNamedObjects("swap.c" "" "swap" 0 "" "" false "DOUBLE") GenerateNamedObjects("axpy.c" "" "axpy" 0 "" "" false "DOUBLE") From 0d007adb18234289562b8556b5a0963125adda5a Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 30 May 2024 23:30:16 +0200 Subject: [PATCH 104/267] fix clang_cl-flang job to use flang-new after the llvm update --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3ae8615a7e..7f82a461dc 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -133,7 +133,7 @@ jobs: mkdir build cd build call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" - cmake -G "Ninja" -DCMAKE_C_COMPILER=cl -DCMAKE_Fortran_COMPILER=flang -DC_LAPACK=1 -DCMAKE_MT=mt -DCMAKE_BUILD_TYPE=Release -DMSVC_STATIC_CRT=ON .. + cmake -G "Ninja" -DCMAKE_C_COMPILER=cl -DCMAKE_Fortran_COMPILER=flang-new -DC_LAPACK=1 -DCMAKE_MT=mt -DCMAKE_BUILD_TYPE=Release -DMSVC_STATIC_CRT=ON .. cmake --build . --config Release ctest From ad2b5c67c8508d568e142f4fa6593a7fb90eb3a3 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 31 May 2024 01:06:58 +0200 Subject: [PATCH 105/267] fix another corner case involving infinity --- kernel/x86_64/zscal.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/kernel/x86_64/zscal.c b/kernel/x86_64/zscal.c index 075b6091fe..05114c7fab 100644 --- a/kernel/x86_64/zscal.c +++ b/kernel/x86_64/zscal.c @@ -278,7 +278,9 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i, temp0 = NAN; else temp0 = -da_i * x[i+1]; - x[i+1] = da_i * x[i]; + if (!isinf(x[i+1])) + x[i+1] = da_i * x[i]; + else x[i+1] = NAN; x[i] = temp0; i += inc_x ; j++; @@ -412,7 +414,9 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i, temp0 = -da_i * x[i+1]; if (isinf(x[i])) temp0 = NAN; - x[i+1] = da_i * x[i]; + if (!isinf(x[i+1])) + x[i+1] = da_i * x[i]; + else x[i+1] = NAN; if ( x[i] == x[i]) //preserve NaN x[i] = temp0; i += 2 ; From a16f8249baa7bba156fef2085ee5a57332e94e79 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 31 May 2024 01:08:17 +0200 Subject: [PATCH 106/267] add tests with the imaginary part of the array infinite --- utest/test_zscal.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/utest/test_zscal.c b/utest/test_zscal.c index 195e4945f3..22642630c7 100644 --- a/utest/test_zscal.c +++ b/utest/test_zscal.c @@ -117,4 +117,31 @@ CTEST(zscal, inf_i_inc_2) ASSERT_TRUE(isinf(i[17])); } +CTEST(zscal, i_0inf) +{ + blasint N=9; + blasint incX=1; + double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; + double inf[] = {0,INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY}; + BLASFUNC(zscal)(&N, i, inf, &incX); + ASSERT_TRUE(isinf(inf[0])); + ASSERT_TRUE(isnan(inf[1])); + ASSERT_TRUE(isinf(inf[16])); + ASSERT_TRUE(isnan(inf[17])); +} + +CTEST(zscal, i_0inf_inc_2) +{ + blasint N=9; + blasint incX=2; + double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; + double inf[] = {0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY, + 0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY}; + BLASFUNC(zscal)(&N, i, inf, &incX); + ASSERT_TRUE(isinf(inf[0])); + ASSERT_TRUE(isnan(inf[1])); + ASSERT_TRUE(isinf(inf[16])); + ASSERT_TRUE(isnan(inf[17])); +} + #endif From ab13cfef9330def014ce64a11a06b93a24696186 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 31 May 2024 14:34:49 +0200 Subject: [PATCH 107/267] more fixes for infinite x --- kernel/x86_64/zscal.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/kernel/x86_64/zscal.c b/kernel/x86_64/zscal.c index 05114c7fab..f557557276 100644 --- a/kernel/x86_64/zscal.c +++ b/kernel/x86_64/zscal.c @@ -258,13 +258,17 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i, temp0 = NAN; else temp0 = -da_i * x[i+1]; - x[i+1] = da_i * x[i]; + if (!isinf(x[i+1])) + x[i+1] = da_i * x[i]; + else x[i+1] = NaN; x[i] = temp0; if (isnan(x[i+inc_x]) || isinf(x[i+inc_x])) temp1 = NAN; else temp1 = -da_i * x[i+1+inc_x]; - x[i+1+inc_x] = da_i * x[i+inc_x]; + if (!isinf(x[i+1+inc_x])) + x[i+1+inc_x] = da_i * x[i+inc_x]; + else x[i+1+inc_x] = NaN; x[i+inc_x] = temp1; i += 2*inc_x ; j+=2; From ce130f11d2663baa3a44fb90a0e9a25c8cc34295 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 31 May 2024 15:09:03 +0200 Subject: [PATCH 108/267] Update zscal.c --- kernel/x86_64/zscal.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/x86_64/zscal.c b/kernel/x86_64/zscal.c index f557557276..7859ef6e38 100644 --- a/kernel/x86_64/zscal.c +++ b/kernel/x86_64/zscal.c @@ -260,7 +260,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i, temp0 = -da_i * x[i+1]; if (!isinf(x[i+1])) x[i+1] = da_i * x[i]; - else x[i+1] = NaN; + else x[i+1] = NAN; x[i] = temp0; if (isnan(x[i+inc_x]) || isinf(x[i+inc_x])) temp1 = NAN; @@ -268,7 +268,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i, temp1 = -da_i * x[i+1+inc_x]; if (!isinf(x[i+1+inc_x])) x[i+1+inc_x] = da_i * x[i+inc_x]; - else x[i+1+inc_x] = NaN; + else x[i+1+inc_x] = NAN; x[i+inc_x] = temp1; i += 2*inc_x ; j+=2; From 9ff4e9714e93c7dcc4deb9934846ec6b4a3099f6 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 31 May 2024 15:44:07 +0200 Subject: [PATCH 109/267] additional fixes for handling INF arguments --- kernel/arm/zscal.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel/arm/zscal.c b/kernel/arm/zscal.c index b2d537d04e..c4855f73ea 100644 --- a/kernel/arm/zscal.c +++ b/kernel/arm/zscal.c @@ -61,7 +61,9 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r,FLOAT da_i, F { temp = - da_i * x[ip+1] ; if (isnan(x[ip]) || isinf(x[ip])) temp = NAN; - x[ip+1] = da_i * x[ip] ; + if (!isinf(x[ip+1])) + x[ip+1] = da_i * x[ip] ; + else x[ip+1] = NAN; } } else From 516743f7dcdf9f2f43041fcb6b8ca6d8dcb76136 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 31 May 2024 16:02:12 +0200 Subject: [PATCH 110/267] fix other instances of mishandling INF --- kernel/riscv64/zscal.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel/riscv64/zscal.c b/kernel/riscv64/zscal.c index b2d537d04e..c4855f73ea 100644 --- a/kernel/riscv64/zscal.c +++ b/kernel/riscv64/zscal.c @@ -61,7 +61,9 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r,FLOAT da_i, F { temp = - da_i * x[ip+1] ; if (isnan(x[ip]) || isinf(x[ip])) temp = NAN; - x[ip+1] = da_i * x[ip] ; + if (!isinf(x[ip+1])) + x[ip+1] = da_i * x[ip] ; + else x[ip+1] = NAN; } } else From 8c05765a5a31e47d84c5c7b6bb89f89dde538c01 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 31 May 2024 18:06:36 +0200 Subject: [PATCH 111/267] fix other corner cases where x=INF --- kernel/mips/zscal.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/kernel/mips/zscal.c b/kernel/mips/zscal.c index 7bb2619414..ae1c87fcea 100644 --- a/kernel/mips/zscal.c +++ b/kernel/mips/zscal.c @@ -48,7 +48,9 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r,FLOAT da_i, F { temp = - da_i * x[ip+1] ; if (isnan(x[ip]) || isinf(x[ip])) temp = NAN; - x[ip+1] = da_i * x[ip] ; + if (!isinf(x[ip+1])) + x[ip+1] = da_i * x[ip] ; + else x[ip+1] = NAN; } } else @@ -56,12 +58,16 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r,FLOAT da_i, F if ( da_i == 0.0 ) { temp = da_r * x[ip] ; - x[ip+1] = da_r * x[ip+1]; + if (!isinf(x[ip+1])) + x[ip+1] = da_r * x[ip+1]; + else x[ip+1] = NAN; } else { temp = da_r * x[ip] - da_i * x[ip+1] ; - x[ip+1] = da_r * x[ip+1] + da_i * x[ip] ; + if (!isinf(x[ip+1])) + x[ip+1] = da_r * x[ip+1] + da_i * x[ip] ; + else x[ip+1] = NAN; } } if ( da_r != da_r ) From 076766df4e3a147da3e57ffec3f2fcba106c78b1 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 31 May 2024 18:23:18 +0200 Subject: [PATCH 112/267] Update CMakeLists.txt --- interface/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index 01e67de6f2..05b893b8c4 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -98,7 +98,7 @@ foreach (CBLAS_FLAG ${CBLAS_FLAGS}) if (BUILD_SINGLE OR BUILD_DOUBLE) GenerateNamedObjects("sdsdot.c" "" "sdsdot" ${CBLAS_FLAG} "" "" true "SINGLE") if(CBLAS_FLAG EQUAL 1) - GenerateNamedObjects("gemm_batch.c" "" "gemm_batch" ${CBLAS_FLAG} "" "" true) + GenerateNamedObjects("gemm_batch.c" "" "gemm_batch" ${CBLAS_FLAG} "" "" false) endif () endif () if (BUILD_DOUBLE) From db070a92238fd451ebb66561d72c2ce2e1524b58 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 31 May 2024 18:29:27 +0200 Subject: [PATCH 113/267] add gemm_batch drivers --- driver/level3/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/driver/level3/CMakeLists.txt b/driver/level3/CMakeLists.txt index 75b25d039c..b1ec94c233 100644 --- a/driver/level3/CMakeLists.txt +++ b/driver/level3/CMakeLists.txt @@ -68,6 +68,8 @@ if (USE_THREAD) endif () foreach (float_type ${FLOAT_TYPES}) + GenerateNamedObjects("gemm_batch_thread.c" "" "gemm_batch_thread" 0 "" "" false ${float_type}) + if (${float_type} STREQUAL "COMPLEX" OR ${float_type} STREQUAL "ZCOMPLEX") GenerateCombinationObjects("zherk_kernel.c" "LOWER;CONJ" "U;N" "HERK" 2 "herk_kernel" false ${float_type}) # TRANS needs to be set/unset when CONJ is set/unset, so can't use it as a combination From 020b3e168279873ce462d09420479ca9f2606a38 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 1 Jun 2024 00:51:18 +0200 Subject: [PATCH 114/267] fix handling of INF arguments --- kernel/x86_64/dscal.c | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/kernel/x86_64/dscal.c b/kernel/x86_64/dscal.c index 05c5c7f16b..0b028b7829 100644 --- a/kernel/x86_64/dscal.c +++ b/kernel/x86_64/dscal.c @@ -169,8 +169,12 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS while(j < n1) { - x[i]=0.0; - x[i+inc_x]=0.0; + if (isinf(x[i])||isnan(x[i])) + x[i]=NAN; + else x[i]=0.0; + if (isinf(x[i+inc_x])||isnan(x[i+inc_x])) + x[i+inc_x]=NAN; + else x[i+inc_x]=0.0; i += 2*inc_x ; j+=2; @@ -179,7 +183,9 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS while(j < n) { - x[i]=0.0; + if (isinf(x[i])||isnan(x[i])) + x[i]=NAN; + else x[i]=0.0; i += inc_x ; j++; @@ -213,9 +219,9 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS BLASLONG n1 = n & -8; if ( n1 > 0 ) { - if ( da == 0.0 ) - dscal_kernel_8_zero(n1 , &da , x); - else +// if ( da == 0.0 ) +// dscal_kernel_8_zero(n1 , &da , x); +// else dscal_kernel_8(n1 , &da , x); } @@ -223,7 +229,9 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS { for ( i=n1 ; i Date: Sat, 1 Jun 2024 11:22:26 +0100 Subject: [PATCH 115/267] Updated CONTRIBUTORS.md As requested on X (https://x.com/KroekerMartin/status/1755218919290278185) --- CONTRIBUTORS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 2033208263..d885a01b96 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -198,6 +198,9 @@ In chronological order: * PingTouGe Semiconductor Co., Ltd. * [2020-10] Add RISC-V Vector (0.7.1) support. Optimize BLAS kernels for Xuantie C910 +* Jake Arkinstall + * [2021-02-10] Remove in-source configure_file to enable builds in read-only contexts (issue #3100, PR #3101) + * River Dillon * [2021-07-10] fix compilation with musl libc From db9f7bc5526e70a601c374709a24424c21ab91a9 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Mon, 3 Jun 2024 00:22:16 +0200 Subject: [PATCH 116/267] fix float array types to include bfloat16 --- interface/gemm_batch.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interface/gemm_batch.c b/interface/gemm_batch.c index 846d8e0f4d..56ccc12ce4 100644 --- a/interface/gemm_batch.c +++ b/interface/gemm_batch.c @@ -118,8 +118,8 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE * transa_array, enum CB blasint * m_array, blasint * n_array, blasint * k_array, #ifndef COMPLEX FLOAT * alpha_array, - FLOAT ** a_array, blasint * lda_array, - FLOAT ** b_array, blasint * ldb_array, + IFLOAT ** a_array, blasint * lda_array, + IFLOAT ** b_array, blasint * ldb_array, FLOAT * beta_array, FLOAT ** c_array, blasint * ldc_array, blasint group_count, blasint * group_size) { #else From df87aeb5a2a3785e15a3d94dbf92e2e03448500f Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 4 Jun 2024 09:49:18 +0200 Subject: [PATCH 117/267] Drop the -static Fortran flag from generic builds as it breaks OpenMP --- Makefile.riscv64 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile.riscv64 b/Makefile.riscv64 index 113cc57c53..9f6e48b7ad 100644 --- a/Makefile.riscv64 +++ b/Makefile.riscv64 @@ -8,13 +8,13 @@ FCOMMON_OPT += -march=rv64imafdcv_zba_zbb_zfh -mabi=lp64d -static endif ifeq ($(CORE), RISCV64_ZVL256B) CCOMMON_OPT += -march=rv64imafdcv_zvl256b -mabi=lp64d -FCOMMON_OPT += -march=rv64imafdcv -mabi=lp64d -static +FCOMMON_OPT += -march=rv64imafdcv -mabi=lp64d endif ifeq ($(CORE), RISCV64_ZVL128B) CCOMMON_OPT += -march=rv64imafdcv -mabi=lp64d -FCOMMON_OPT += -march=rv64imafdcv -mabi=lp64d -static +FCOMMON_OPT += -march=rv64imafdcv -mabi=lp64d endif ifeq ($(CORE), RISCV64_GENERIC) CCOMMON_OPT += -march=rv64imafdc -mabi=lp64d -FCOMMON_OPT += -march=rv64imafdc -mabi=lp64d -static +FCOMMON_OPT += -march=rv64imafdc -mabi=lp64d endif From 8ab2e9ec65c8c182a7189cf5574e4f2f51e02e5c Mon Sep 17 00:00:00 2001 From: gxw Date: Sat, 16 Sep 2023 11:19:12 +0800 Subject: [PATCH 118/267] LoongArch: DGEMM small matrix opt --- Makefile.system | 3 + kernel/loongarch64/KERNEL.LOONGSON3R5 | 10 + .../loongarch64/dgemm_small_kernel_nn_lasx.S | 549 +++++++++++++++ .../loongarch64/dgemm_small_kernel_nt_lasx.S | 500 ++++++++++++++ .../loongarch64/dgemm_small_kernel_tn_lasx.S | 639 ++++++++++++++++++ .../loongarch64/dgemm_small_kernel_tt_lasx.S | 534 +++++++++++++++ .../loongarch64/dgemm_small_matrix_permit.c | 44 ++ 7 files changed, 2279 insertions(+) create mode 100644 kernel/loongarch64/dgemm_small_kernel_nn_lasx.S create mode 100644 kernel/loongarch64/dgemm_small_kernel_nt_lasx.S create mode 100644 kernel/loongarch64/dgemm_small_kernel_tn_lasx.S create mode 100644 kernel/loongarch64/dgemm_small_kernel_tt_lasx.S create mode 100644 kernel/loongarch64/dgemm_small_matrix_permit.c diff --git a/Makefile.system b/Makefile.system index f452011ad2..4cd4e4a1c7 100644 --- a/Makefile.system +++ b/Makefile.system @@ -269,6 +269,9 @@ else ifeq ($(ARCH), power) SMALL_MATRIX_OPT = 1 BUILD_BFLOAT16 = 1 endif +ifeq ($(ARCH), loongarch64) +SMALL_MATRIX_OPT = 1 +endif ifeq ($(SMALL_MATRIX_OPT), 1) CCOMMON_OPT += -DSMALL_MATRIX_OPT endif diff --git a/kernel/loongarch64/KERNEL.LOONGSON3R5 b/kernel/loongarch64/KERNEL.LOONGSON3R5 index 2c1ab87e59..eff1581d9c 100644 --- a/kernel/loongarch64/KERNEL.LOONGSON3R5 +++ b/kernel/loongarch64/KERNEL.LOONGSON3R5 @@ -162,4 +162,14 @@ STRSMKERNEL_LN = ../generic/trsm_kernel_LN.c STRSMKERNEL_LT = ../generic/trsm_kernel_LT.c STRSMKERNEL_RN = ../generic/trsm_kernel_RN.c STRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +DGEMM_SMALL_M_PERMIT = dgemm_small_matrix_permit.c +DGEMM_SMALL_K_NN = dgemm_small_kernel_nn_lasx.S +DGEMM_SMALL_K_B0_NN = dgemm_small_kernel_nn_lasx.S +DGEMM_SMALL_K_NT = dgemm_small_kernel_nt_lasx.S +DGEMM_SMALL_K_B0_NT = dgemm_small_kernel_nt_lasx.S +DGEMM_SMALL_K_TN = dgemm_small_kernel_tn_lasx.S +DGEMM_SMALL_K_B0_TN = dgemm_small_kernel_tn_lasx.S +DGEMM_SMALL_K_TT = dgemm_small_kernel_tt_lasx.S +DGEMM_SMALL_K_B0_TT = dgemm_small_kernel_tt_lasx.S endif diff --git a/kernel/loongarch64/dgemm_small_kernel_nn_lasx.S b/kernel/loongarch64/dgemm_small_kernel_nn_lasx.S new file mode 100644 index 0000000000..a50350ddce --- /dev/null +++ b/kernel/loongarch64/dgemm_small_kernel_nn_lasx.S @@ -0,0 +1,549 @@ +/*************************************************************************** +Copyright (c) 2024, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#define ASSEMBLER + +#include "common.h" +#include "loongarch64_asm.S" + +#define M $a0 +#define N $a1 +#define K $a2 +#define A $a3 +#define LDA $a4 +#define ALPHA $f0 +#define B $a5 +#define LDB $a6 +#define C $a7 +#define LDC $t0 +#ifdef B0 +#define BETA $f1 +#endif +#undef ZERO +#define ZERO $r0 + +#define M16 $t1 +#define M8 $t1 +#define M4 $t1 +#define M2 $t1 +#define M1 $t1 +#define N4 $t2 +#define N2 $t2 +#define N1 $t2 +#define K8 $t3 +#define A0 $t4 +#define X0 $t5 +#define B1 $t6 +#define B2 $t7 +#define B3 $t8 +#define C0 $s0 +#define C1 $s1 +#define C2 $s2 +#define C3 $s3 +#define K1 $s4 + +#define VALPHA $xr0 +#ifndef B0 +#define VBETA $xr1 +#endif +#define D0 $xr2 +#define D1 $xr3 +#define D2 $xr4 +#define D3 $xr5 +#define D4 $xr6 +#define D5 $xr7 +#define D6 $xr8 +#define D7 $xr9 +#define D8 $xr10 +#define D9 $xr11 +#define D10 $xr12 +#define D11 $xr13 +#define D12 $xr14 +#define D13 $xr15 +#define D14 $xr16 +#define D15 $xr17 +#define S0 $xr18 +#define S1 $xr19 +#define S2 $xr20 +#define S3 $xr21 +#define Z0 $xr22 +#define Z1 $xr23 +#define Z2 $xr24 +#define Z3 $xr25 +#define V0 $vr2 +#define V1 $vr3 +#define V2 $vr4 +#define V3 $vr5 +#define F0 $f2 +#define F1 $f3 +#define F2 $f4 +#define F3 $f5 + +.macro DGEMM_SMALL_KERNEL_NN_TAIL M + PTR_SRAI N4, N, 2 // N >> 2 + move A0, A // Restore A0 + move X0, B // Restore X0 + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C // Restore C0 + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M\M\()_N3 +.L_M\M\()_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K // Restore K1 + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M\M\()_N4_END +.L_M\M\()_N4_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00 + GLDREPL xv, d, Z0, X0, 0x00, Z1, B1, 0x00, Z2, B2, 0x00, Z3, B3, 0x00 + GMADD xvf, d, D0, S0, Z0, D0, D1, S0, Z1, D1, D2, S0, Z2, D2, D3, S0, Z3, D3 + PTR_ADDI X0, X0, 0x08 + PTR_ADDI B1, B1, 0x08 + PTR_ADDI B2, B2, 0x08 + PTR_ADDI B3, B3, 0x08 + PTR_ADD A0, A0, LDA + bnez K1, .L_M\M\()_N4_K1 +.L_M\M\()_N4_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0 + GLD xv, , S0, C1, 0x00 + GMADD xvf, d, D1, S0, VBETA, D1 + GLD xv, , S0, C2, 0x00 + GMADD xvf, d, D2, S0, VBETA, D2 + GLD xv, , S0, C3, 0x00 + GMADD xvf, d, D3, S0, VBETA, D3 +#endif +.if \M == 4 + GST xv, , D0, C0, 0x00, D1, C1, 0x00, D2, C2, 0x00, D3, C3, 0x00 +.elseif \M == 2 + GST v, , V0, C0, 0x00, V1, C1, 0x00, V2, C2, 0x00, V3, C3, 0x00 +.elseif \M == 1 + GST f, d, F0, C0, 0x00, F1, C1, 0x00, F2, C2, 0x00, F3, C3, 0x00 +.endif + // Update C0, C1, C2, C3 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + // Update X0, B1, B2, B3 + PTR_SUB X0, X0, K8 + PTR_SUB B1, B1, K8 + PTR_SUB B2, B2, K8 + PTR_SUB B3, B3, K8 + PTR_ALSL X0, LDB, X0, 2 + PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2 + PTR_ALSL B3, LDB, B3, 2 + // Restore A0 + move A0, A + bnez N4, .L_M\M\()_N4 +.L_M\M\()_N3: + andi N2, N, 0x02 + beqz N2, .L_M\M\()_N1 +.L_M\M\()_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1 + move K1, K // Restore K1 + bge ZERO, K, .L_M\M\()_N2_END +.L_M\M\()_N2_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00 + GLDREPL xv, d, Z0, X0, 0x00, Z1, B1, 0x00 + GMADD xvf, d, D0, S0, Z0, D0, D1, S0, Z1, D1 + PTR_ADDI X0, X0, 0x08 + PTR_ADDI B1, B1, 0x08 + PTR_ADD A0, A0, LDA + bnez K1, .L_M\M\()_N2_K1 +.L_M\M\()_N2_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0 + GLD xv, , S0, C1, 0x00 + GMADD xvf, d, D1, S0, VBETA, D1 +#endif +.if \M == 4 + GST xv, , D0, C0, 0x00, D1, C1, 0x00 +.elseif \M == 2 + GST v, , V0, C0, 0x00, V1, C1, 0x00 +.elseif \M == 1 + GST f, d, F0, C0, 0x00, F1, C1, 0x00 +.endif + // Update C0, C1 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + // Update X0, B1 + PTR_SUB X0, X0, K8 + PTR_SUB B1, B1, K8 + PTR_ALSL X0, LDB, X0, 1 + PTR_ALSL B1, LDB, B1, 1 + // Restore A0 + move A0, A +.L_M\M\()_N1: + andi N1, N, 0x01 + beqz N1, .L_M\M\()_END + GXOR xv, v, D0, D0, D0 + move K1, K // Restore K1 + bge ZERO, K, .L_M\M\()_N1_END +.L_M\M\()_N1_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00 + GLDREPL xv, d, Z0, X0, 0x00 + GMADD xvf, d, D0, S0, Z0, D0 + PTR_ADDI X0, X0, 0x08 + PTR_ADD A0, A0, LDA + bnez K1, .L_M\M\()_N1_K1 +.L_M\M\()_N1_END: + GMUL xvf, d, D0, D0, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0 +#endif +.if \M == 4 + GST xv, , D0, C0, 0x00 +.elseif \M == 2 + GST v, , V0, C0, 0x00 +.elseif \M == 1 + GST f, d, F0, C0, 0x00 +.endif +.L_M\M\()_END: +.if \M == 4 + PTR_ADDI A, A, 0x20 + PTR_ADDI C, C, 0x20 +.elseif \M == 2 + PTR_ADDI A, A, 0x10 + PTR_ADDI C, C, 0x10 +.elseif \M == 1 +.endif +.endm + + PROLOGUE + PTR_LD LDC, $sp, 0 + push_if_used 5, 2 + xvreplve0.d VALPHA, VALPHA +#ifndef B0 + xvreplve0.d VBETA, VBETA +#endif + PTR_SLLI LDA, LDA, 3 + PTR_SLLI LDB, LDB, 3 + PTR_SLLI LDC, LDC, 3 + PTR_SLLI K8, K, 3 + PTR_SRAI M16, M, 4 // M >> 4 + beqz M16, .L_M15 +.L_M16: + PTR_SRAI N4, N, 2 // N >> 2 + move A0, A // Restore A0 + move X0, B // Restore X0 + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C // Restore C0 + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M16_N3 +.L_M16_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3, \ + D4, D4, D4, D5, D5, D5, D6, D6, D6, D7, D7, D7, \ + D8, D8, D8, D9, D9, D9, D10, D10, D10, D11, D11, D11, \ + D12, D12, D12, D13, D13, D13, D14, D14, D14, D15, D15, D15 + move K1, K // Restore K1 + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M16_N4_END +.L_M16_N4_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20, S2, A0, 0x40, S3, A0, 0x60 + GLDREPL xv, d, Z0, X0, 0x00, Z1, B1, 0x00, Z2, B2, 0x00, Z3, B3, 0x00 + GMADD xvf, d, D0, S0, Z0, D0, D1, S1, Z0, D1, D2, S2, Z0, D2, D3, S3, Z0, D3, \ + D4, S0, Z1, D4, D5, S1, Z1, D5, D6, S2, Z1, D6, D7, S3, Z1, D7, \ + D8, S0, Z2, D8, D9, S1, Z2, D9, D10, S2, Z2, D10, D11, S3, Z2, D11, \ + D12, S0, Z3, D12, D13, S1, Z3, D13, D14, S2, Z3, D14, D15, S3, Z3, D15 + PTR_ADDI X0, X0, 0x08 + PTR_ADDI B1, B1, 0x08 + PTR_ADDI B2, B2, 0x08 + PTR_ADDI B3, B3, 0x08 + PTR_ADD A0, A0, LDA + bnez K1, .L_M16_N4_K1 + .L_M16_N4_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA, \ + D4, D4, VALPHA, D5, D5, VALPHA, D6, D6, VALPHA, D7, D7, VALPHA, \ + D8, D8, VALPHA, D9, D9, VALPHA, D10, D10, VALPHA, D11, D11, VALPHA, \ + D12, D12, VALPHA, D13, D13, VALPHA, D14, D14, VALPHA, D15, D15, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20, S2, C0, 0x40, S3, C0, 0x60 + GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20, S2, C1, 0x40, S3, C1, 0x60 + GMADD xvf, d, D4, S0, VBETA, D4, D5, S1, VBETA, D5, D6, S2, VBETA, D6, D7, S3, VBETA, D7 + GLD xv, , S0, C2, 0x00, S1, C2, 0x20, S2, C2, 0x40, S3, C2, 0x60 + GMADD xvf, d, D8, S0, VBETA, D8, D9, S1, VBETA, D9, D10, S2, VBETA, D10, D11, S3, VBETA, D11 + GLD xv, , S0, C3, 0x00, S1, C3, 0x20, S2, C3, 0x40, S3, C3, 0x60 + GMADD xvf, d, D12, S0, VBETA, D12, D13, S1, VBETA, D13, D14, S2, VBETA, D14, D15, S3, VBETA, D15 +#endif + GST xv, , D12, C3, 0x00, D13, C3, 0x20, D14, C3, 0x40, D15, C3, 0x60, \ + D8, C2, 0x00, D9, C2, 0x20, D10, C2, 0x40, D11, C2, 0x60, \ + D4, C1, 0x00, D5, C1, 0x20, D6, C1, 0x40, D7, C1, 0x60, \ + D0, C0, 0x00, D1, C0, 0x20, D2, C0, 0x40, D3, C0, 0x60 + // Update C0, C1, C2, C3 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + // Update X0, B1, B2, B3 + PTR_SUB X0, X0, K8 + PTR_SUB B1, B1, K8 + PTR_SUB B2, B2, K8 + PTR_SUB B3, B3, K8 + + PTR_ALSL X0, LDB, X0, 2 + PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2 + PTR_ALSL B3, LDB, B3, 2 + // Restore A0 + move A0, A + bnez N4, .L_M16_N4 +.L_M16_N3: + andi N2, N, 0x02 + beqz N2, .L_M16_N1 +.L_M16_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3, \ + D4, D4, D4, D5, D5, D5, D6, D6, D6, D7, D7, D7 + move K1, K // Restore K1 + bge ZERO, K, .L_M16_N2_END +.L_M16_N2_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20, S2, A0, 0x40, S3, A0, 0x60 + GLDREPL xv, d, Z0, X0, 0x00, Z1, B1, 0x00 + GMADD xvf, d, D0, S0, Z0, D0, D1, S1, Z0, D1, D2, S2, Z0, D2, D3, S3, Z0, D3, \ + D4, S0, Z1, D4, D5, S1, Z1, D5, D6, S2, Z1, D6, D7, S3, Z1, D7 + PTR_ADDI X0, X0, 0x08 + PTR_ADDI B1, B1, 0x08 + PTR_ADD A0, A0, LDA + bnez K1, .L_M16_N2_K1 +.L_M16_N2_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA, \ + D4, D4, VALPHA, D5, D5, VALPHA, D6, D6, VALPHA, D7, D7, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20, S2, C0, 0x40, S3, C0, 0x60 + GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20, S2, C1, 0x40, S3, C1, 0x60 + GMADD xvf, d, D4, S0, VBETA, D4, D5, S1, VBETA, D5, D6, S2, VBETA, D6, D7, S3, VBETA, D7 +#endif + GST xv, , D4, C1, 0x00, D5, C1, 0x20, D6, C1, 0x40, D7, C1, 0x60, \ + D0, C0, 0x00, D1, C0, 0x20, D2, C0, 0x40, D3, C0, 0x60 + // Update C0, C1, C2, C3 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + // Update X0, B1, B2, B3 + PTR_SUB X0, X0, K8 + PTR_SUB B1, B1, K8 + PTR_ALSL X0, LDB, X0, 1 + PTR_ALSL B1, LDB, B1, 1 + // Restore A0 + move A0, A +.L_M16_N1: + andi N1, N, 0x01 + beqz N1, .L_M16_END + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K // Restore K1 + bge ZERO, K, .L_M16_N1_END +.L_M16_N1_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20, S2, A0, 0x40, S3, A0, 0x60 + GLDREPL xv, d, Z0, X0, 0x00 + GMADD xvf, d, D0, S0, Z0, D0, D1, S1, Z0, D1, D2, S2, Z0, D2, D3, S3, Z0, D3 + PTR_ADDI X0, X0, 0x08 + PTR_ADD A0, A0, LDA + bnez K1, .L_M16_N1_K1 +.L_M16_N1_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20, S2, C0, 0x40, S3, C0, 0x60 + GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3 +#endif + GST xv, , D0, C0, 0x00, D1, C0, 0x20, D2, C0, 0x40, D3, C0, 0x60 + // Update C0, C1, C2, C3 + PTR_ALSL C0, LDC, C0, 2 + // Update X0, B1, B2, B3 + PTR_SUB X0, X0, K8 + PTR_ALSL X0, LDB, X0, 2 + // Restore A0 + move A0, A +.L_M16_END: + PTR_ADDI M16, M16, -1 + PTR_ADDI A, A, 0x80 + PTR_ADDI C, C, 0x80 + bnez M16, .L_M16 +.L_M15: + andi M8, M, 0x08 + beqz M8, .L_M7 +.L_M8: + PTR_SRAI N4, N, 2 // N >> 2 + move A0, A // Restore A0 + move X0, B // Restore X0 + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C // Restore C0 + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M8_N3 +.L_M8_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3, \ + D4, D4, D4, D5, D5, D5, D6, D6, D6, D7, D7, D7 + move K1, K // Restore K1 + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M8_N4_END +.L_M8_N4_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20 + GLDREPL xv, d, Z0, X0, 0x00, Z1, B1, 0x00, Z2, B2, 0x00, Z3, B3, 0x00 + GMADD xvf, d, D0, S0, Z0, D0, D1, S1, Z0, D1, \ + D2, S0, Z1, D2, D3, S1, Z1, D3, \ + D4, S0, Z2, D4, D5, S1, Z2, D5, \ + D6, S0, Z3, D6, D7, S1, Z3, D7, + PTR_ADDI X0, X0, 0x08 + PTR_ADDI B1, B1, 0x08 + PTR_ADDI B2, B2, 0x08 + PTR_ADDI B3, B3, 0x08 + PTR_ADD A0, A0, LDA + bnez K1, .L_M8_N4_K1 +.L_M8_N4_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA, \ + D4, D4, VALPHA, D5, D5, VALPHA, D6, D6, VALPHA, D7, D7, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20 + GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20 + GMADD xvf, d, D2, S0, VBETA, D2, D3, S1, VBETA, D3 + GLD xv, , S0, C2, 0x00, S1, C2, 0x20 + GMADD xvf, d, D4, S0, VBETA, D4, D5, S1, VBETA, D5 + GLD xv, , S0, C3, 0x00, S1, C3, 0x20 + GMADD xvf, d, D6, S0, VBETA, D6, D7, S1, VBETA, D7 +#endif + GST xv, , D4, C2, 0x00, D5, C2, 0x20, D6, C3, 0x00, D7, C3, 0x20, \ + D0, C0, 0x00, D1, C0, 0x20, D2, C1, 0x00, D3, C1, 0x20 + // Update C0, C1, C2, C3 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + // Update X0, B1, B2, B3 + PTR_SUB X0, X0, K8 + PTR_SUB B1, B1, K8 + PTR_SUB B2, B2, K8 + PTR_SUB B3, B3, K8 + PTR_ALSL X0, LDB, X0, 2 + PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2 + PTR_ALSL B3, LDB, B3, 2 + // Restore A0 + move A0, A + bnez N4, .L_M8_N4 +.L_M8_N3: + andi N2, N, 0x02 + beqz N2, .L_M8_N1 +.L_M8_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K // Restore K1 + bge ZERO, K, .L_M8_N2_END +.L_M8_N2_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20 + GLDREPL xv, d, Z0, X0, 0x00, Z1, B1, 0x00 + GMADD xvf, d, D0, S0, Z0, D0, D1, S1, Z0, D1, \ + D2, S0, Z1, D2, D3, S1, Z1, D3 + PTR_ADDI X0, X0, 0x08 + PTR_ADDI B1, B1, 0x08 + PTR_ADD A0, A0, LDA + bnez K1, .L_M8_N2_K1 +.L_M8_N2_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20 + GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20 + GMADD xvf, d, D2, S0, VBETA, D2, D3, S1, VBETA, D3 +#endif + GST xv, , D0, C0, 0x00, D1, C0, 0x20, D2, C1, 0x00, D3, C1, 0x20 + // Update C0, C1 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + // Update X0, B1 + PTR_SUB X0, X0, K8 + PTR_SUB B1, B1, K8 + PTR_ALSL X0, LDB, X0, 1 + PTR_ALSL B1, LDB, B1, 1 + // Restore A0 + move A0, A +.L_M8_N1: + andi N1, N, 0x01 + beqz N1, .L_M8_END + GXOR xv, v, D0, D0, D0, D1, D1, D1 + move K1, K // Restore K1 + bge ZERO, K, .L_M8_N1_END +.L_M8_N1_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20 + GLDREPL xv, d, Z0, X0, 0x00 + GMADD xvf, d, D0, S0, Z0, D0, D1, S1, Z0, D1 + PTR_ADDI X0, X0, 0x08 + PTR_ADD A0, A0, LDA + bnez K1, .L_M8_N1_K1 +.L_M8_N1_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20 + GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1 +#endif + GST xv, , D0, C0, 0x00, D1, C0, 0x20 +.L_M8_END: + PTR_ADDI A, A, 0x40 + PTR_ADDI C, C, 0x40 +.L_M7: + andi M4, M, 0x04 + beqz M4, .L_M3 +.L_M4: + DGEMM_SMALL_KERNEL_NN_TAIL 4 +.L_M3: + andi M2, M, 0x02 + beqz M2, .L_M1 +.L_M2: + DGEMM_SMALL_KERNEL_NN_TAIL 2 +.L_M1: + andi M1, M, 0x01 + beqz M1, .L_M0 + DGEMM_SMALL_KERNEL_NN_TAIL 1 +.L_M0: + pop_if_used 5, 2 + jirl $r0, $r1, 0x0 + EPILOGUE diff --git a/kernel/loongarch64/dgemm_small_kernel_nt_lasx.S b/kernel/loongarch64/dgemm_small_kernel_nt_lasx.S new file mode 100644 index 0000000000..aee0586f52 --- /dev/null +++ b/kernel/loongarch64/dgemm_small_kernel_nt_lasx.S @@ -0,0 +1,500 @@ +/*************************************************************************** +Copyright (c) 2024 The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#define ASSEMBLER + +#include "common.h" +#include "loongarch64_asm.S" + +#define M $a0 +#define N $a1 +#define K $a2 +#define A $a3 +#define LDA $a4 +#define ALPHA $f0 +#define B $a5 +#define LDB $a6 +#define C $a7 +#define LDC $t0 +#ifdef B0 +#define BETA $f1 +#endif +#undef ZERO +#define ZERO $r0 + +#define M16 $t1 +#define M8 $t1 +#define M4 $t1 +#define M2 $t1 +#define M1 $t1 +#define N4 $t2 +#define N2 $t2 +#define N1 $t2 +#define K_LDB $t3 +#define A0 $t4 +#define X0 $t5 +#define C0 $t6 +#define C1 $t7 +#define C2 $t8 +#define C3 $s0 +#define K1 $s1 + +#define VALPHA $xr0 +#ifndef B0 +#define VBETA $xr1 +#endif +#define D0 $xr2 +#define D1 $xr3 +#define D2 $xr4 +#define D3 $xr5 +#define D4 $xr6 +#define D5 $xr7 +#define D6 $xr8 +#define D7 $xr9 +#define D8 $xr10 +#define D9 $xr11 +#define D10 $xr12 +#define D11 $xr13 +#define D12 $xr14 +#define D13 $xr15 +#define D14 $xr16 +#define D15 $xr17 +#define S0 $xr18 +#define S1 $xr19 +#define S2 $xr20 +#define S3 $xr21 +#define Z0 $xr22 +#define Z1 $xr23 +#define Z2 $xr24 +#define Z3 $xr25 +#define V0 $vr2 +#define V1 $vr3 +#define V2 $vr4 +#define V3 $vr5 +#define F0 $f2 +#define F1 $f3 +#define F2 $f4 +#define F3 $f5 + +.macro DGEMM_SMALL_KERNEL_NT_TAIL M + PTR_SRAI N4, N, 2 // N >> 2 + move A0, A // Restore A0 + move X0, B // Restore X0 + move C0, C // Restore C0 + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M\M\()_N3 +.L_M\M\()_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K // Restore K1 + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M\M\()_N4_END +.L_M\M\()_N4_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00 + GLDREPL xv, d, Z0, X0, 0x00, Z1, X0, 0x08, Z2, X0, 0x10, Z3, X0, 0x18 + GMADD xvf, d, D0, S0, Z0, D0, D1, S0, Z1, D1, D2, S0, Z2, D2, D3, S0, Z3, D3 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M\M\()_N4_K1 +.L_M\M\()_N4_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0 + GLD xv, , S0, C1, 0x00 + GMADD xvf, d, D1, S0, VBETA, D1 + GLD xv, , S0, C2, 0x00 + GMADD xvf, d, D2, S0, VBETA, D2 + GLD xv, , S0, C3, 0x00 + GMADD xvf, d, D3, S0, VBETA, D3 +#endif +.if \M == 4 + GST xv, , D0, C0, 0x00, D1, C1, 0x00, D2, C2, 0x00, D3, C3, 0x00 +.elseif \M == 2 + GST v, , V0, C0, 0x00, V1, C1, 0x00, V2, C2, 0x00, V3, C3, 0x00 +.elseif \M == 1 + GST f, d, F0, C0, 0x00, F1, C1, 0x00, F2, C2, 0x00, F3, C3, 0x00 +.endif + // Update C0, C1, C2, C3 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + // Update X0 + PTR_SUB X0, X0, K_LDB + PTR_ADDI X0, X0, 0x20 + // Restore A0 + move A0, A + bnez N4, .L_M\M\()_N4 +.L_M\M\()_N3: + andi N2, N, 0x02 + beqz N2, .L_M\M\()_N1 +.L_M\M\()_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1 + move K1, K // Restore K1 + bge ZERO, K, .L_M\M\()_N2_END +.L_M\M\()_N2_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00 + GLDREPL xv, d, Z0, X0, 0x00, Z1, X0, 0x08 + GMADD xvf, d, D0, S0, Z0, D0, D1, S0, Z1, D1 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M\M\()_N2_K1 +.L_M\M\()_N2_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0 + GLD xv, , S0, C1, 0x00 + GMADD xvf, d, D1, S0, VBETA, D1 +#endif +.if \M == 4 + GST xv, , D0, C0, 0x00, D1, C1, 0x00 +.elseif \M == 2 + GST v, , V0, C0, 0x00, V1, C1, 0x00 +.elseif \M == 1 + GST f, d, F0, C0, 0x00, F1, C1, 0x00 +.endif + // Update C0, C1 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + // Update X0 + PTR_SUB X0, X0, K_LDB + PTR_ADDI X0, X0, 0x10 + // Restore A0 + move A0, A +.L_M\M\()_N1: + andi N1, N, 0x01 + beqz N1, .L_M\M\()_END + GXOR xv, v, D0, D0, D0 + move K1, K // Restore K1 + bge ZERO, K, .L_M\M\()_N1_END +.L_M\M\()_N1_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00 + GLDREPL xv, d, Z0, X0, 0x00 + GMADD xvf, d, D0, S0, Z0, D0 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M\M\()_N1_K1 +.L_M\M\()_N1_END: + GMUL xvf, d, D0, D0, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0 +#endif +.if \M == 4 + GST xv, , D0, C0, 0x00 +.elseif \M == 2 + GST v, , V0, C0, 0x00 +.elseif \M == 1 + GST f, d, F0, C0, 0x00 +.endif +.L_M\M\()_END: +.if \M == 4 + PTR_ADDI A, A, 0x20 + PTR_ADDI C, C, 0x20 +.elseif \M == 2 + PTR_ADDI A, A, 0x10 + PTR_ADDI C, C, 0x10 +.elseif \M == 1 +.endif +.endm + + PROLOGUE + PTR_LD LDC, $sp, 0 + push_if_used 2, 2 + xvreplve0.d VALPHA, VALPHA +#ifndef B0 + xvreplve0.d VBETA, VBETA +#endif + PTR_SLLI LDA, LDA, 3 + PTR_SLLI LDB, LDB, 3 + PTR_SLLI LDC, LDC, 3 + PTR_MUL K_LDB, K, LDB + PTR_SRAI M16, M, 4 // M >> 4 + beqz M16, .L_M15 +.L_M16: + PTR_SRAI N4, N, 2 // N >> 2 + move A0, A // Restore A0 + move X0, B // Restore X0 + move C0, C // Restore C0 + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M16_N3 +.L_M16_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3, \ + D4, D4, D4, D5, D5, D5, D6, D6, D6, D7, D7, D7, \ + D8, D8, D8, D9, D9, D9, D10, D10, D10, D11, D11, D11, \ + D12, D12, D12, D13, D13, D13, D14, D14, D14, D15, D15, D15 + move K1, K // Restore K1 + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M16_N4_END +.L_M16_N4_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20, S2, A0, 0x40, S3, A0, 0x60 + GLDREPL xv, d, Z0, X0, 0x00, Z1, X0, 0x08, Z2, X0, 0x10, Z3, X0, 0x18 + GMADD xvf, d, D0, S0, Z0, D0, D1, S1, Z0, D1, D2, S2, Z0, D2, D3, S3, Z0, D3, \ + D4, S0, Z1, D4, D5, S1, Z1, D5, D6, S2, Z1, D6, D7, S3, Z1, D7, \ + D8, S0, Z2, D8, D9, S1, Z2, D9, D10, S2, Z2, D10, D11, S3, Z2, D11, \ + D12, S0, Z3, D12, D13, S1, Z3, D13, D14, S2, Z3, D14, D15, S3, Z3, D15 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M16_N4_K1 + .L_M16_N4_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA, \ + D4, D4, VALPHA, D5, D5, VALPHA, D6, D6, VALPHA, D7, D7, VALPHA, \ + D8, D8, VALPHA, D9, D9, VALPHA, D10, D10, VALPHA, D11, D11, VALPHA, \ + D12, D12, VALPHA, D13, D13, VALPHA, D14, D14, VALPHA, D15, D15, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20, S2, C0, 0x40, S3, C0, 0x60 + GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20, S2, C1, 0x40, S3, C1, 0x60 + GMADD xvf, d, D4, S0, VBETA, D4, D5, S1, VBETA, D5, D6, S2, VBETA, D6, D7, S3, VBETA, D7 + GLD xv, , S0, C2, 0x00, S1, C2, 0x20, S2, C2, 0x40, S3, C2, 0x60 + GMADD xvf, d, D8, S0, VBETA, D8, D9, S1, VBETA, D9, D10, S2, VBETA, D10, D11, S3, VBETA, D11 + GLD xv, , S0, C3, 0x00, S1, C3, 0x20, S2, C3, 0x40, S3, C3, 0x60 + GMADD xvf, d, D12, S0, VBETA, D12, D13, S1, VBETA, D13, D14, S2, VBETA, D14, D15, S3, VBETA, D15 +#endif + GST xv, , D12, C3, 0x00, D13, C3, 0x20, D14, C3, 0x40, D15, C3, 0x60, \ + D8, C2, 0x00, D9, C2, 0x20, D10, C2, 0x40, D11, C2, 0x60, \ + D4, C1, 0x00, D5, C1, 0x20, D6, C1, 0x40, D7, C1, 0x60, \ + D0, C0, 0x00, D1, C0, 0x20, D2, C0, 0x40, D3, C0, 0x60 + // Update C0, C1, C2, C3 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + // Update X0 + PTR_SUB X0, X0, K_LDB + PTR_ADDI X0, X0, 0x20 + // Restore A0 + move A0, A + bnez N4, .L_M16_N4 +.L_M16_N3: + andi N2, N, 0x02 + beqz N2, .L_M16_N1 +.L_M16_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3, \ + D4, D4, D4, D5, D5, D5, D6, D6, D6, D7, D7, D7 + move K1, K // Restore K1 + bge ZERO, K, .L_M16_N2_END +.L_M16_N2_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20, S2, A0, 0x40, S3, A0, 0x60 + GLDREPL xv, d, Z0, X0, 0x00, Z1, X0, 0x08 + GMADD xvf, d, D0, S0, Z0, D0, D1, S1, Z0, D1, D2, S2, Z0, D2, D3, S3, Z0, D3, \ + D4, S0, Z1, D4, D5, S1, Z1, D5, D6, S2, Z1, D6, D7, S3, Z1, D7 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M16_N2_K1 +.L_M16_N2_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA, \ + D4, D4, VALPHA, D5, D5, VALPHA, D6, D6, VALPHA, D7, D7, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20, S2, C0, 0x40, S3, C0, 0x60 + GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20, S2, C1, 0x40, S3, C1, 0x60 + GMADD xvf, d, D4, S0, VBETA, D4, D5, S1, VBETA, D5, D6, S2, VBETA, D6, D7, S3, VBETA, D7 +#endif + GST xv, , D4, C1, 0x00, D5, C1, 0x20, D6, C1, 0x40, D7, C1, 0x60, \ + D0, C0, 0x00, D1, C0, 0x20, D2, C0, 0x40, D3, C0, 0x60 + // Update C0, C1 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + // Update X0 + PTR_SUB X0, X0, K_LDB + PTR_ADDI X0, X0, 0x10 + // Restore A0 + move A0, A +.L_M16_N1: + andi N1, N, 0x01 + beqz N1, .L_M16_END + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K // Restore K1 + bge ZERO, K, .L_M16_N1_END +.L_M16_N1_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20, S2, A0, 0x40, S3, A0, 0x60 + GLDREPL xv, d, Z0, X0, 0x00 + GMADD xvf, d, D0, S0, Z0, D0, D1, S1, Z0, D1, D2, S2, Z0, D2, D3, S3, Z0, D3 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M16_N1_K1 +.L_M16_N1_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20, S2, C0, 0x40, S3, C0, 0x60 + GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3 +#endif + GST xv, , D0, C0, 0x00, D1, C0, 0x20, D2, C0, 0x40, D3, C0, 0x60 + // Update C0 + PTR_ALSL C0, LDC, C0, 2 + // Update X0 + PTR_SUB X0, X0, K_LDB + PTR_ADDI X0, X0, 0x08 + // Restore A0 + move A0, A +.L_M16_END: + PTR_ADDI M16, M16, -1 + PTR_ADDI A, A, 0x80 + PTR_ADDI C, C, 0x80 + bnez M16, .L_M16 +.L_M15: + andi M8, M, 0x08 + beqz M8, .L_M7 +.L_M8: + PTR_SRAI N4, N, 2 // N >> 2 + move A0, A // Restore A0 + move X0, B // Restore X0 + move C0, C // Restore C0 + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M8_N3 +.L_M8_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3, \ + D4, D4, D4, D5, D5, D5, D6, D6, D6, D7, D7, D7 + move K1, K // Restore K1 + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M8_N4_END +.L_M8_N4_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20 + GLDREPL xv, d, Z0, X0, 0x00, Z1, X0, 0x08, Z2, X0, 0x10, Z3, X0, 0x18 + GMADD xvf, d, D0, S0, Z0, D0, D1, S1, Z0, D1, \ + D2, S0, Z1, D2, D3, S1, Z1, D3, \ + D4, S0, Z2, D4, D5, S1, Z2, D5, \ + D6, S0, Z3, D6, D7, S1, Z3, D7, + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M8_N4_K1 +.L_M8_N4_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA, \ + D4, D4, VALPHA, D5, D5, VALPHA, D6, D6, VALPHA, D7, D7, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20 + GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20 + GMADD xvf, d, D2, S0, VBETA, D2, D3, S1, VBETA, D3 + GLD xv, , S0, C2, 0x00, S1, C2, 0x20 + GMADD xvf, d, D4, S0, VBETA, D4, D5, S1, VBETA, D5 + GLD xv, , S0, C3, 0x00, S1, C3, 0x20 + GMADD xvf, d, D6, S0, VBETA, D6, D7, S1, VBETA, D7 +#endif + GST xv, , D4, C2, 0x00, D5, C2, 0x20, D6, C3, 0x00, D7, C3, 0x20, \ + D0, C0, 0x00, D1, C0, 0x20, D2, C1, 0x00, D3, C1, 0x20 + // Update C0, C1, C2, C3 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + // Update X0 + PTR_SUB X0, X0, K_LDB + PTR_ADDI X0, X0, 0x20 + // Restore A0 + move A0, A + bnez N4, .L_M8_N4 +.L_M8_N3: + andi N2, N, 0x02 + beqz N2, .L_M8_N1 +.L_M8_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K // Restore K1 + bge ZERO, K, .L_M8_N2_END +.L_M8_N2_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20 + GLDREPL xv, d, Z0, X0, 0x00, Z1, X0, 0x08 + GMADD xvf, d, D0, S0, Z0, D0, D1, S1, Z0, D1, \ + D2, S0, Z1, D2, D3, S1, Z1, D3 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M8_N2_K1 +.L_M8_N2_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20 + GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20 + GMADD xvf, d, D2, S0, VBETA, D2, D3, S1, VBETA, D3 +#endif + GST xv, , D0, C0, 0x00, D1, C0, 0x20, D2, C1, 0x00, D3, C1, 0x20 + // Update C0, C1 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + // Update X0 + PTR_SUB X0, X0, K_LDB + PTR_ADDI X0, X0, 0x10 + // Restore A0 + move A0, A +.L_M8_N1: + andi N1, N, 0x01 + beqz N1, .L_M8_END + GXOR xv, v, D0, D0, D0, D1, D1, D1 + move K1, K // Restore K1 + bge ZERO, K, .L_M8_N1_END +.L_M8_N1_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20 + GLDREPL xv, d, Z0, X0, 0x00 + GMADD xvf, d, D0, S0, Z0, D0, D1, S1, Z0, D1 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M8_N1_K1 +.L_M8_N1_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20 + GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1 +#endif + GST xv, , D0, C0, 0x00, D1, C0, 0x20 +.L_M8_END: + PTR_ADDI A, A, 0x40 + PTR_ADDI C, C, 0x40 +.L_M7: + andi M4, M, 0x04 + beqz M4, .L_M3 +.L_M4: + DGEMM_SMALL_KERNEL_NT_TAIL 4 +.L_M3: + andi M2, M, 0x02 + beqz M2, .L_M1 +.L_M2: + DGEMM_SMALL_KERNEL_NT_TAIL 2 +.L_M1: + andi M1, M, 0x01 + beqz M1, .L_M0 + DGEMM_SMALL_KERNEL_NT_TAIL 1 +.L_M0: + pop_if_used 2, 2 + jirl $r0, $r1, 0x0 + EPILOGUE diff --git a/kernel/loongarch64/dgemm_small_kernel_tn_lasx.S b/kernel/loongarch64/dgemm_small_kernel_tn_lasx.S new file mode 100644 index 0000000000..b1e5886064 --- /dev/null +++ b/kernel/loongarch64/dgemm_small_kernel_tn_lasx.S @@ -0,0 +1,639 @@ +/*************************************************************************** +Copyright (c) 2024, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#define ASSEMBLER + +#include "common.h" +#include "loongarch64_asm.S" + +#define M $a0 +#define N $a1 +#define K $a2 +#define A $a3 +#define LDA $a4 +#define ALPHA $f0 +#define B $a5 +#define LDB $a6 +#define C $a7 +#define LDC $t0 +#ifdef B0 +#define BETA $f1 +#endif +#undef ZERO +#define ZERO $r0 + +#define M4 $t1 +#define M2 $t1 +#define M1 $t1 +#define N4 $t2 +#define N2 $t2 +#define N1 $t2 +#define K8 $t3 +#define A0 $t4 +#define X0 $t5 +#define B1 $t6 +#define B2 $t7 +#define B3 $t8 +#define C0 $s0 +#define C1 $s1 +#define C2 $s2 +#define C3 $s3 +#define K1 $s4 +#define A1 $s5 +#define A2 $s6 +#define A3 $s7 + +#define VALPHA $xr0 +#ifndef B0 +#define VBETA $xr1 +#endif +#define D0 $xr2 +#define D1 $xr3 +#define D2 $xr4 +#define D3 $xr5 +#define T0 $xr6 +#define T1 $xr7 +#define T2 $xr8 +#define T3 $xr9 +#define Y0 $xr10 +#define Y1 $xr11 +#define Y2 $xr12 +#define Y3 $xr13 +#define G0 $xr14 +#define G1 $xr15 +#define G2 $xr16 +#define G3 $xr17 +#define S0 $xr18 +#define S1 $xr19 +#define S2 $xr20 +#define S3 $xr21 +#define Z0 $xr22 +#define Z1 $xr23 +#define Z2 $xr24 +#define Z3 $xr25 +#define V0 $vr2 +#define V1 $vr3 +#define V2 $vr4 +#define V3 $vr5 +#define F0 $f2 +#define F1 $f3 +#define F2 $f4 +#define F3 $f5 + + PROLOGUE + PTR_LD LDC, $sp, 0 + push_if_used 8, 2 + xvreplve0.d VALPHA, VALPHA +#ifndef B0 + xvreplve0.d VBETA, VBETA +#endif + PTR_SLLI LDA, LDA, 3 + PTR_SLLI LDB, LDB, 3 + PTR_SLLI LDC, LDC, 3 + PTR_SLLI K8, K, 3 + PTR_SRAI M4, M, 2 // M >> 2 + beqz M4, .L_M3 +.L_M4: + PTR_SRAI N4, N, 2 // N >> 2 + move A0, A // Restore A0 + PTR_ADD A1, A0, LDA + PTR_ADD A2, A1, LDA + PTR_ADD A3, A2, LDA + move X0, B // Restore X0 + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C // Restore C0 + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M4_N3 +.L_M4_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K // Restore K1 + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M4_N4_END + PTR_SRAI K1, K1, 3 + beq ZERO, K1, .L_M4_N4_K7 +.L_M4_N4_K8: + PTR_ADDI K1, K1, -1 + GLD xv, , T0, A0, 0x00, T1, A1, 0x00, T2, A2, 0x00, T3, A3, 0x00 + GTRANSPOSE4x4_D T0, T1, T2, T3, S0, S1, S2, S3, Z0, Z1 + + GLDREPL xv, d, Z0, X0, 0x00, Z1, B1, 0x00, Z2, B2, 0x00, Z3, B3, 0x00 + GLDREPL xv, d, T0, X0, 0x08, T1, B1, 0x08, T2, B2, 0x08, T3, B3, 0x08 + GLDREPL xv, d, Y0, X0, 0x10, Y1, B1, 0x10, Y2, B2, 0x10, Y3, B3, 0x10 + GLDREPL xv, d, G0, X0, 0x18, G1, B1, 0x18, G2, B2, 0x18, G3, B3, 0x18 + GMADD xvf, d, D0, S0, Z0, D0, \ + D1, S0, Z1, D1, \ + D2, S0, Z2, D2, \ + D3, S0, Z3, D3 + GMADD xvf, d, D0, S1, T0, D0, \ + D1, S1, T1, D1, \ + D2, S1, T2, D2, \ + D3, S1, T3, D3 + GMADD xvf, d, D0, S2, Y0, D0, \ + D1, S2, Y1, D1, \ + D2, S2, Y2, D2, \ + D3, S2, Y3, D3 + GMADD xvf, d, D0, S3, G0, D0, \ + D1, S3, G1, D1, \ + D2, S3, G2, D2, \ + D3, S3, G3, D3 + + GLD xv, , T0, A0, 0x20, T1, A1, 0x20, T2, A2, 0x20, T3, A3, 0x20 + GTRANSPOSE4x4_D T0, T1, T2, T3, S0, S1, S2, S3, Z0, Z1 + + GLDREPL xv, d, Z0, X0, 0x20, Z1, B1, 0x20, Z2, B2, 0x20, Z3, B3, 0x20 + GLDREPL xv, d, T0, X0, 0x28, T1, B1, 0x28, T2, B2, 0x28, T3, B3, 0x28 + GLDREPL xv, d, Y0, X0, 0x30, Y1, B1, 0x30, Y2, B2, 0x30, Y3, B3, 0x30 + GLDREPL xv, d, G0, X0, 0x38, G1, B1, 0x38, G2, B2, 0x38, G3, B3, 0x38 + GMADD xvf, d, D0, S0, Z0, D0, \ + D1, S0, Z1, D1, \ + D2, S0, Z2, D2, \ + D3, S0, Z3, D3 + GMADD xvf, d, D0, S1, T0, D0, \ + D1, S1, T1, D1, \ + D2, S1, T2, D2, \ + D3, S1, T3, D3 + GMADD xvf, d, D0, S2, Y0, D0, \ + D1, S2, Y1, D1, \ + D2, S2, Y2, D2, \ + D3, S2, Y3, D3 + GMADD xvf, d, D0, S3, G0, D0, \ + D1, S3, G1, D1, \ + D2, S3, G2, D2, \ + D3, S3, G3, D3 + + PTR_ADDI X0, X0, 0x40 + PTR_ADDI B1, B1, 0x40 + PTR_ADDI B2, B2, 0x40 + PTR_ADDI B3, B3, 0x40 + PTR_ADDI A0, A0, 0x40 + PTR_ADDI A1, A1, 0x40 + PTR_ADDI A2, A2, 0x40 + PTR_ADDI A3, A3, 0x40 + bnez K1, .L_M4_N4_K8 + .L_M4_N4_K7: + andi K1, K, 4 + beqz K1, .L_M4_N4_3 + .L_M4_N4_K4: + GLD xv, , T0, A0, 0x00, T1, A1, 0x00, T2, A2, 0x00, T3, A3, 0x00 + GTRANSPOSE4x4_D T0, T1, T2, T3, S0, S1, S2, S3, Z0, Z1 + + GLDREPL xv, d, Z0, X0, 0x00, Z1, B1, 0x00, Z2, B2, 0x00, Z3, B3, 0x00 + GLDREPL xv, d, T0, X0, 0x08, T1, B1, 0x08, T2, B2, 0x08, T3, B3, 0x08 + GLDREPL xv, d, Y0, X0, 0x10, Y1, B1, 0x10, Y2, B2, 0x10, Y3, B3, 0x10 + GLDREPL xv, d, G0, X0, 0x18, G1, B1, 0x18, G2, B2, 0x18, G3, B3, 0x18 + GMADD xvf, d, D0, S0, Z0, D0, \ + D1, S0, Z1, D1, \ + D2, S0, Z2, D2, \ + D3, S0, Z3, D3 + GMADD xvf, d, D0, S1, T0, D0, \ + D1, S1, T1, D1, \ + D2, S1, T2, D2, \ + D3, S1, T3, D3 + GMADD xvf, d, D0, S2, Y0, D0, \ + D1, S2, Y1, D1, \ + D2, S2, Y2, D2, \ + D3, S2, Y3, D3 + GMADD xvf, d, D0, S3, G0, D0, \ + D1, S3, G1, D1, \ + D2, S3, G2, D2, \ + D3, S3, G3, D3 + PTR_ADDI X0, X0, 0x20 + PTR_ADDI B1, B1, 0x20 + PTR_ADDI B2, B2, 0x20 + PTR_ADDI B3, B3, 0x20 + PTR_ADDI A0, A0, 0x20 + PTR_ADDI A1, A1, 0x20 + PTR_ADDI A2, A2, 0x20 + PTR_ADDI A3, A3, 0x20 + .L_M4_N4_3: + andi K1, K, 3 + beqz K1, .L_M4_N4_END + .L_M4_N4_K1: + GLD xv, , S0, A0, 0x00, S1, A1, 0x00, S2, A2, 0x00, S3, A3, 0x00 + GINSVE0 xv, d, S0, S1, 1, S0, S2, 2, S0, S3, 3 + GLDREPL xv, d, Z0, X0, 0x00, Z1, B1, 0x00, Z2, B2, 0x00, Z3, B3, 0x00 + GMADD xvf, d, D0, S0, Z0, D0, \ + D1, S0, Z1, D1, \ + D2, S0, Z2, D2, \ + D3, S0, Z3, D3 + PTR_ADDI K1, K1, -1 + PTR_ADDI X0, X0, 0x08 + PTR_ADDI B1, B1, 0x08 + PTR_ADDI B2, B2, 0x08 + PTR_ADDI B3, B3, 0x08 + PTR_ADDI A0, A0, 0x08 + PTR_ADDI A1, A1, 0x08 + PTR_ADDI A2, A2, 0x08 + PTR_ADDI A3, A3, 0x08 + bnez K1, .L_M4_N4_K1 + .L_M4_N4_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C1, 0x00, S2, C2, 0x00, S3, C3, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0, \ + D1, S1, VBETA, D1, \ + D2, S2, VBETA, D2, \ + D3, S3, VBETA, D3 +#endif + GST xv, , D3, C3, 0x00, \ + D2, C2, 0x00, \ + D1, C1, 0x00, \ + D0, C0, 0x00 + // Update C0, C1, C2, C3 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + // Update X0, B1, B2, B3 + PTR_SUB X0, X0, K8 + PTR_SUB B1, B1, K8 + PTR_SUB B2, B2, K8 + PTR_SUB B3, B3, K8 + + PTR_ALSL X0, LDB, X0, 2 + PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2 + PTR_ALSL B3, LDB, B3, 2 + // Restore A0, A1, A2, A3 + move A0, A + PTR_ADD A1, A0, LDA + PTR_ADD A2, A1, LDA + PTR_ADD A3, A2, LDA + bnez N4, .L_M4_N4 +.L_M4_N3: + andi N2, N, 0x02 + beqz N2, .L_M4_N1 +.L_M4_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1 + move K1, K // Restore K1 + bge ZERO, K, .L_M4_N2_END +.L_M4_N2_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A1, 0x00, S2, A2, 0x00, S3, A3, 0x00 + GINSVE0 xv, d, S0, S1, 1, S0, S2, 2, S0, S3, 3 + GLDREPL xv, d, Z0, X0, 0x00, Z1, B1, 0x00 + GMADD xvf, d, D0, S0, Z0, D0, \ + D1, S0, Z1, D1 + PTR_ADDI X0, X0, 0x08 + PTR_ADDI B1, B1, 0x08 + PTR_ADDI A0, A0, 0x08 + PTR_ADDI A1, A1, 0x08 + PTR_ADDI A2, A2, 0x08 + PTR_ADDI A3, A3, 0x08 + bnez K1, .L_M4_N2_K1 +.L_M4_N2_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C1, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1 +#endif + GST xv, , D1, C1, 0x00, \ + D0, C0, 0x00 + // Update C0, C1 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + // Update X0, B1 + PTR_SUB X0, X0, K8 + PTR_SUB B1, B1, K8 + PTR_ALSL X0, LDB, X0, 1 + PTR_ALSL B1, LDB, B1, 1 + // Restore A0 + move A0, A + PTR_ADD A1, A0, LDA + PTR_ADD A2, A1, LDA + PTR_ADD A3, A2, LDA +.L_M4_N1: + andi N1, N, 0x01 + beqz N1, .L_M4_END + GXOR xv, v, D0, D0, D0 + move K1, K // Restore K1 + bge ZERO, K, .L_M4_N1_END +.L_M4_N1_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A1, 0x00, S2, A2, 0x00, S3, A3, 0x00 + GINSVE0 xv, d, S0, S1, 1, S0, S2, 2, S0, S3, 3 + GLDREPL xv, d, Z0, X0, 0x00 + GMADD xvf, d, D0, S0, Z0, D0 + PTR_ADDI X0, X0, 0x08 + PTR_ADDI A0, A0, 0x08 + PTR_ADDI A1, A1, 0x08 + PTR_ADDI A2, A2, 0x08 + PTR_ADDI A3, A3, 0x08 + bnez K1, .L_M4_N1_K1 +.L_M4_N1_END: + GMUL xvf, d, D0, D0, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0 +#endif + GST xv, , D0, C0, 0x00 + // Update C0 + PTR_ALSL C0, LDC, C0, 2 + // Update X0 + PTR_SUB X0, X0, K8 + PTR_ALSL X0, LDB, X0, 2 + // Restore A0 + move A0, A + PTR_ADD A1, A0, LDA + PTR_ADD A2, A1, LDA + PTR_ADD A3, A2, LDA +.L_M4_END: + PTR_ADDI M4, M4, -1 + PTR_ALSL A, LDA, A, 2 // A += LDA << 2; + PTR_ADDI C, C, 0x20 + bnez M4, .L_M4 +.L_M3: + andi M2, M, 0x02 + beqz M2, .L_M1 +.L_M2: + PTR_SRAI N4, N, 2 // N >> 2 + move A0, A // Restore A0 + PTR_ADD A1, A0, LDA + move X0, B // Restore X0 + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C // Restore C0 + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M2_N3 +.L_M2_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K // Restore K1 + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M2_N4_END +.L_M2_N4_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A1, 0x00 + GINSVE0 xv, d, S0, S1, 1 + GLDREPL xv, d, Z0, X0, 0x00, Z1, B1, 0x00, Z2, B2, 0x00, Z3, B3, 0x00 + GMADD xvf, d, D0, S0, Z0, D0, \ + D1, S0, Z1, D1, \ + D2, S0, Z2, D2, \ + D3, S0, Z3, D3 + PTR_ADDI X0, X0, 0x08 + PTR_ADDI B1, B1, 0x08 + PTR_ADDI B2, B2, 0x08 + PTR_ADDI B3, B3, 0x08 + PTR_ADDI A0, A0, 0x08 + PTR_ADDI A1, A1, 0x08 + bnez K1, .L_M2_N4_K1 + .L_M2_N4_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C1, 0x00, S2, C2, 0x00, S3, C3, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3 +#endif + GST v, , V3, C3, 0x00, \ + V2, C2, 0x00, \ + V1, C1, 0x00, \ + V0, C0, 0x00 + // Update C0, C1, C2, C3 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + // Update X0, B1, B2, B3 + PTR_SUB X0, X0, K8 + PTR_SUB B1, B1, K8 + PTR_SUB B2, B2, K8 + PTR_SUB B3, B3, K8 + + PTR_ALSL X0, LDB, X0, 2 + PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2 + PTR_ALSL B3, LDB, B3, 2 + // Restore A0, A1 + move A0, A + PTR_ADD A1, A0, LDA + bnez N4, .L_M2_N4 +.L_M2_N3: + andi N2, N, 0x02 + beqz N2, .L_M2_N1 +.L_M2_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1 + move K1, K // Restore K1 + bge ZERO, K, .L_M2_N2_END +.L_M2_N2_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A1, 0x00 + GINSVE0 xv, d, S0, S1, 1 + GLDREPL xv, d, Z0, X0, 0x00, Z1, B1, 0x00 + GMADD xvf, d, D0, S0, Z0, D0, \ + D1, S0, Z1, D1 + PTR_ADDI X0, X0, 0x08 + PTR_ADDI B1, B1, 0x08 + PTR_ADDI A0, A0, 0x08 + PTR_ADDI A1, A1, 0x08 + bnez K1, .L_M2_N2_K1 +.L_M2_N2_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C1, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1 +#endif + GST v, , V1, C1, 0x00, \ + V0, C0, 0x00 + // Update C0, C1 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + // Update X0, B1 + PTR_SUB X0, X0, K8 + PTR_SUB B1, B1, K8 + PTR_ALSL X0, LDB, X0, 1 + PTR_ALSL B1, LDB, B1, 1 + // Restore A0, A1 + move A0, A + PTR_ADD A1, A0, LDA +.L_M2_N1: + andi N1, N, 0x01 + beqz N1, .L_M2_END + GXOR xv, v, D0, D0, D0 + move K1, K // Restore K1 + bge ZERO, K, .L_M2_N1_END +.L_M2_N1_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A1, 0x00 + GINSVE0 xv, d, S0, S1, 1 + GLDREPL xv, d, Z0, X0, 0x00 + GMADD xvf, d, D0, S0, Z0, D0 + PTR_ADDI X0, X0, 0x08 + PTR_ADDI A0, A0, 0x08 + PTR_ADDI A1, A1, 0x08 + bnez K1, .L_M2_N1_K1 +.L_M2_N1_END: + GMUL xvf, d, D0, D0, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0 +#endif + GST v, , V0, C0, 0x00 + // Update C0 + PTR_ALSL C0, LDC, C0, 2 + // Update X0 + PTR_SUB X0, X0, K8 + PTR_ALSL X0, LDB, X0, 2 + // Restore A0, A1 + move A0, A + PTR_ADD A1, A0, LDA +.L_M2_END: + PTR_ALSL A, LDA, A, 1 // A += LDA << 1; + PTR_ADDI C, C, 0x10 +.L_M1: + andi M1, M, 0x01 + beqz M1, .L_M0 + + PTR_SRAI N4, N, 2 // N >> 2 + move A0, A // Restore A0 + move X0, B // Restore X0 + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C // Restore C0 + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M1_N3 +.L_M1_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K // Restore K1 + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M1_N4_END +.L_M1_N4_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00 + GLDREPL xv, d, Z0, X0, 0x00, Z1, B1, 0x00, Z2, B2, 0x00, Z3, B3, 0x00 + GMADD xvf, d, D0, S0, Z0, D0, \ + D1, S0, Z1, D1, \ + D2, S0, Z2, D2, \ + D3, S0, Z3, D3 + PTR_ADDI X0, X0, 0x08 + PTR_ADDI B1, B1, 0x08 + PTR_ADDI B2, B2, 0x08 + PTR_ADDI B3, B3, 0x08 + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_M1_N4_K1 + .L_M1_N4_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C1, 0x00, S2, C2, 0x00, S3, C3, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3 +#endif + GST f, d, F3, C3, 0x00, \ + F2, C2, 0x00, \ + F1, C1, 0x00, \ + F0, C0, 0x00 + // Update C0, C1, C2, C3 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + // Update X0, B1, B2, B3 + PTR_SUB X0, X0, K8 + PTR_SUB B1, B1, K8 + PTR_SUB B2, B2, K8 + PTR_SUB B3, B3, K8 + + PTR_ALSL X0, LDB, X0, 2 + PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2 + PTR_ALSL B3, LDB, B3, 2 + // Restore A0, A1 + move A0, A + bnez N4, .L_M1_N4 +.L_M1_N3: + andi N2, N, 0x02 + beqz N2, .L_M1_N1 +.L_M1_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1 + move K1, K // Restore K1 + bge ZERO, K, .L_M1_N2_END +.L_M1_N2_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00 + GLDREPL xv, d, Z0, X0, 0x00, Z1, B1, 0x00 + GMADD xvf, d, D0, S0, Z0, D0, \ + D1, S0, Z1, D1 + PTR_ADDI X0, X0, 0x08 + PTR_ADDI B1, B1, 0x08 + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_M1_N2_K1 +.L_M1_N2_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C1, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1 +#endif + GST f, d, F1, C1, 0x00, \ + F0, C0, 0x00 + // Update C0, C1 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + // Update X0, B1 + PTR_SUB X0, X0, K8 + PTR_SUB B1, B1, K8 + PTR_ALSL X0, LDB, X0, 1 + PTR_ALSL B1, LDB, B1, 1 + // Restore A0 + move A0, A +.L_M1_N1: + andi N1, N, 0x01 + beqz N1, .L_M0 + GXOR xv, v, D0, D0, D0 + move K1, K // Restore K1 + bge ZERO, K, .L_M1_N1_END +.L_M1_N1_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00 + GLDREPL xv, d, Z0, X0, 0x00 + GMADD xvf, d, D0, S0, Z0, D0 + PTR_ADDI X0, X0, 0x08 + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_M1_N1_K1 +.L_M1_N1_END: + GMUL xvf, d, D0, D0, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0 +#endif + GST f, d, F0, C0, 0x00 + // Update C0 + PTR_ALSL C0, LDC, C0, 2 + // Update X0 + PTR_SUB X0, X0, K8 + PTR_ALSL X0, LDB, X0, 2 + // Restore A0 + move A0, A +.L_M0: + pop_if_used 8, 2 + jirl $r0, $r1, 0x0 + EPILOGUE diff --git a/kernel/loongarch64/dgemm_small_kernel_tt_lasx.S b/kernel/loongarch64/dgemm_small_kernel_tt_lasx.S new file mode 100644 index 0000000000..b3e338514e --- /dev/null +++ b/kernel/loongarch64/dgemm_small_kernel_tt_lasx.S @@ -0,0 +1,534 @@ +/*************************************************************************** +Copyright (c) 2024, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#define ASSEMBLER + +#include "common.h" +#include "loongarch64_asm.S" + +#define M $a0 +#define N $a1 +#define K $a2 +#define A $a3 +#define LDA $a4 +#define ALPHA $f0 +#define B $a5 +#define LDB $a6 +#define C $a7 +#define LDC $t0 +#ifdef B0 +#define BETA $f1 +#endif +#undef ZERO +#define ZERO $r0 + +#define M4 $t1 +#define M2 $t1 +#define M1 $t1 +#define N4 $t2 +#define N2 $t2 +#define N1 $t2 +#define K_LDB $t3 +#define A0 $t4 +#define X0 $t5 +#define A1 $t6 +#define A2 $t7 +#define A3 $t8 +#define C0 $s0 +#define C1 $s1 +#define C2 $s2 +#define C3 $s3 +#define K1 $s4 +#define B1 $s5 +#define B2 $s6 +#define B3 $s7 + +#define VALPHA $xr0 +#ifndef B0 +#define VBETA $xr1 +#endif +#define D0 $xr2 +#define D1 $xr3 +#define D2 $xr4 +#define D3 $xr5 +#define T0 $xr6 +#define T1 $xr7 +#define T2 $xr8 +#define T3 $xr9 +#define Y0 $xr10 +#define Y1 $xr11 +#define Y2 $xr12 +#define Y3 $xr13 +#define G0 $xr14 +#define G1 $xr15 +#define G2 $xr16 +#define G3 $xr17 +#define S0 $xr18 +#define S1 $xr19 +#define S2 $xr20 +#define S3 $xr21 +#define Z0 $xr22 +#define Z1 $xr23 +#define Z2 $xr24 +#define Z3 $xr25 +#define V0 $vr2 +#define V1 $vr3 +#define V2 $vr4 +#define V3 $vr5 +#define F0 $f2 +#define F1 $f3 +#define F2 $f4 +#define F3 $f5 + + PROLOGUE + PTR_LD LDC, $sp, 0 + push_if_used 8, 2 + xvreplve0.d VALPHA, VALPHA +#ifndef B0 + xvreplve0.d VBETA, VBETA +#endif + PTR_SLLI LDA, LDA, 3 + PTR_SLLI LDB, LDB, 3 + PTR_SLLI LDC, LDC, 3 + PTR_MUL K_LDB, K, LDB + PTR_SRAI M4, M, 2 // M >> 2 + beqz M4, .L_M3 +.L_M4: + PTR_SRAI N4, N, 2 // N >> 2 + move A0, A // Restore A0 + PTR_ADD A1, A0, LDA + PTR_ADD A2, A1, LDA + PTR_ADD A3, A2, LDA + move X0, B // Restore X0 + move C0, C // Restore C0 + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M4_N3 +.L_M4_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K // Restore K1 + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M4_N4_END + PTR_SRAI K1, K1, 2 + beq ZERO, K1, .L_M4_N4_K3 + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB +.L_M4_N4_K4: + PTR_ADDI K1, K1, -1 + GLD xv, , T0, A0, 0x00, T1, A1, 0x00, T2, A2, 0x00, T3, A3, 0x00 + GTRANSPOSE4x4_D T0, T1, T2, T3, S0, S1, S2, S3, Z0, Z1 + GLDREPL xv, d, Z0, X0, 0x00, Z1, B1, 0x00, Z2, B2, 0x00, Z3, B3, 0x00 + GLDREPL xv, d, T0, X0, 0x08, T1, B1, 0x08, T2, B2, 0x08, T3, B3, 0x08 + GLDREPL xv, d, Y0, X0, 0x10, Y1, B1, 0x10, Y2, B2, 0x10, Y3, B3, 0x10 + GLDREPL xv, d, G0, X0, 0x18, G1, B1, 0x18, G2, B2, 0x18, G3, B3, 0x18 + GMADD xvf, d, D0, S0, Z0, D0, \ + D1, S0, T0, D1, \ + D2, S0, Y0, D2, \ + D3, S0, G0, D3 + GMADD xvf, d, D0, S1, Z1, D0, \ + D1, S1, T1, D1, \ + D2, S1, Y1, D2, \ + D3, S1, G1, D3 + GMADD xvf, d, D0, S2, Z2, D0, \ + D1, S2, T2, D1, \ + D2, S2, Y2, D2, \ + D3, S2, G2, D3 + GMADD xvf, d, D0, S3, Z3, D0, \ + D1, S3, T3, D1, \ + D2, S3, Y3, D2, \ + D3, S3, G3, D3 + PTR_ALSL X0, LDB, X0, 2 + PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2 + PTR_ALSL B3, LDB, B3, 2 + + PTR_ADDI A0, A0, 0x20 + PTR_ADDI A1, A1, 0x20 + PTR_ADDI A2, A2, 0x20 + PTR_ADDI A3, A3, 0x20 + bnez K1, .L_M4_N4_K4 +.L_M4_N4_K3: + andi K1, K, 3 + beqz K1, .L_M4_N4_END +.L_M4_N4_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A1, 0x00, S2, A2, 0x00, S3, A3, 0x00 + GINSVE0 xv, d, S0, S1, 1, S0, S2, 2, S0, S3, 3 + GLDREPL xv, d, Z0, X0, 0x00, Z1, X0, 0x08, Z2, X0, 0x10, Z3, X0, 0x18 + GMADD xvf, d, D0, S0, Z0, D0, \ + D1, S0, Z1, D1, \ + D2, S0, Z2, D2, \ + D3, S0, Z3, D3 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x08 + PTR_ADDI A1, A1, 0x08 + PTR_ADDI A2, A2, 0x08 + PTR_ADDI A3, A3, 0x08 + bnez K1, .L_M4_N4_K1 + .L_M4_N4_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C1, 0x00, S2, C2, 0x00, S3, C3, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0, \ + D1, S1, VBETA, D1, \ + D2, S2, VBETA, D2, \ + D3, S3, VBETA, D3 +#endif + GST xv, , D3, C3, 0x00, \ + D2, C2, 0x00, \ + D1, C1, 0x00, \ + D0, C0, 0x00 + // Update C0, C1, C2, C3 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + // Update X0 + PTR_SUB X0, X0, K_LDB + PTR_ADDI X0, X0, 0x20 + // Restore A0, A1, A2, A3 + move A0, A + PTR_ADD A1, A0, LDA + PTR_ADD A2, A1, LDA + PTR_ADD A3, A2, LDA + bnez N4, .L_M4_N4 +.L_M4_N3: + andi N2, N, 0x02 + beqz N2, .L_M4_N1 +.L_M4_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1 + move K1, K // Restore K1 + bge ZERO, K, .L_M4_N2_END +.L_M4_N2_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A1, 0x00, S2, A2, 0x00, S3, A3, 0x00 + GINSVE0 xv, d, S0, S1, 1, S0, S2, 2, S0, S3, 3 + GLDREPL xv, d, Z0, X0, 0x00, Z1, X0, 0x08 + GMADD xvf, d, D0, S0, Z0, D0, \ + D1, S0, Z1, D1 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x08 + PTR_ADDI A1, A1, 0x08 + PTR_ADDI A2, A2, 0x08 + PTR_ADDI A3, A3, 0x08 + bnez K1, .L_M4_N2_K1 +.L_M4_N2_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C1, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1 +#endif + GST xv, , D1, C1, 0x00, \ + D0, C0, 0x00 + // Update C0, C1 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + // Update X0 + PTR_SUB X0, X0, K_LDB + PTR_ADDI X0, X0, 0x10 + // Restore A0 + move A0, A + PTR_ADD A1, A0, LDA + PTR_ADD A2, A1, LDA + PTR_ADD A3, A2, LDA +.L_M4_N1: + andi N1, N, 0x01 + beqz N1, .L_M4_END + GXOR xv, v, D0, D0, D0 + move K1, K // Restore K1 + bge ZERO, K, .L_M4_N1_END +.L_M4_N1_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A1, 0x00, S2, A2, 0x00, S3, A3, 0x00 + GINSVE0 xv, d, S0, S1, 1, S0, S2, 2, S0, S3, 3 + GLDREPL xv, d, Z0, X0, 0x00 + GMADD xvf, d, D0, S0, Z0, D0 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x08 + PTR_ADDI A1, A1, 0x08 + PTR_ADDI A2, A2, 0x08 + PTR_ADDI A3, A3, 0x08 + bnez K1, .L_M4_N1_K1 +.L_M4_N1_END: + GMUL xvf, d, D0, D0, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0 +#endif + GST xv, , D0, C0, 0x00 + // Update C0 + PTR_ALSL C0, LDC, C0, 2 + // Update X0 + PTR_SUB X0, X0, K_LDB + PTR_ADDI X0, X0, 0x08 + // Restore A0 + move A0, A + PTR_ADD A1, A0, LDA + PTR_ADD A2, A1, LDA + PTR_ADD A3, A2, LDA +.L_M4_END: + PTR_ADDI M4, M4, -1 + PTR_ALSL A, LDA, A, 2 // A += LDA << 2; + PTR_ADDI C, C, 0x20 + bnez M4, .L_M4 +.L_M3: + andi M2, M, 0x02 + beqz M2, .L_M1 +.L_M2: + PTR_SRAI N4, N, 2 // N >> 2 + move A0, A // Restore A0 + PTR_ADD A1, A0, LDA + move X0, B // Restore X0 + move C0, C // Restore C0 + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M2_N3 +.L_M2_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K // Restore K1 + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M2_N4_END +.L_M2_N4_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A1, 0x00 + GINSVE0 xv, d, S0, S1, 1 + GLDREPL xv, d, Z0, X0, 0x00, Z1, X0, 0x08, Z2, X0, 0x10, Z3, X0, 0x18 + GMADD xvf, d, D0, S0, Z0, D0, \ + D1, S0, Z1, D1, \ + D2, S0, Z2, D2, \ + D3, S0, Z3, D3 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x08 + PTR_ADDI A1, A1, 0x08 + bnez K1, .L_M2_N4_K1 + .L_M2_N4_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C1, 0x00, S2, C2, 0x00, S3, C3, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3 +#endif + GST v, , V3, C3, 0x00, \ + V2, C2, 0x00, \ + V1, C1, 0x00, \ + V0, C0, 0x00 + // Update C0, C1, C2, C3 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + // Update X0 + PTR_SUB X0, X0, K_LDB + PTR_ADDI X0, X0, 0x20 + // Restore A0, A1 + move A0, A + PTR_ADD A1, A0, LDA + bnez N4, .L_M2_N4 +.L_M2_N3: + andi N2, N, 0x02 + beqz N2, .L_M2_N1 +.L_M2_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1 + move K1, K // Restore K1 + bge ZERO, K, .L_M2_N2_END +.L_M2_N2_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A1, 0x00 + GINSVE0 xv, d, S0, S1, 1 + GLDREPL xv, d, Z0, X0, 0x00, Z1, X0, 0x08 + GMADD xvf, d, D0, S0, Z0, D0, \ + D1, S0, Z1, D1 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x08 + PTR_ADDI A1, A1, 0x08 + bnez K1, .L_M2_N2_K1 +.L_M2_N2_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C1, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1 +#endif + GST v, , V1, C1, 0x00, \ + V0, C0, 0x00 + // Update C0, C1 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + // Update X0 + PTR_SUB X0, X0, K_LDB + PTR_ADDI X0, X0, 0x10 + // Restore A0, A1 + move A0, A + PTR_ADD A1, A0, LDA +.L_M2_N1: + andi N1, N, 0x01 + beqz N1, .L_M2_END + GXOR xv, v, D0, D0, D0 + move K1, K // Restore K1 + bge ZERO, K, .L_M2_N1_END +.L_M2_N1_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A1, 0x00 + GINSVE0 xv, d, S0, S1, 1 + GLDREPL xv, d, Z0, X0, 0x00 + GMADD xvf, d, D0, S0, Z0, D0 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x08 + PTR_ADDI A1, A1, 0x08 + bnez K1, .L_M2_N1_K1 +.L_M2_N1_END: + GMUL xvf, d, D0, D0, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0 +#endif + GST v, , V0, C0, 0x00 + // Update C0 + PTR_ALSL C0, LDC, C0, 2 + // Update X0 + PTR_SUB X0, X0, K_LDB + PTR_ADDI X0, X0, 0x08 + // Restore A0, A1 + move A0, A + PTR_ADD A1, A0, LDA +.L_M2_END: + PTR_ALSL A, LDA, A, 1 // A += LDA << 1; + PTR_ADDI C, C, 0x10 +.L_M1: + andi M1, M, 0x01 + beqz M1, .L_M0 + + PTR_SRAI N4, N, 2 // N >> 2 + move A0, A // Restore A0 + move X0, B // Restore X0 + move C0, C // Restore C0 + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M1_N3 +.L_M1_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K // Restore K1 + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M1_N4_END +.L_M1_N4_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00 + GLDREPL xv, d, Z0, X0, 0x00, Z1, X0, 0x08, Z2, X0, 0x10, Z3, X0, 0x18 + GMADD xvf, d, D0, S0, Z0, D0, \ + D1, S0, Z1, D1, \ + D2, S0, Z2, D2, \ + D3, S0, Z3, D3 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_M1_N4_K1 + .L_M1_N4_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C1, 0x00, S2, C2, 0x00, S3, C3, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3 +#endif + GST f, d, F3, C3, 0x00, \ + F2, C2, 0x00, \ + F1, C1, 0x00, \ + F0, C0, 0x00 + // Update C0, C1, C2, C3 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + // Update X0 + PTR_SUB X0, X0, K_LDB + PTR_ADDI X0, X0, 0x20 + // Restore A0, A1 + move A0, A + bnez N4, .L_M1_N4 +.L_M1_N3: + andi N2, N, 0x02 + beqz N2, .L_M1_N1 +.L_M1_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1 + move K1, K // Restore K1 + bge ZERO, K, .L_M1_N2_END +.L_M1_N2_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00 + GLDREPL xv, d, Z0, X0, 0x00, Z1, X0, 0x08 + GMADD xvf, d, D0, S0, Z0, D0, \ + D1, S0, Z1, D1 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_M1_N2_K1 +.L_M1_N2_END: + GMUL xvf, d, D0, D0, VALPHA, D1, D1, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C1, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1 +#endif + GST f, d, F1, C1, 0x00, \ + F0, C0, 0x00 + // Update C0, C1 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + // Update X0 + PTR_SUB X0, X0, K_LDB + PTR_ADDI X0, X0, 0x10 + // Restore A0 + move A0, A +.L_M1_N1: + andi N1, N, 0x01 + beqz N1, .L_M0 + GXOR xv, v, D0, D0, D0 + move K1, K // Restore K1 + bge ZERO, K, .L_M1_N1_END +.L_M1_N1_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00 + GLDREPL xv, d, Z0, X0, 0x00 + GMADD xvf, d, D0, S0, Z0, D0 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_M1_N1_K1 +.L_M1_N1_END: + GMUL xvf, d, D0, D0, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, d, D0, S0, VBETA, D0 +#endif + GST f, d, F0, C0, 0x00 + // Update C0 + PTR_ALSL C0, LDC, C0, 2 + // Update X0 + PTR_SUB X0, X0, K_LDB + PTR_ADDI X0, X0, 0x08 + // Restore A0 + move A0, A +.L_M0: + pop_if_used 8, 2 + jirl $r0, $r1, 0x0 + EPILOGUE diff --git a/kernel/loongarch64/dgemm_small_matrix_permit.c b/kernel/loongarch64/dgemm_small_matrix_permit.c new file mode 100644 index 0000000000..df262a6bb3 --- /dev/null +++ b/kernel/loongarch64/dgemm_small_matrix_permit.c @@ -0,0 +1,44 @@ +/*************************************************************************** +Copyright (c) 2024, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "common.h" + +int CNAME(int transa, int transb, BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, FLOAT beta) +{ + double MNK = (double) M * (double) N * (double) K; + + if (transa) { + if (MNK <= 24.0 * 24.0 * 24.0) + return 1; + } else { + if (MNK <= 64.0 * 64.0 * 64.0) + return 1; + } + + return 0; +} + From 4e9144b39faa6b0542e1b7c183e0dab1a8637a07 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 5 Jun 2024 23:43:52 +0200 Subject: [PATCH 119/267] Update .cirrus.yml (#4735) * Update versions (and paths) of XCode, and update FreeBSD version --- .cirrus.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index e8a0b85c02..d0e1eeff71 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -41,7 +41,7 @@ macos_instance: # - make CC=gcc-11 FC=gfortran-11 USE_OPENMP=1 macos_instance: - image: ghcr.io/cirruslabs/macos-monterey-xcode:latest + image: ghcr.io/cirruslabs/macos-sonoma-xcode:latest task: name: AppleM1/LLVM x86_64 xbuild compile_script: @@ -58,8 +58,8 @@ task: - export VALID_ARCHS="i386 x86_64" - xcrun --sdk macosx --show-sdk-path - xcodebuild -version - - export CC=/Applications/Xcode-15.3.0.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang - - export CFLAGS="-O2 -unwindlib=none -Wno-macro-redefined -isysroot /Applications/Xcode-15.3.0.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX14.4.sdk -arch x86_64" + - export CC=/Applications/Xcode_15.4.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang + - export CFLAGS="-O2 -unwindlib=none -Wno-macro-redefined -isysroot /Applications/Xcode_15.4.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX14.5.sdk -arch x86_64" - make TARGET=CORE2 DYNAMIC_ARCH=1 NUM_THREADS=32 HOSTCC=clang NOFORTRAN=1 RANLIB="ls -l" always: config_artifacts: @@ -70,7 +70,7 @@ task: # type: application/octet-streamm macos_instance: - image: ghcr.io/cirruslabs/macos-monterey-xcode:latest + image: ghcr.io/cirruslabs/macos-sonoma-xcode:latest task: name: AppleM1/LLVM armv8-ios xbuild compile_script: @@ -78,8 +78,10 @@ task: - export #PATH=/opt/homebrew/opt/llvm/bin:$PATH - export #LDFLAGS="-L/opt/homebrew/opt/llvm/lib" - export #CPPFLAGS="-I/opt/homebrew/opt/llvm/include" - - export CC=/Applications/Xcode-15.3.0.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang - - export CFLAGS="-O2 -unwindlib=none -Wno-macro-redefined -isysroot /Applications/Xcode-15.3.0.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS17.4.sdk -arch arm64 -miphoneos-version-min=10.0" + - export CC=/Applications/Xcode_15.4.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang + - export CFLAGS="-O2 -unwindlib=none -Wno-macro-redefined -isysroot /Applications/Xcode_15.4.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS17.5.sdk -arch arm64 -miphoneos-version-min=10.0" + - xcrun --sdk iphoneos --show-sdk-path + - ls -l /Applications - make TARGET=ARMV8 NUM_THREADS=32 HOSTCC=clang NOFORTRAN=1 CROSS=1 always: config_artifacts: @@ -96,11 +98,11 @@ task: - export #LDFLAGS="-L/opt/homebrew/opt/llvm/lib" - export #CPPFLAGS="-I/opt/homebrew/opt/llvm/include" - ls /System/Volumes/Data/opt/homebrew - - ls -l /System/Volumes/Data/opt/homebrew/Caskroom/ + - ls -l /System/Volumes/Data/opt/homebrew/Caskroom/android-ndk - find /System/Volumes/Data/opt/homebrew -name "armv7a-linux-androideabi*-ranlib" - #export CC=/Applications/Xcode-13.4.1.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang - #export CFLAGS="-O2 -unwindlib=none -Wno-macro-redefined -isysroot /Applications/Xcode-13.4.1.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS16.0.sdk -arch arm64 -miphoneos-version-min=10.0" - - export CC=/System/Volumes/Data/opt/homebrew/Caskroom/android-ndk/26c/AndroidNDK*.app/Contents/NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/armv7a-linux-androideabi23-clang + - export CC=/System/Volumes/Data/opt/homebrew/Caskroom/android-ndk/26d/AndroidNDK*.app/Contents/NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/armv7a-linux-androideabi23-clang - make TARGET=ARMV7 ARM_SOFTFP_ABI=1 NUM_THREADS=32 HOSTCC=clang NOFORTRAN=1 RANLIB="ls -l" always: config_artifacts: @@ -132,7 +134,7 @@ task: FreeBSD_task: name: FreeBSD-gcc12 freebsd_instance: - image_family: freebsd-13-2 + image_family: freebsd-13-3 install_script: - pkg update -f && pkg upgrade -y && pkg install -y gmake gcc compile_script: @@ -143,7 +145,7 @@ FreeBSD_task: FreeBSD_task: name: freebsd-gcc12-ilp64 freebsd_instance: - image_family: freebsd-13-2 + image_family: freebsd-13-3 install_script: - pkg update -f && pkg upgrade -y && pkg install -y gmake gcc compile_script: @@ -153,7 +155,7 @@ FreeBSD_task: FreeBSD_task: name: FreeBSD-clang-openmp freebsd_instance: - image_family: freebsd-13-2 + image_family: freebsd-13-3 install_script: - pkg update -f && pkg upgrade -y && pkg install -y gmake gcc - ln -s /usr/local/lib/gcc13/libgfortran.so.5.0.0 /usr/lib/libgfortran.so From af73ae6208b3d8ecbdb37bca07ac63734f847c0f Mon Sep 17 00:00:00 2001 From: gxw Date: Thu, 6 Jun 2024 16:43:09 +0800 Subject: [PATCH 120/267] LoongArch: Fixed issue 4728 --- kernel/loongarch64/scal.S | 74 -------------------------------- kernel/loongarch64/scal_lasx.S | 75 ++------------------------------ kernel/loongarch64/scal_lsx.S | 78 +--------------------------------- 3 files changed, 5 insertions(+), 222 deletions(-) diff --git a/kernel/loongarch64/scal.S b/kernel/loongarch64/scal.S index 566bce6cbe..8de710f413 100644 --- a/kernel/loongarch64/scal.S +++ b/kernel/loongarch64/scal.S @@ -56,80 +56,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. MTC a1, $r0 slli.d INCX, INCX, BASE_SHIFT bge $r0, N, .L999 - CMPEQ $fcc0, ALPHA, a1 - bceqz $fcc0, .L50 - srai.d I, N, 3 - bne INCX, TEMP, .L20 - bge $r0, I, .L15 - .align 3 - -.L12: - ST a1, X, 0 * SIZE - ST a1, X, 1 * SIZE - ST a1, X, 2 * SIZE - ST a1, X, 3 * SIZE - ST a1, X, 4 * SIZE - ST a1, X, 5 * SIZE - ST a1, X, 6 * SIZE - ST a1, X, 7 * SIZE - addi.w I, I, -1 - addi.d X, X, 8 * SIZE - blt $r0, I, .L12 - .align 3 - -.L15: - andi I, N, 7 - bge $r0, I, .L999 - .align 3 -.L16: - ST a1, X, 0 * SIZE - addi.d I, I, -1 - addi.d X, X, SIZE - blt $r0, I, .L16 - move $r4, $r17 - fmov.d $f0, $f22 - jirl $r0, $r1, 0x0 - .align 3 - -.L20: - srai.d I, N, 3 - bge $r0, I, .L25 - .align 3 - -.L22: - ST a1, X, 0 * SIZE - add.d X, X, INCX - ST a1, X, 0 * SIZE - add.d X, X, INCX - ST a1, X, 0 * SIZE - add.d X, X, INCX - ST a1, X, 0 * SIZE - add.d X, X, INCX - ST a1, X, 0 * SIZE - add.d X, X, INCX - ST a1, X, 0 * SIZE - add.d X, X, INCX - ST a1, X, 0 * SIZE - add.d X, X, INCX - ST a1, X, 0 * SIZE - addi.d I, I, -1 - add.d X, X, INCX - blt $r0, I, .L22 - .align 3 - -.L25: - andi I, N, 7 - bge $r0, I, .L999 - .align 3 -.L26: - addi.d I, I, -1 - ST a1, X, 0 * SIZE - add.d X, X, INCX - blt $r0, I, .L26 - move $r4, $r17 - fmov.d $f0, $f22 - jirl $r0, $r1, 0x0 - .align 3 .L50: srai.d I, N, 3 diff --git a/kernel/loongarch64/scal_lasx.S b/kernel/loongarch64/scal_lasx.S index 48e2c0718d..b4585c1b96 100644 --- a/kernel/loongarch64/scal_lasx.S +++ b/kernel/loongarch64/scal_lasx.S @@ -58,12 +58,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. FFINT a2, a2 slli.d TEMP, TEMP, BASE_SHIFT slli.d INCX, INCX, BASE_SHIFT - CMPEQ $fcc0, ALPHA, a1 - bcnez $fcc0, .L20 //ALPHA==0 CMPEQ $fcc0, ALPHA, a2 bcnez $fcc0, .L999 //ALPHA==1 return + srai.d I, N, 3 - beq INCX, TEMP, .L30 //ALPHA！=0|1 and INCX==1 + beq INCX, TEMP, .L30 //ALPHA!=1 and INCX==1 MTG TEMP, ALPHA #ifdef DOUBLE xvreplgr2vr.d VALPHA, TEMP @@ -73,7 +72,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. move XX, X .align 3 -.L10: //ALPHA！=0|1 and INCX!=1 +.L10: //ALPHA!=1 and INCX!=1 bge $r0, I, .L32 .align 3 .L11: @@ -166,74 +165,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. blt $r0, I, .L11 b .L32 .align 3 - -.L20: - srai.d I, N, 3 - beq INCX, TEMP, .L24 - bge $r0, I, .L22 - .align 3 - -.L21: - ST a1, X, 0 - add.d X, X, INCX - ST a1, X, 0 - add.d X, X, INCX - ST a1, X, 0 - add.d X, X, INCX - ST a1, X, 0 - add.d X, X, INCX - ST a1, X, 0 - add.d X, X, INCX - ST a1, X, 0 - add.d X, X, INCX - ST a1, X, 0 - add.d X, X, INCX - ST a1, X, 0 - add.d X, X, INCX - addi.d I, I, -1 - blt $r0, I, .L21 - .align 3 - -.L22: - andi I, N, 7 - bge $r0, I, .L999 - .align 3 - -.L23: - ST a1, X, 0 * SIZE - addi.d I, I, -1 - add.d X, X, INCX - blt $r0, I, .L23 - jirl $r0, $r1, 0 - .align 3 - -.L24: - bge $r0, I, .L26 /*N<8 INCX==1*/ - .align 3 -.L25: - xvxor.v VX0, VX0, VX0 - xvst VX0, X, 0 * SIZE -#ifdef DOUBLE - xvst VX0, X, 4 * SIZE -#endif - addi.d I, I, -1 - addi.d X, X, 8 * SIZE - blt $r0, I, .L25 - .align 3 - -.L26: - andi I, N, 7 - bge $r0, I, .L999 - .align 3 - -.L27: - ST a1, X, 0 * SIZE - addi.d I, I, -1 - addi.d X, X, SIZE - blt $r0, I, .L27 - jirl $r0, $r1, 0 - .align 3 - .L30: bge $r0, I, .L32/*N<8 INCX==1*/ MTG TEMP, ALPHA diff --git a/kernel/loongarch64/scal_lsx.S b/kernel/loongarch64/scal_lsx.S index 1ffce7db27..a27e050ed8 100644 --- a/kernel/loongarch64/scal_lsx.S +++ b/kernel/loongarch64/scal_lsx.S @@ -58,12 +58,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. FFINT a2, a2 slli.d TEMP, TEMP, BASE_SHIFT slli.d INCX, INCX, BASE_SHIFT - CMPEQ $fcc0, ALPHA, a1 - bcnez $fcc0, .L20 //ALPHA==0 CMPEQ $fcc0, ALPHA, a2 bcnez $fcc0, .L999 //ALPHA==1 return srai.d I, N, 3 - beq INCX, TEMP, .L30 //ALPHA！=0|1 and INCX==1 + beq INCX, TEMP, .L30 //ALPHA!=1 and INCX==1 MTG TEMP, ALPHA #ifdef DOUBLE vreplgr2vr.d VALPHA, TEMP @@ -73,7 +71,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. move XX, X .align 3 -.L10: //ALPHA！=0|1 and INCX!=1 +.L10: //ALPHA!=1 and INCX!=1 bge $r0, I, .L32 .align 3 @@ -171,78 +169,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. b .L32 .align 3 -.L20: - srai.d I, N, 3 - beq INCX, TEMP, .L24 - bge $r0, I, .L22 - .align 3 - -.L21: - ST a1, X, 0 - add.d X, X, INCX - ST a1, X, 0 - add.d X, X, INCX - ST a1, X, 0 - add.d X, X, INCX - ST a1, X, 0 - add.d X, X, INCX - ST a1, X, 0 - add.d X, X, INCX - ST a1, X, 0 - add.d X, X, INCX - ST a1, X, 0 - add.d X, X, INCX - ST a1, X, 0 - add.d X, X, INCX - addi.d I, I, -1 - blt $r0, I, .L21 - .align 3 - -.L22: - andi I, N, 7 - bge $r0, I, .L999 - .align 3 - -.L23: - ST a1, X, 0 * SIZE - addi.d I, I, -1 - add.d X, X, INCX - blt $r0, I, .L23 - jirl $r0, $r1, 0 - .align 3 - -.L24: - bge $r0, I, .L26 /*N<8 INCX==1*/ - .align 3 - -.L25: - vxor.v VX0, VX0, VX0 - vst VX0, X, 0 * SIZE -#ifdef DOUBLE - vst VX0, X, 2 * SIZE - vst VX0, X, 4 * SIZE - vst VX0, X, 6 * SIZE -#else - vst VX0, X, 4 * SIZE -#endif - addi.d I, I, -1 - addi.d X, X, 8 * SIZE - blt $r0, I, .L25 - .align 3 - -.L26: - andi I, N, 7 - bge $r0, I, .L999 - .align 3 - -.L27: - ST a1, X, 0 * SIZE - addi.d I, I, -1 - addi.d X, X, SIZE - blt $r0, I, .L27 - jirl $r0, $r1, 0 - .align 3 - .L30: bge $r0, I, .L32/*N<8 INCX==1*/ MTG TEMP, ALPHA From 2787c9f8e4ee356dfff7f31c7bcb718c01552edc Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 6 Jun 2024 14:39:50 +0200 Subject: [PATCH 121/267] Disable GEMM3M for generic targets (not implemented) --- kernel/Makefile.L3 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/kernel/Makefile.L3 b/kernel/Makefile.L3 index 863f376e9e..87fd5ca100 100644 --- a/kernel/Makefile.L3 +++ b/kernel/Makefile.L3 @@ -17,6 +17,16 @@ ifeq ($(ARCH), ia64) USE_GEMM3M = 1 endif +ifneq ($(DYNAMIC_ARCH), 1) +ifeq ($(TARGET), GENERIC) +USE_GEMM3M = 0 +endif +else +ifeq ($(CORE), GENERIC) +USE_GEMM3M = 0 +endif +endif + ifeq ($(ARCH), arm) USE_TRMM = 1 endif From f96ee867112ed5fe053fd352166b3d78a88cf996 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 6 Jun 2024 21:17:36 +0200 Subject: [PATCH 122/267] remove .mod files during make clean --- lapack-netlib/SRC/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lapack-netlib/SRC/Makefile b/lapack-netlib/SRC/Makefile index de22427016..2a20fcdab4 100644 --- a/lapack-netlib/SRC/Makefile +++ b/lapack-netlib/SRC/Makefile @@ -668,7 +668,7 @@ FRC: .PHONY: clean cleanobj cleanlib clean: cleanobj cleanlib cleanobj: - rm -f *.o DEPRECATED/*.o + rm -f *.o *.mod DEPRECATED/*.o DEPRECATED/*.mod cleanlib: rm -f $(LAPACKLIB) From ffc1ab3f6eae71346f5e25cd9a50953f299e9203 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 6 Jun 2024 23:58:16 +0200 Subject: [PATCH 123/267] Test corner cases of all SCAL variants --- utest/test_zscal.c | 448 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 446 insertions(+), 2 deletions(-) diff --git a/utest/test_zscal.c b/utest/test_zscal.c index 22642630c7..09e63752c2 100644 --- a/utest/test_zscal.c +++ b/utest/test_zscal.c @@ -1,5 +1,449 @@ #include "openblas_utest.h" #include +#ifdef BUILD_SINGLE + +#ifndef NAN +#define NAN 0.0/0.0 +#endif +#ifndef INFINITY +#define INFINITY 1.0/0.0 +#endif + +CTEST(sscal, 0_nan) +{ + blasint N=9; + blasint incX=1; + float i = 0.0; + float x[] = {NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN}; + BLASFUNC(sscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +CTEST(sscal, 0_nan_inc_2) +{ + blasint N=9; + blasint incX=2; + float i = 0.0; + float x[] = {NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, + NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN}; + BLASFUNC(sscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +CTEST(sscal, nan_0) +{ + blasint N=9; + blasint incX=1; + float i = NAN; + float x[] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + BLASFUNC(sscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +CTEST(sscal, nan_0_inc_2) +{ + blasint N=9; + blasint incX=2; + float i = NAN; + float x[] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + BLASFUNC(sscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +CTEST(sscal, 0_inf) +{ + blasint N=9; + blasint incX=1; + float i = 0.0; + float x[] = {INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY}; + BLASFUNC(sscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +CTEST(sscal, 0_inf_inc_2) +{ + blasint N=9; + blasint incX=2; + float i = 0.0; + float x[] = {INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, + INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY}; + BLASFUNC(sscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +CTEST(sscal, inf_0) +{ + blasint N=9; + blasint incX=1; + float i = INFINITY; + float x[] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + BLASFUNC(sscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +CTEST(sscal, inf_0_inc_2) +{ + blasint N=9; + blasint incX=2; + float i = INFINITY; + float x[] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + BLASFUNC(sscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +CTEST(sscal, nan_inf) +{ + blasint N=9; + blasint incX=1; + float i = NAN; + float x[] = {INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY}; + BLASFUNC(sscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +CTEST(sscal, nan_inf_inc_2) +{ + blasint N=9; + blasint incX=2; + float i = NAN; + float x[] = {INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, + INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY}; + BLASFUNC(sscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +CTEST(sscal, inf_nan) +{ + blasint N=9; + blasint incX=1; + float i = INFINITY; + float x[] = {NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN}; + BLASFUNC(sscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +CTEST(sscal, inf_nan_inc_2) +{ + blasint N=9; + blasint incX=2; + float i = INFINITY; + float x[] = {NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, + NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN}; + BLASFUNC(sscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +#endif + +#ifdef BUILD_DOUBLE + +#ifndef NAN +#define NAN 0.0/0.0 +#endif +#ifndef INFINITY +#define INFINITY 1.0/0.0 +#endif + +CTEST(dscal, 0_nan) +{ + blasint N=9; + blasint incX=1; + double i = 0.0; + double x[] = {NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN}; + BLASFUNC(dscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +CTEST(dscal, 0_nan_inc_2) +{ + blasint N=9; + blasint incX=2; + double i = 0.0; + double x[] = {NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, + NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN}; + BLASFUNC(dscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +CTEST(dscal, nan_0) +{ + blasint N=9; + blasint incX=1; + double i = NAN; + double x[] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + BLASFUNC(dscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +CTEST(dscal, nan_0_inc_2) +{ + blasint N=9; + blasint incX=2; + double i = NAN; + double x[] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + BLASFUNC(dscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +CTEST(dscal, 0_inf) +{ + blasint N=9; + blasint incX=1; + double i = 0.0; + double x[] = {INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY}; + BLASFUNC(dscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +CTEST(dscal, 0_inf_inc_2) +{ + blasint N=9; + blasint incX=2; + double i = 0.0; + double x[] = {INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, + INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY}; + BLASFUNC(dscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +CTEST(dscal, inf_0) +{ + blasint N=9; + blasint incX=1; + double i = INFINITY; + double x[] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + BLASFUNC(dscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +CTEST(dscal, inf_0_inc_2) +{ + blasint N=9; + blasint incX=2; + double i = INFINITY; + double x[] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + BLASFUNC(dscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +CTEST(dscal, nan_inf) +{ + blasint N=9; + blasint incX=1; + double i = NAN; + double x[] = {INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY}; + BLASFUNC(dscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +CTEST(dscal, nan_inf_inc_2) +{ + blasint N=9; + blasint incX=2; + double i = NAN; + double x[] = {INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, + INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY, INFINITY}; + BLASFUNC(dscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +CTEST(dscal, inf_nan) +{ + blasint N=9; + blasint incX=1; + double i = INFINITY; + double x[] = {NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN}; + BLASFUNC(dscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +CTEST(dscal, inf_nan_inc_2) +{ + blasint N=9; + blasint incX=2; + double i = INFINITY; + double x[] = {NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, + NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN, NAN}; + BLASFUNC(dscal)(&N, &i, x, &incX); + ASSERT_TRUE(isnan(x[0])); + ASSERT_TRUE(isnan(x[8])); +} + +#endif + +#ifdef BUILD_COMPLEX + +#ifndef NAN +#define NAN 0.0/0.0 +#endif +#ifndef INFINITY +#define INFINITY 1.0/0.0 +#endif + +CTEST(cscal, i_nan) +{ + blasint N=9; + blasint incX=1; + float i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; + float nan[] = {NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0}; + BLASFUNC(cscal)(&N, i, nan, &incX); + ASSERT_TRUE(isnan(nan[0])); + ASSERT_TRUE(isnan(nan[1])); + ASSERT_TRUE(isnan(nan[16])); + ASSERT_TRUE(isnan(nan[17])); +} + +CTEST(cscal, i_nan_inc_2) +{ + blasint N=9; + blasint incX=2; + float i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; + float nan[] = {NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, + NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0}; + BLASFUNC(cscal)(&N, i, nan, &incX); + ASSERT_TRUE(isnan(nan[0])); + ASSERT_TRUE(isnan(nan[1])); + ASSERT_TRUE(isnan(nan[16])); + ASSERT_TRUE(isnan(nan[17])); +} + +CTEST(cscal, nan_i) +{ + blasint N=9; + blasint incX=1; + float i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; + float nan[] = {NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0}; + BLASFUNC(cscal)(&N, nan, i, &incX); + ASSERT_TRUE(isnan(i[0])); + ASSERT_TRUE(isnan(i[1])); + ASSERT_TRUE(isnan(i[16])); + ASSERT_TRUE(isnan(i[17])); +} + +CTEST(cscal, nan_i_inc_2) +{ + blasint N=9; + blasint incX=2; + float i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, + 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; + float nan[] = {NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0}; + BLASFUNC(cscal)(&N, nan, i, &incX); + ASSERT_TRUE(isnan(i[0])); + ASSERT_TRUE(isnan(i[1])); + ASSERT_TRUE(isnan(i[16])); + ASSERT_TRUE(isnan(i[17])); +} + +CTEST(cscal, i_inf) +{ + blasint N=9; + blasint incX=1; + float i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; + float inf[] = {INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0}; + BLASFUNC(cscal)(&N, i, inf, &incX); + ASSERT_TRUE(isnan(inf[0])); + ASSERT_TRUE(isinf(inf[1])); + ASSERT_TRUE(isnan(inf[16])); + ASSERT_TRUE(isinf(inf[17])); +} + +CTEST(cscal, i_inf_inc_2) +{ + blasint N=9; + blasint incX=2; + float i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; + float inf[] = {INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, + INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0}; + BLASFUNC(cscal)(&N, i, inf, &incX); + ASSERT_TRUE(isnan(inf[0])); + ASSERT_TRUE(isinf(inf[1])); + ASSERT_TRUE(isnan(inf[16])); + ASSERT_TRUE(isinf(inf[17])); +} + +CTEST(cscal, inf_i) +{ + blasint N=9; + blasint incX=1; + float i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; + float inf[] = {INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0}; + BLASFUNC(cscal)(&N, inf, i, &incX); + ASSERT_TRUE(isnan(i[0])); + ASSERT_TRUE(isinf(i[1])); + ASSERT_TRUE(isnan(i[16])); + ASSERT_TRUE(isinf(i[17])); +} + +CTEST(cscal, inf_i_inc_2) +{ + blasint N=9; + blasint incX=2; + float i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, + 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; + float inf[] = {INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0}; + BLASFUNC(cscal)(&N, inf, i, &incX); + ASSERT_TRUE(isnan(i[0])); + ASSERT_TRUE(isinf(i[1])); + ASSERT_TRUE(isnan(i[16])); + ASSERT_TRUE(isinf(i[17])); +} + +CTEST(cscal, i_0inf) +{ + blasint N=9; + blasint incX=1; + float i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; + float inf[] = {0,INFINITY, 0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY,0, INFINITY}; + BLASFUNC(cscal)(&N, i, inf, &incX); + ASSERT_TRUE(isinf(inf[0])); + ASSERT_TRUE(isnan(inf[1])); + ASSERT_TRUE(isinf(inf[16])); + ASSERT_TRUE(isnan(inf[17])); +} + +CTEST(cscal, i_0inf_inc_2) +{ + blasint N=9; + blasint incX=2; + float i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; + float inf[] = {0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY, + 0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY, 0,INFINITY}; + BLASFUNC(cscal)(&N, i, inf, &incX); + ASSERT_TRUE(isinf(inf[0])); + ASSERT_TRUE(isnan(inf[1])); + ASSERT_TRUE(isinf(inf[16])); + ASSERT_TRUE(isnan(inf[17])); +} + +#endif + #ifdef BUILD_COMPLEX16 #ifndef NAN @@ -25,7 +469,7 @@ CTEST(zscal, i_nan) CTEST(zscal, i_nan_inc_2) { blasint N=9; - blasint incX=1; + blasint incX=2; double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double nan[] = {NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0}; @@ -52,7 +496,7 @@ CTEST(zscal, nan_i) CTEST(zscal, nan_i_inc_2) { blasint N=9; - blasint incX=1; + blasint incX=2; double i[] = {0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1, 0,1 }; double nan[] = {NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0, NAN,0}; From 1abafcd9b2c32a07a67f9db7b57485c464828bad Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 6 Jun 2024 23:59:43 +0200 Subject: [PATCH 124/267] handle corner cases involving NAN and/or INF --- kernel/x86_64/cscal.c | 59 ++++++++++++++++++++++++++++++------------- kernel/x86_64/dscal.c | 5 ++++ kernel/x86_64/sscal.c | 44 +++++++++++++++++++++----------- 3 files changed, 76 insertions(+), 32 deletions(-) diff --git a/kernel/x86_64/cscal.c b/kernel/x86_64/cscal.c index 95a99b8b97..212a215941 100644 --- a/kernel/x86_64/cscal.c +++ b/kernel/x86_64/cscal.c @@ -259,11 +259,22 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i, while(j < n1) { + if (isnan(x[i]) || isinf(x[i])) + temp0 = NAN; + else temp0 = -da_i * x[i+1]; + if (!isinf(x[i+1])) x[i+1] = da_i * x[i]; + else + x[i+1] = NAN; x[i] = temp0; + if (isnan(x[i+inc_x]) || isinf(x[i+inc_x])) + temp1 = NAN; + else temp1 = -da_i * x[i+1+inc_x]; - x[i+1+inc_x] = da_i * x[i+inc_x]; + if (!isinf(x[i+1+inc_x])) + x[i+1+inc_x] = da_i * x[i+inc_x]; + else x[i+1+inc_x] = NAN; x[i+inc_x] = temp1; i += 2*inc_x ; j+=2; @@ -272,9 +283,14 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i, while(j < n) { - - temp0 = -da_i * x[i+1]; + + if (isnan(x[i]) || isinf(x[i])) + temp0 = NAN; + else + temp0 = -da_i * x[i+1]; + if (!isinf(x[i+1])) x[i+1] = da_i * x[i]; + else x[i+1] = NAN; x[i] = temp0; i += inc_x ; j++; @@ -365,42 +381,51 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i, else cscal_kernel_16_zero_r(n1 , alpha , x); else - if ( da_i == 0 ) - cscal_kernel_16_zero_i(n1 , alpha , x); - else cscal_kernel_16(n1 , alpha , x); i = n1 << 1; j = n1; } - - if ( da_r == 0.0 ) + if ( da_r == 0.0 || isnan(da_r) ) { - if ( da_i == 0.0 ) { - + FLOAT res=0.0; + if (isnan(da_r)) res= da_r; while(j < n) { - - x[i]=0.0; - x[i+1]=0.0; + x[i]=res; + x[i+1]=res; i += 2 ; j++; } } - else + else if (isinf(da_r)) { + while(j < n) + { + x[i]= NAN; + x[i+1] = da_r; + i += 2 ; + j++; + + } + + } else { while(j < n) { - temp0 = -da_i * x[i+1]; - x[i+1] = da_i * x[i]; - x[i] = temp0; + if (isinf(x[i])) + temp0 = NAN; + if (!isinf(x[i+1])) + x[i+1] = da_i * x[i]; + else x[i+1] = NAN; + if ( x[i] == x[i]) //preserve NaN + x[i] = temp0; i += 2 ; j++; diff --git a/kernel/x86_64/dscal.c b/kernel/x86_64/dscal.c index 0b028b7829..e7182c5ce8 100644 --- a/kernel/x86_64/dscal.c +++ b/kernel/x86_64/dscal.c @@ -234,6 +234,11 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS else x[i] = 0.0; } } + else if (isinf(da)){ + for ( i=n1 ; i 0 ) { @@ -151,10 +154,9 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS i = n1 * inc_x; j = n1; } - +#endif while(j < n) { - x[i] *= da; i += inc_x ; j++; @@ -162,16 +164,15 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS } } - return(0); } BLASLONG n1 = n & -16; if ( n1 > 0 ) { - if ( da == 0.0 ) - sscal_kernel_16_zero(n1 , &da , x); - else + //if ( da == 0.0 ) + // sscal_kernel_16_zero(n1 , &da , x); + //else sscal_kernel_16(n1 , &da , x); } @@ -179,7 +180,18 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS { for ( i=n1 ; i Date: Fri, 7 Jun 2024 08:40:17 +0530 Subject: [PATCH 125/267] Dynamic locking in Pthread Backend to allow multiple BLAS calls to be executed parallelly --- driver/level3/level3_thread.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/driver/level3/level3_thread.c b/driver/level3/level3_thread.c index 9fec7afca2..ddb39abd66 100644 --- a/driver/level3/level3_thread.c +++ b/driver/level3/level3_thread.c @@ -570,6 +570,8 @@ static int gemm_driver(blas_arg_t *args, BLASLONG *range_m, BLASLONG InitializeCriticalSection((PCRITICAL_SECTION)&level3_lock); #else static pthread_mutex_t level3_lock = PTHREAD_MUTEX_INITIALIZER; + static pthread_cond_t level3_wakeup = PTHREAD_COND_INITIALIZER; + volatile static BLASLONG CPU_AVAILABLE = MAX_CPU_NUMBER; #endif blas_arg_t newarg; @@ -639,6 +641,12 @@ static int gemm_driver(blas_arg_t *args, BLASLONG *range_m, BLASLONG EnterCriticalSection((PCRITICAL_SECTION)&level3_lock); #else pthread_mutex_lock(&level3_lock); + while(CPU_AVAILABLE < nthreads) { + pthread_cond_wait(&level3_wakeup, &level3_lock); + } + CPU_AVAILABLE -= nthreads; + WMB; + pthread_mutex_unlock(&level3_lock); #endif #ifdef USE_ALLOC_HEAP @@ -783,6 +791,10 @@ static int gemm_driver(blas_arg_t *args, BLASLONG *range_m, BLASLONG #elif defined(OS_WINDOWS) LeaveCriticalSection((PCRITICAL_SECTION)&level3_lock); #else + pthread_mutex_lock(&level3_lock); + CPU_AVAILABLE += nthreads; + WMB; + pthread_cond_signal(&level3_wakeup); pthread_mutex_unlock(&level3_lock); #endif From 5ed4f24d6e11072c4d0a3542dc7b8adc4557cca7 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 7 Jun 2024 09:39:08 +0200 Subject: [PATCH 126/267] Handle corner cases with INF and NAN arguments --- kernel/mips/scal.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kernel/mips/scal.c b/kernel/mips/scal.c index 01f708b1d9..d51fd9ccdd 100644 --- a/kernel/mips/scal.c +++ b/kernel/mips/scal.c @@ -35,7 +35,12 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS { if ( da == 0.0 ) - x[i]=0.0; + if (isnan(x[i])||isinf(x[i])) + x[i]=NAN; + else + x[i]=0.0; + else if (isnan(da)) + x[i]=NAN; else x[i] = da * x[i] ; From c7cacd9b3897c5df725905905c2ef92b111dec39 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 7 Jun 2024 13:48:56 +0200 Subject: [PATCH 127/267] disable the shortcut for da=0 to ensure proper handling of INF and NAN --- kernel/arm64/scal.S | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/arm64/scal.S b/kernel/arm64/scal.S index 09c41cdaab..5029890f67 100644 --- a/kernel/arm64/scal.S +++ b/kernel/arm64/scal.S @@ -168,8 +168,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmp N, xzr ble .Lscal_kernel_L999 - fcmp DA, #0.0 - beq .Lscal_kernel_zero + //fcmp DA, #0.0 + //beq .Lscal_kernel_zero cmp INC_X, #1 bne .Lscal_kernel_S_BEGIN From 6ffaf998179ab4fbf17c140f84e244ab02dc72f4 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 7 Jun 2024 14:46:58 +0200 Subject: [PATCH 128/267] disable da=0 shortcut to handle NAN and INF correctly --- kernel/x86/scal.S | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/x86/scal.S b/kernel/x86/scal.S index 377d4ef616..a24c832d35 100644 --- a/kernel/x86/scal.S +++ b/kernel/x86/scal.S @@ -71,8 +71,9 @@ je .L300 # Alpha != ZERO /* Alpha == ZERO */ - cmpl $1,%esi - jne .L104 +// cmpl $1,%esi +// jne .L104 + jmp .L104 movl %edx, %ecx # ecx = n sarl $3, %ecx # (n >> 3) From 2f12a47405447a4c29ef17ddaff06feed76962dd Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sun, 9 Jun 2024 20:32:10 +0200 Subject: [PATCH 129/267] fix build options for CAXPYC/ZAXPYC --- interface/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index 05b893b8c4..449072bae1 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -137,7 +137,7 @@ endif () foreach (float_type ${FLOAT_TYPES}) if (${float_type} STREQUAL "COMPLEX" OR ${float_type} STREQUAL "ZCOMPLEX") - GenerateNamedObjects("zaxpy.c" "" "axpyc" ${CBLAS_FLAG} "" "" false ${float_type}) + GenerateNamedObjects("zaxpy.c" "CONJ" "axpyc" ${CBLAS_FLAG} "" "" false ${float_type}) GenerateNamedObjects("zger.c" "" "geru" ${CBLAS_FLAG} "" "" false ${float_type}) GenerateNamedObjects("zger.c" "CONJ" "gerc" ${CBLAS_FLAG} "" "" false ${float_type}) From 1ca1bb829dbad8bc861109ca0cabbe0462d159cb Mon Sep 17 00:00:00 2001 From: guxiwei Date: Thu, 13 Jun 2024 09:37:12 +0800 Subject: [PATCH 130/267] LoongArch64: Update QEMU Compile the community version of QEMU to support LSX/LASX extension instructions --- .github/workflows/loongarch64.yml | 95 ++++++++++++++----------- .github/workflows/loongarch64_clang.yml | 95 ++++++++++++++----------- 2 files changed, 108 insertions(+), 82 deletions(-) diff --git a/.github/workflows/loongarch64.yml b/.github/workflows/loongarch64.yml index b23c81f965..da7f6c9a0c 100644 --- a/.github/workflows/loongarch64.yml +++ b/.github/workflows/loongarch64.yml @@ -33,10 +33,8 @@ jobs: - name: Install APT deps run: | - sudo add-apt-repository ppa:savoury1/virtualisation sudo apt-get update - sudo apt-get install autoconf automake autotools-dev ninja-build make ccache \ - qemu-user-static + sudo apt-get install autoconf automake autotools-dev ninja-build make ccache - name: Download and install loongarch64-toolchain run: | @@ -44,6 +42,20 @@ jobs: #wget https://github.com/loongson/build-tools/releases/download/2023.08.08/CLFS-loongarch64-8.1-x86_64-cross-tools-gcc-glibc.tar.xz tar -xf CLFS-loongarch64-8.1-x86_64-cross-tools-gcc-glibc.tar.xz -C /opt + - name: Checkout qemu + uses: actions/checkout@v3 + with: + repository: qemu/qemu + path: qemu + ref: master + + - name: Install qemu + run: | + cd qemu + ./configure --prefix=$GITHUB_WORKSPACE/qemu-install --target-list=loongarch64-linux-user --disable-system --static + make -j$(nproc) + make install + - name: Set env run: | echo "LD_LIBRARY_PATH=/opt/cross-tools/target/usr/lib64:/opt/cross-tools/loongarch64-unknown-linux-gnu/lib64:$LD_LIBRARY_PATH" >> $GITHUB_ENV @@ -76,45 +88,46 @@ jobs: - name: Test run: | - qemu-loongarch64-static ./utest/openblas_utest - qemu-loongarch64-static ./utest/openblas_utest_ext - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xscblat1 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xdcblat1 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xccblat1 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xzcblat1 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xscblat2 < ./ctest/sin2 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xdcblat2 < ./ctest/din2 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xccblat2 < ./ctest/cin2 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xzcblat2 < ./ctest/zin2 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xscblat3 < ./ctest/sin3 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xdcblat3 < ./ctest/din3 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xccblat3 < ./ctest/cin3 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xzcblat3 < ./ctest/zin3 - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/sblat1 - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/dblat1 - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/cblat1 - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/zblat1 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/sblat1 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/dblat1 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/cblat1 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/zblat1 + export PATH=$GITHUB_WORKSPACE/qemu-install/bin/:$PATH + qemu-loongarch64 ./utest/openblas_utest + qemu-loongarch64 ./utest/openblas_utest_ext + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xscblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xdcblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xccblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xzcblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xscblat2 < ./ctest/sin2 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xdcblat2 < ./ctest/din2 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xccblat2 < ./ctest/cin2 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xzcblat2 < ./ctest/zin2 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xscblat3 < ./ctest/sin3 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xdcblat3 < ./ctest/din3 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xccblat3 < ./ctest/cin3 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xzcblat3 < ./ctest/zin3 + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/sblat1 + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/dblat1 + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/cblat1 + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/zblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/sblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/dblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/cblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/zblat1 rm -f ./test/?BLAT2.SUMM - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/sblat2 < ./test/sblat2.dat - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/dblat2 < ./test/dblat2.dat - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/cblat2 < ./test/cblat2.dat - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/zblat2 < ./test/zblat2.dat + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/sblat2 < ./test/sblat2.dat + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/dblat2 < ./test/dblat2.dat + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/cblat2 < ./test/cblat2.dat + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/zblat2 < ./test/zblat2.dat rm -f ./test/?BLAT2.SUMM - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/sblat2 < ./test/sblat2.dat - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/dblat2 < ./test/dblat2.dat - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/cblat2 < ./test/cblat2.dat - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/zblat2 < ./test/zblat2.dat + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/sblat2 < ./test/sblat2.dat + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/dblat2 < ./test/dblat2.dat + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/cblat2 < ./test/cblat2.dat + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/zblat2 < ./test/zblat2.dat rm -f ./test/?BLAT3.SUMM - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/sblat3 < ./test/sblat3.dat - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/dblat3 < ./test/dblat3.dat - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/cblat3 < ./test/cblat3.dat - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/zblat3 < ./test/zblat3.dat + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/sblat3 < ./test/sblat3.dat + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/dblat3 < ./test/dblat3.dat + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/cblat3 < ./test/cblat3.dat + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/zblat3 < ./test/zblat3.dat rm -f ./test/?BLAT3.SUMM - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/sblat3 < ./test/sblat3.dat - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/dblat3 < ./test/dblat3.dat - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/cblat3 < ./test/cblat3.dat - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/zblat3 < ./test/zblat3.dat + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/sblat3 < ./test/sblat3.dat + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/dblat3 < ./test/dblat3.dat + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/cblat3 < ./test/cblat3.dat + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/zblat3 < ./test/zblat3.dat diff --git a/.github/workflows/loongarch64_clang.yml b/.github/workflows/loongarch64_clang.yml index b32e4d7319..1a2484395a 100644 --- a/.github/workflows/loongarch64_clang.yml +++ b/.github/workflows/loongarch64_clang.yml @@ -34,10 +34,8 @@ jobs: - name: Install APT deps run: | - sudo add-apt-repository ppa:savoury1/virtualisation sudo apt-get update - sudo apt-get install autoconf automake autotools-dev ninja-build make ccache \ - qemu-user-static + sudo apt-get install autoconf automake autotools-dev ninja-build make ccache - name: Download and install loongarch64-toolchain run: | @@ -46,6 +44,20 @@ jobs: tar -xf clang+llvm_8.0.1-6_amd64-linux-gnu_debian-10.tar.gz -C /opt tar -xf loongson-gnu-toolchain-8.3-x86_64-loongarch64-linux-gnu-rc1.3.tar.xz -C /opt + - name: Checkout qemu + uses: actions/checkout@v3 + with: + repository: qemu/qemu + path: qemu + ref: master + + - name: Install qemu + run: | + cd qemu + ./configure --prefix=$GITHUB_WORKSPACE/qemu-install --target-list=loongarch64-linux-user --disable-system --static + make -j$(nproc) + make install + - name: Set env run: | echo "PATH=$GITHUB_WORKSPACE:/opt/clang+llvm_8.0.1-6_amd64-linux-gnu_debian-10/bin:/opt/loongson-gnu-toolchain-8.3-x86_64-loongarch64-linux-gnu-rc1.3/bin:$PATH" >> $GITHUB_ENV @@ -77,46 +89,47 @@ jobs: - name: Test run: | - qemu-loongarch64-static ./utest/openblas_utest - qemu-loongarch64-static ./utest/openblas_utest_ext - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xscblat1 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xdcblat1 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xccblat1 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xzcblat1 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xscblat2 < ./ctest/sin2 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xdcblat2 < ./ctest/din2 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xccblat2 < ./ctest/cin2 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xzcblat2 < ./ctest/zin2 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xscblat3 < ./ctest/sin3 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xdcblat3 < ./ctest/din3 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xccblat3 < ./ctest/cin3 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./ctest/xzcblat3 < ./ctest/zin3 - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/sblat1 - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/dblat1 - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/cblat1 - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/zblat1 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/sblat1 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/dblat1 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/cblat1 - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/zblat1 + export PATH=$GITHUB_WORKSPACE/qemu-install/bin/:$PATH + qemu-loongarch64 ./utest/openblas_utest + qemu-loongarch64 ./utest/openblas_utest_ext + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xscblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xdcblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xccblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xzcblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xscblat2 < ./ctest/sin2 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xdcblat2 < ./ctest/din2 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xccblat2 < ./ctest/cin2 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xzcblat2 < ./ctest/zin2 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xscblat3 < ./ctest/sin3 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xdcblat3 < ./ctest/din3 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xccblat3 < ./ctest/cin3 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./ctest/xzcblat3 < ./ctest/zin3 + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/sblat1 + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/dblat1 + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/cblat1 + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/zblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/sblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/dblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/cblat1 + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/zblat1 rm -f ./test/?BLAT2.SUMM - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/sblat2 < ./test/sblat2.dat - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/dblat2 < ./test/dblat2.dat - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/cblat2 < ./test/cblat2.dat - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/zblat2 < ./test/zblat2.dat + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/sblat2 < ./test/sblat2.dat + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/dblat2 < ./test/dblat2.dat + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/cblat2 < ./test/cblat2.dat + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/zblat2 < ./test/zblat2.dat rm -f ./test/?BLAT2.SUMM - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/sblat2 < ./test/sblat2.dat - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/dblat2 < ./test/dblat2.dat - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/cblat2 < ./test/cblat2.dat - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/zblat2 < ./test/zblat2.dat + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/sblat2 < ./test/sblat2.dat + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/dblat2 < ./test/dblat2.dat + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/cblat2 < ./test/cblat2.dat + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/zblat2 < ./test/zblat2.dat rm -f ./test/?BLAT3.SUMM - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/sblat3 < ./test/sblat3.dat - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/dblat3 < ./test/dblat3.dat - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/cblat3 < ./test/cblat3.dat - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64-static ./test/zblat3 < ./test/zblat3.dat + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/sblat3 < ./test/sblat3.dat + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/dblat3 < ./test/dblat3.dat + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/cblat3 < ./test/cblat3.dat + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 qemu-loongarch64 ./test/zblat3 < ./test/zblat3.dat rm -f ./test/?BLAT3.SUMM - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/sblat3 < ./test/sblat3.dat - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/dblat3 < ./test/dblat3.dat - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/cblat3 < ./test/cblat3.dat - OPENBLAS_NUM_THREADS=2 qemu-loongarch64-static ./test/zblat3 < ./test/zblat3.dat + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/sblat3 < ./test/sblat3.dat + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/dblat3 < ./test/dblat3.dat + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/cblat3 < ./test/cblat3.dat + OPENBLAS_NUM_THREADS=2 qemu-loongarch64 ./test/zblat3 < ./test/zblat3.dat From ed5db5b122399489ba6648017962bb9e22830e43 Mon Sep 17 00:00:00 2001 From: guxiwei Date: Thu, 13 Jun 2024 11:09:53 +0800 Subject: [PATCH 131/267] LoongArch64: Update the address for obtaining the Clang cross-toolchain Improve the stability and speed of testing --- .github/workflows/loongarch64_clang.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/loongarch64_clang.yml b/.github/workflows/loongarch64_clang.yml index 1a2484395a..d08e56f627 100644 --- a/.github/workflows/loongarch64_clang.yml +++ b/.github/workflows/loongarch64_clang.yml @@ -39,8 +39,8 @@ jobs: - name: Download and install loongarch64-toolchain run: | - wget http://ftp.loongnix.cn/toolchain/llvm/llvm8/clang+llvm_8.0.1-6_amd64-linux-gnu_debian-10.tar.gz - wget http://ftp.loongnix.cn/toolchain/gcc/release/loongarch/gcc8/loongson-gnu-toolchain-8.3-x86_64-loongarch64-linux-gnu-rc1.3.tar.xz + wget https://github.com/XiWeiGu/loongarch64_toolchain/releases/download/V0.1/clang+llvm_8.0.1-6_amd64-linux-gnu_debian-10.tar.gz + wget https://github.com/XiWeiGu/loongarch64_toolchain/releases/download/V0.1/loongson-gnu-toolchain-8.3-x86_64-loongarch64-linux-gnu-rc1.3.tar.xz tar -xf clang+llvm_8.0.1-6_amd64-linux-gnu_debian-10.tar.gz -C /opt tar -xf loongson-gnu-toolchain-8.3-x86_64-loongarch64-linux-gnu-rc1.3.tar.xz -C /opt From dd7efcf9efe836fedd49cffb6293d1e9a791be54 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 14 Jun 2024 14:21:13 +0200 Subject: [PATCH 132/267] Avoid exceeding the configured thread count in x86_64 TOBF16 (#4748) * avoid setting nthreads higher than available --- kernel/x86_64/tobf16.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kernel/x86_64/tobf16.c b/kernel/x86_64/tobf16.c index a88fdcc2e6..e8e2289552 100644 --- a/kernel/x86_64/tobf16.c +++ b/kernel/x86_64/tobf16.c @@ -144,10 +144,11 @@ void CNAME(BLASLONG n, FLOAT_TYPE * in, BLASLONG inc_in, bfloat16 * out, BLASLON if (inc_in == 0 || inc_out == 0 || n <= 100000) { nthreads = 1; } else { + nthreads = num_cpu_avail(1); if (n/100000 < 100) { - nthreads = 4; - } else { - nthreads = 16; + nthreads = MAX(nthreads,4); +// } else { +// nthreads = MAX(nthreads,16); } } From 3d8054fb16c9b7db8d8424bba569bfdc8b404571 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 14 Jun 2024 22:07:44 +0200 Subject: [PATCH 133/267] add clobber list --- kernel/arm64/dot_kernel_sve.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kernel/arm64/dot_kernel_sve.c b/kernel/arm64/dot_kernel_sve.c index 16f4cd5377..bc9975214b 100644 --- a/kernel/arm64/dot_kernel_sve.c +++ b/kernel/arm64/dot_kernel_sve.c @@ -108,7 +108,12 @@ dot_kernel_sve(BLASLONG n, FLOAT* x, FLOAT* y) [N_] "r" (n), [X_] "r" (x), [Y_] "r" (y) - :); + : "cc", + "memory", + "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", + "x8", "x9", "x10", "x11", "x12", "x13", "d1", + "z0", "z1" + ); return ret; } From 21c0f769efe35ae1f88f4c664ef1405a519bb432 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 14 Jun 2024 23:54:27 +0200 Subject: [PATCH 134/267] ensure that cpu-specific -march options are always applied to icx --- cmake/cc.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/cc.cmake b/cmake/cc.cmake index 9dce9ea35a..775239e1cd 100644 --- a/cmake/cc.cmake +++ b/cmake/cc.cmake @@ -6,6 +6,7 @@ include(CheckCCompilerFlag) if (${CMAKE_C_COMPILER_ID} MATCHES "IntelLLVM") set(CCOMMON_OPT "${CCOMMON_OPT} -fp-model=consistent") + set(GCC_VERSION 100) endif () if (${CMAKE_C_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_C_COMPILER_ID} STREQUAL "LSB" OR ${CMAKE_C_COMPILER_ID} MATCHES "Clang") From d25ee4d0f553229cb322c8df988d0e9a75e04999 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 14 Jun 2024 23:58:45 +0200 Subject: [PATCH 135/267] Fix detection of Intel ifx and apply -fp-model option to it --- cmake/fc.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/fc.cmake b/cmake/fc.cmake index 69246385c4..8798ce8b4c 100644 --- a/cmake/fc.cmake +++ b/cmake/fc.cmake @@ -117,12 +117,12 @@ if (${F_COMPILER} STREQUAL "GFORTRAN" OR ${F_COMPILER} STREQUAL "F95" OR CMAKE_F endif () endif () -if (${F_COMPILER} STREQUAL "INTEL") +if (${F_COMPILER} STREQUAL "INTEL" OR CMAKE_Fortran_COMPILER_ID MATCHES "Intel") set(CCOMMON_OPT "${CCOMMON_OPT} -DF_INTERFACE_INTEL") if (INTERFACE64) set(FCOMMON_OPT "${FCOMMON_OPT} -i8") endif () - set(FCOMMON_OPT "${FCOMMON_OPT} -recursive") + set(FCOMMON_OPT "${FCOMMON_OPT} -recursive -fp-model=consistent") if (USE_OPENMP) set(FCOMMON_OPT "${FCOMMON_OPT} -openmp") endif () From 33bb4b98a4725efef26517d6fbec7a81260ee531 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 15 Jun 2024 14:16:48 +0200 Subject: [PATCH 136/267] Improve error message output from the fork() utest (#4753) * Add perror to report the reason for a fork failure * reword the malloc failure message --- utest/test_fork.c | 7 +++++-- utest/test_post_fork.c | 9 +++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/utest/test_fork.c b/utest/test_fork.c index 558026031b..44a619349a 100644 --- a/utest/test_fork.c +++ b/utest/test_fork.c @@ -33,6 +33,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include #include #include "openblas_utest.h" @@ -41,7 +42,7 @@ static void* xmalloc(size_t n) void* tmp; tmp = malloc(n); if (tmp == NULL) { - fprintf(stderr, "You are about to die\n"); + fprintf(stderr, "Failed to allocate memory for the testcase.\n"); exit(1); } else { return tmp; @@ -103,6 +104,7 @@ exit(0); fork_pid = fork(); if (fork_pid == -1) { + perror("fork"); CTEST_ERR("Failed to fork process."); } else if (fork_pid == 0) { // Compute a DGEMM product in the child process to check that the @@ -113,7 +115,8 @@ exit(0); // recursively fork_pid_nested = fork(); if (fork_pid_nested == -1) { - CTEST_ERR("Failed to fork process."); + perror("fork"); + CTEST_ERR("Failed to fork nested process."); exit(1); } else if (fork_pid_nested == 0) { check_dgemm(a, b, d, c, n); diff --git a/utest/test_post_fork.c b/utest/test_post_fork.c index 6d640aebb6..d6e87f2ba2 100644 --- a/utest/test_post_fork.c +++ b/utest/test_post_fork.c @@ -33,6 +33,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include #include #ifdef USE_OPENMP #include @@ -44,7 +45,7 @@ static void* xmalloc(size_t n) void* tmp; tmp = malloc(n); if (tmp == NULL) { - fprintf(stderr, "You are about to die\n"); + fprintf(stderr, "Failed to allocate memory for the test payload.\n"); exit(1); } else { return tmp; @@ -114,7 +115,11 @@ exit(0); fork_pid = fork(); if (fork_pid == -1) { - CTEST_ERR("Failed to fork process."); + perror("fork"); + CTEST_ERR("Failed to fork subprocesses in a loop."); +#ifdef USE_OPENMP + CTEST_ERR("Number of OpenMP threads was %d in this attempt.",i); +#endif } else if (fork_pid == 0) { // Just pretend to do something, e.g. call `uname`, then exit exit(0); From cf2962bdb5a0d4ad9607fee0b0e4ac4a08e11225 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 18 Jun 2024 11:15:44 +0200 Subject: [PATCH 137/267] fix possible infinite loop on error (Reference-LAPACK PR 1024) --- lapack-netlib/TESTING/EIG/ddrvst.f | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lapack-netlib/TESTING/EIG/ddrvst.f b/lapack-netlib/TESTING/EIG/ddrvst.f index 805fd8271c..b589303004 100644 --- a/lapack-netlib/TESTING/EIG/ddrvst.f +++ b/lapack-netlib/TESTING/EIG/ddrvst.f @@ -2772,7 +2772,7 @@ SUBROUTINE DDRVST( NSIZES, NN, NTYPES, DOTYPE, ISEED, THRESH, RESULT( NTEST ) = ULPINV RESULT( NTEST+1 ) = ULPINV RESULT( NTEST+2 ) = ULPINV - GO TO 700 + GO TO 1750 END IF END IF * @@ -2797,13 +2797,13 @@ SUBROUTINE DDRVST( NSIZES, NN, NTYPES, DOTYPE, ISEED, THRESH, RETURN ELSE RESULT( NTEST ) = ULPINV - GO TO 700 + GO TO 1750 END IF END IF * IF( M3.EQ.0 .AND. N.GT.0 ) THEN RESULT( NTEST ) = ULPINV - GO TO 700 + GO TO 1750 END IF * * Do test 78 (or +54) @@ -2819,7 +2819,9 @@ SUBROUTINE DDRVST( NSIZES, NN, NTYPES, DOTYPE, ISEED, THRESH, $ MAX( UNFL, TEMP3*ULP ) * CALL DLACPY( ' ', N, N, V, LDU, A, LDA ) -* + + 1750 CONTINUE + 1720 CONTINUE * * End of Loop -- Check for RESULT(j) > THRESH From bf521a2ced44f658edca74152401fabb9d894a0d Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 18 Jun 2024 11:18:04 +0200 Subject: [PATCH 138/267] fix possible infinite loop on error (Reference-LAPACK PR 1024) --- lapack-netlib/TESTING/EIG/sdrvst.f | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lapack-netlib/TESTING/EIG/sdrvst.f b/lapack-netlib/TESTING/EIG/sdrvst.f index be6d33cee4..2d02f54edf 100644 --- a/lapack-netlib/TESTING/EIG/sdrvst.f +++ b/lapack-netlib/TESTING/EIG/sdrvst.f @@ -2772,7 +2772,7 @@ SUBROUTINE SDRVST( NSIZES, NN, NTYPES, DOTYPE, ISEED, THRESH, RESULT( NTEST ) = ULPINV RESULT( NTEST+1 ) = ULPINV RESULT( NTEST+2 ) = ULPINV - GO TO 700 + GO TO 1750 END IF END IF * @@ -2797,13 +2797,13 @@ SUBROUTINE SDRVST( NSIZES, NN, NTYPES, DOTYPE, ISEED, THRESH, RETURN ELSE RESULT( NTEST ) = ULPINV - GO TO 700 + GO TO 1750 END IF END IF * IF( M3.EQ.0 .AND. N.GT.0 ) THEN RESULT( NTEST ) = ULPINV - GO TO 700 + GO TO 1750 END IF * * Do test 78 (or +54) @@ -2819,6 +2819,8 @@ SUBROUTINE SDRVST( NSIZES, NN, NTYPES, DOTYPE, ISEED, THRESH, $ MAX( UNFL, TEMP3*ULP ) * CALL SLACPY( ' ', N, N, V, LDU, A, LDA ) +* + 1750 CONTINUE * 1720 CONTINUE * From a9817b421235e3a1cba2297149460c7a4da00d24 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 18 Jun 2024 11:20:22 +0200 Subject: [PATCH 139/267] fix reference in format (Reference-LAPACK PR 1024) --- lapack-netlib/TESTING/EIG/dlahd2.f | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lapack-netlib/TESTING/EIG/dlahd2.f b/lapack-netlib/TESTING/EIG/dlahd2.f index a6c65635f0..cccbd2537c 100644 --- a/lapack-netlib/TESTING/EIG/dlahd2.f +++ b/lapack-netlib/TESTING/EIG/dlahd2.f @@ -534,8 +534,8 @@ SUBROUTINE DLAHD2( IOUNIT, PATH ) $ / ' 2: norm( I - Q'' Q ) / ( m ulp )', $ / ' 3: norm( I - PT PT'' ) / ( n ulp )', $ / ' 4: norm( Y - Q'' C ) / ( norm(Y) max(m,nrhs) ulp )' ) - 9968 FORMAT( / ' Tests performed: See sdrvst.f' ) - 9967 FORMAT( / ' Tests performed: See cdrvst.f' ) + 9968 FORMAT( / ' Tests performed: See ddrvst.f' ) + 9967 FORMAT( / ' Tests performed: See zdrvst.f' ) * * End of DLAHD2 * From 215279662ed1c9ffd6552d51e351ea8cf15b8dc4 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 18 Jun 2024 11:21:33 +0200 Subject: [PATCH 140/267] fix possible infinite loop on error (Reference-LAPACK PR 1024) --- lapack-netlib/TESTING/EIG/ddrvst.f | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lapack-netlib/TESTING/EIG/ddrvst.f b/lapack-netlib/TESTING/EIG/ddrvst.f index b589303004..311684238d 100644 --- a/lapack-netlib/TESTING/EIG/ddrvst.f +++ b/lapack-netlib/TESTING/EIG/ddrvst.f @@ -2819,9 +2819,9 @@ SUBROUTINE DDRVST( NSIZES, NN, NTYPES, DOTYPE, ISEED, THRESH, $ MAX( UNFL, TEMP3*ULP ) * CALL DLACPY( ' ', N, N, V, LDU, A, LDA ) - +* 1750 CONTINUE - +* 1720 CONTINUE * * End of Loop -- Check for RESULT(j) > THRESH From 758279605f905db17ebb664c972cca202068296f Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 20 Jun 2024 09:21:06 +0200 Subject: [PATCH 141/267] Add support forZhaoxin KX7000 --- cpuid_x86.c | 1 + 1 file changed, 1 insertion(+) diff --git a/cpuid_x86.c b/cpuid_x86.c index 40735c5634..e157232cf2 100644 --- a/cpuid_x86.c +++ b/cpuid_x86.c @@ -2525,6 +2525,7 @@ int get_coretype(void){ case 0x7: switch (exmodel) { case 5: + case 6: if (support_avx2()) return CORE_ZEN; else From 9b2a0c79cb165affdda50ee7db0b68ad4ee160c4 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 20 Jun 2024 09:23:08 +0200 Subject: [PATCH 142/267] Add Zhaoxin KX7000 --- driver/others/dynamic.c | 1 + 1 file changed, 1 insertion(+) diff --git a/driver/others/dynamic.c b/driver/others/dynamic.c index e3f9052653..1f7142009f 100644 --- a/driver/others/dynamic.c +++ b/driver/others/dynamic.c @@ -927,6 +927,7 @@ static gotoblas_t *get_coretype(void){ case 0x7: switch (exmodel) { case 5: + case 6: if (support_avx2()) return &gotoblas_ZEN; else From 3ec59922b6313365e9b31a164bcb42972fb51bc9 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 20 Jun 2024 16:19:32 +0200 Subject: [PATCH 143/267] Add a clobber list to fix utest errors seen with gcc13 on Apple M --- kernel/arm64/zdot_thunderx2t99.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kernel/arm64/zdot_thunderx2t99.c b/kernel/arm64/zdot_thunderx2t99.c index 6f65e5cfd1..d48392412b 100644 --- a/kernel/arm64/zdot_thunderx2t99.c +++ b/kernel/arm64/zdot_thunderx2t99.c @@ -292,7 +292,10 @@ static void zdot_compute(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLON : "cc", "memory", "x0", "x1", "x2", "x3", "x4", "x5", - "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7" + "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", + "v16", "v17", "v18", "v19", "v20", "v21", "v22", + "v23", "v24", "v25", "v26", "v27", "v28", "v29", + "v30", "v31" ); cf=OPENBLAS_MAKE_COMPLEX_FLOAT(dotr, doti); From f1248b849d11d6a436a6e19983c9c6695c143bc7 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 22 Jun 2024 15:55:29 +0200 Subject: [PATCH 144/267] handle INF and NAN in input --- kernel/power/dscal.c | 57 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 13 deletions(-) diff --git a/kernel/power/dscal.c b/kernel/power/dscal.c index 96c4e51bc5..2bbc1ea6d7 100644 --- a/kernel/power/dscal.c +++ b/kernel/power/dscal.c @@ -73,14 +73,38 @@ static void dscal_kernel_8_zero (BLASLONG n, FLOAT *x) for( i=0; i> 3) & 0x3; for (j = 0; j < align; j++) { - x[j] = 0.0; + if (isfinite(x[j])) + x[j] = 0.0; + else + x[j] = NAN; } } BLASLONG n1 = (n-j) & -16; @@ -127,8 +154,10 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS while(j < n) { - - x[j]=0.0; + if (!isfinite(x[j])) + x[j]=NAN; + else + x[j]=0.0; j++; } @@ -176,8 +205,10 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS while(j < n) { - - x[i]=0.0; + if (!isfinite(x[i])) + x[i]=NAN; + else + x[i]=0.0; i += inc_x ; j++; } From 7f8f037a36c05d3d8cf0ffc74db2202e64b95a22 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 22 Jun 2024 16:03:30 +0200 Subject: [PATCH 145/267] handle INF and NAN in input --- kernel/power/sscal.c | 57 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 13 deletions(-) diff --git a/kernel/power/sscal.c b/kernel/power/sscal.c index 65572a8c1f..12246b0a38 100644 --- a/kernel/power/sscal.c +++ b/kernel/power/sscal.c @@ -74,14 +74,38 @@ static void sscal_kernel_16_zero( BLASLONG n, FLOAT *x ) for( i=0; i> 2) & 0x7; for (j = 0; j < align; j++) { - x[j] = 0.0; + if (isfinite(x[j])) + x[j] = 0.0; + else + x[j] = NAN; } } BLASLONG n1 = (n-j) & -32; @@ -129,8 +156,10 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS while(j < n) { - - x[j]=0.0; + if (isfinite(x[j])) + x[j]=0.0; + else + x[j]=NAN; j++; } @@ -178,8 +207,10 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS while(j < n) { - - x[i]=0.0; + if (isfinite(x[i])) + x[i]=0.0; + else + x[i]=NAN; i += inc_x ; j++; } From 0a744a939a8d52827b23724364d61ac9dca9e06d Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 22 Jun 2024 21:07:43 +0200 Subject: [PATCH 146/267] temporarily(?) disable the alpha=0 branch to handle NaN/Inf in x --- kernel/x86_64/scal_sse2.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/x86_64/scal_sse2.S b/kernel/x86_64/scal_sse2.S index 20dd7fa2d8..b778895ba3 100644 --- a/kernel/x86_64/scal_sse2.S +++ b/kernel/x86_64/scal_sse2.S @@ -75,7 +75,7 @@ comisd %xmm0, %xmm1 jne .L100 # Alpha != ZERO jp .L100 # For Alpha = NaN - + je .L100 # disable the Alpha=zero path as it does not handle x=inf or nan /* Alpha == ZERO */ cmpq $SIZE, INCX jne .L50 From 68f2501958181da615837874f55fb113558da4e0 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 22 Jun 2024 21:08:57 +0200 Subject: [PATCH 147/267] temporarily(?) disable the alpha=0 branch to handle Inf/NaN in x --- kernel/x86_64/scal_sse.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/x86_64/scal_sse.S b/kernel/x86_64/scal_sse.S index b92688d9e4..91149af3ff 100644 --- a/kernel/x86_64/scal_sse.S +++ b/kernel/x86_64/scal_sse.S @@ -76,7 +76,7 @@ shufps $0, %xmm0, %xmm0 jne .L100 # Alpha != ZERO - + je .L100 /* Alpha == ZERO */ cmpq $SIZE, INCX jne .L50 From bd47630bcfb7c2f974d4b21e9c64ea9571929c35 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sun, 23 Jun 2024 00:54:39 +0200 Subject: [PATCH 148/267] exclude the alpha=0 branch as it does not handle NaN or Inf in x --- kernel/mips64/scal.S | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernel/mips64/scal.S b/kernel/mips64/scal.S index b28b8a309d..e392a9c6a5 100644 --- a/kernel/mips64/scal.S +++ b/kernel/mips64/scal.S @@ -79,6 +79,9 @@ bc1f $fcc0, .L50 NOP + bc1t $fcc0, .L50 + NOP + bne INCX, TEMP, .L20 dsra I, N, 3 From c08113c2798b907dce8efbea41b5f6545c71d913 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sun, 23 Jun 2024 01:12:33 +0200 Subject: [PATCH 149/267] fix special cases of x= NAN or INF --- kernel/mips/dscal_msa.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/kernel/mips/dscal_msa.c b/kernel/mips/dscal_msa.c index 2e41d8bef2..e95f0a6552 100644 --- a/kernel/mips/dscal_msa.c +++ b/kernel/mips/dscal_msa.c @@ -42,7 +42,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, if (1 == inc_x) { - if (0.0 == da) + if (0) //if (0.0 == da ) { v2f64 zero_v = {0.0, 0.0}; @@ -243,9 +243,11 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, if (da == 0.0) { for (i = n; i--;) - { - *x = 0.0; - + { + if (isfinite(*x)) + *x = 0.0; + else + *x = NAN; x += inc_x; } } From 541e1b6959790f2af0955a6f9540788b68a0abee Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sun, 23 Jun 2024 10:37:55 +0200 Subject: [PATCH 150/267] disable the fast path for inc=1, alpha=0 as it does not handle x=NaN or Inf --- kernel/mips/sscal_msa.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/kernel/mips/sscal_msa.c b/kernel/mips/sscal_msa.c index 66e17b8446..a1c7df4528 100644 --- a/kernel/mips/sscal_msa.c +++ b/kernel/mips/sscal_msa.c @@ -42,7 +42,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, if (1 == inc_x) { - if (0.0 == da) + if (0) // if (0.0 == da) { v4f32 zero_v = {0.0, 0.0, 0.0, 0.0}; @@ -259,7 +259,10 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, { for (i = n; i--;) { - *x = 0; + if (isfinite(*x) + *x = 0; + else + *x = NAN; x += inc_x; } } From a11f086c1717c074c34a9232f49338ed94f81b57 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sun, 23 Jun 2024 12:55:19 +0200 Subject: [PATCH 151/267] Update sscal_msa.c --- kernel/mips/sscal_msa.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/mips/sscal_msa.c b/kernel/mips/sscal_msa.c index a1c7df4528..bfd477b6a5 100644 --- a/kernel/mips/sscal_msa.c +++ b/kernel/mips/sscal_msa.c @@ -259,7 +259,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, { for (i = n; i--;) { - if (isfinite(*x) + if (isfinite(*x)) *x = 0; else *x = NAN; From 9e24121e7e3bb983a468e115e514c408c9fb0cb7 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sun, 23 Jun 2024 17:48:18 +0200 Subject: [PATCH 152/267] temporarily(?) disable da=0 shortcut to handle x=Inf or NAN --- kernel/x86/scal.S | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/x86/scal.S b/kernel/x86/scal.S index a24c832d35..b0c232b1bd 100644 --- a/kernel/x86/scal.S +++ b/kernel/x86/scal.S @@ -68,12 +68,12 @@ ftst fnstsw %ax andb $68, %ah - je .L300 # Alpha != ZERO +// je .L300 # Alpha != ZERO + jmp .L300 /* Alpha == ZERO */ -// cmpl $1,%esi -// jne .L104 - jmp .L104 + cmpl $1,%esi + jne .L104 movl %edx, %ecx # ecx = n sarl $3, %ecx # (n >> 3) From 8c472ef7e30d019776dde92b96b02f4c998f7487 Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Mon, 24 Jun 2024 10:47:47 +0100 Subject: [PATCH 153/267] Further tweak small GEMM for AArch64 --- kernel/arm64/dgemm_small_kernel_nn_sve.c | 134 ++- kernel/arm64/dgemm_small_kernel_nt_sve.c | 169 ++-- kernel/arm64/dgemm_small_kernel_tn_sve.c | 137 ++- kernel/arm64/dgemm_small_kernel_tt_sve.c | 137 ++- kernel/arm64/gemm_small_kernel_permit_sve.c | 12 +- kernel/arm64/sgemm_small_kernel_nn_sve.c | 673 ++----------- kernel/arm64/sgemm_small_kernel_nt_sve.c | 406 ++------ kernel/arm64/sgemm_small_kernel_tn_sve.c | 998 ++------------------ kernel/arm64/sgemm_small_kernel_tt_sve.c | 390 ++------ 9 files changed, 771 insertions(+), 2285 deletions(-) diff --git a/kernel/arm64/dgemm_small_kernel_nn_sve.c b/kernel/arm64/dgemm_small_kernel_nn_sve.c index 8baef82775..4176334713 100644 --- a/kernel/arm64/dgemm_small_kernel_nn_sve.c +++ b/kernel/arm64/dgemm_small_kernel_nn_sve.c @@ -46,13 +46,27 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. }) #endif -#define A_ELEMENT_K(m, offset_k) A[(i + (m)) + (k + offset_k) * lda] +#define RESET_A_POINTER() a_offset = A; + +#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale; +#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale; +#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k) * lda) #define A_ELEMENT(m) A_ELEMENT_K(m, 0) -#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) + (j + (n)) * ldb] +#define RESET_B_POINTER() b_offset = B; + +#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale * ldb; +#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale * ldb; +#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k)) #define B_ELEMENT(n) B_ELEMENT_K(n, 0) -#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc] +#define CREATE_C_POINTER(n, scale) FLOAT* c_offset##n = c_offset + scale * ldc; +#define INCR_C_POINTER(m, incr) // c_offset ## m += incr; +#define UPDATE_C_POINTER(scale) c_offset = c_offset + scale * ldc; +#define C_ELEMENT(m, n) *(c_offset##n + ((m * v_size) + i)) + +// #undef C_ELEMENT +// #define C_ELEMENT(m, n) C[(i+(m))+(j+(n))*ldc] #define PACK_ELEMENT_K(n, offset_k) packed_b[(k + offset_k) * 4 + n] #define PACK_ELEMENT(n) PACK_ELEMENT_K(n, 0) @@ -112,8 +126,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define BROADCAST_LOAD_B(n, offset_k) \ svfloat64_t b##s##n##_k##offset_k = svdup_f64(B_ELEMENT_K(n, offset_k)); #define VECTOR_LOAD_A(pg, m, offset_k) \ - svfloat64_t a##s##m##_k##offset_k = \ - svld1(pg, &A_ELEMENT_K(v_size * m, offset_k)); + svfloat64_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k)); #define QUADWORD_LOAD_B(n, offset_k) \ svfloat64_t b##s##n##_k##offset_k = \ svld1rq(pg_true, &B_ELEMENT_K(n, offset_k)); @@ -140,26 +153,23 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifdef B0 #define VECTOR_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); + svst1(pg, &C_ELEMENT(m, n), result##m##n); #define SCATTER_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - svst1_scatter_index( \ - pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n); + svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n); #else #define VECTOR_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ result##m##n = \ - svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(v_size * m, n)), beta_vec); \ - svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); + svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec); \ + svst1(pg, &C_ELEMENT(m, n), result##m##n); #define SCATTER_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - result##m##n = svmla_m( \ - pg, \ - result##m##n, \ - svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u64(0LL, ldc)), \ - beta_vec); \ - svst1_scatter_index( \ - pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n); + result##m##n = svmla_m(pg, \ + result##m##n, \ + svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec), \ + beta_vec); \ + svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n); #endif #ifndef LIKELY @@ -169,13 +179,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define LIKELY(x) (x) #endif #endif -#ifndef UNLIKELY -#ifdef __GNUC__ -#define UNLIKELY(x) __builtin_expect(!!(x), 0) -#else -#define UNLIKELY(x) (x) -#endif -#endif #ifdef B0 int @@ -223,12 +226,29 @@ CNAME(BLASLONG M, FLOAT* packed_b = (pack_b) ? packed_b = (FLOAT*)malloc(K * 4 * sizeof(FLOAT)) : NULL; + FLOAT* b_offset = B; + FLOAT* a_offset = A; + FLOAT* c_offset = C; + BLASLONG j = 0; for (; j < n4; j += 4) { + CREATE_C_POINTER(0, 0); + CREATE_C_POINTER(1, 1); + CREATE_C_POINTER(2, 2); + CREATE_C_POINTER(3, 3); + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + CREATE_B_POINTER(2, 2); + CREATE_B_POINTER(3, 3); + BLASLONG i = 0; for (; i < v_m2; i += v_size2) { + CREATE_A_POINTER(0, 0); + CREATE_A_POINTER(1, v_size); + UPDATE_A_POINTER(v_size2); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); @@ -372,9 +392,16 @@ CNAME(BLASLONG M, VECTOR_STORE(pg_true, 1, 1); VECTOR_STORE(pg_true, 1, 2); VECTOR_STORE(pg_true, 1, 3); + INCR_C_POINTER(0, v_size2); + INCR_C_POINTER(1, v_size2); + INCR_C_POINTER(2, v_size2); + INCR_C_POINTER(3, v_size2); } for (; i < v_m1; i += v_size) { + CREATE_A_POINTER(0, 0); + UPDATE_A_POINTER(v_size); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); @@ -431,9 +458,15 @@ CNAME(BLASLONG M, VECTOR_STORE(pg_true, 0, 1); VECTOR_STORE(pg_true, 0, 2); VECTOR_STORE(pg_true, 0, 3); + INCR_C_POINTER(0, v_size); + INCR_C_POINTER(1, v_size); + INCR_C_POINTER(2, v_size); + INCR_C_POINTER(3, v_size); } for (; i < M; i += v_size) { const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M)); + CREATE_A_POINTER(0, 0); + UPDATE_A_POINTER(0); BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); @@ -491,13 +524,30 @@ CNAME(BLASLONG M, VECTOR_STORE(pg_tail, 0, 1); VECTOR_STORE(pg_tail, 0, 2); VECTOR_STORE(pg_tail, 0, 3); + INCR_C_POINTER(0, 0); + INCR_C_POINTER(1, 0); + INCR_C_POINTER(2, 0); + INCR_C_POINTER(3, 0); } + + UPDATE_B_POINTER(4); + RESET_A_POINTER(); + UPDATE_C_POINTER(4); } for (; j < n2; j += 2) { + CREATE_C_POINTER(0, 0); + CREATE_C_POINTER(1, 1); + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + BLASLONG i = 0; for (; i < v_m2; i += v_size2) { + CREATE_A_POINTER(0, 0); + CREATE_A_POINTER(1, v_size); + UPDATE_A_POINTER(v_size2); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); @@ -538,9 +588,14 @@ CNAME(BLASLONG M, VECTOR_STORE(pg_true, 0, 1); VECTOR_STORE(pg_true, 1, 0); VECTOR_STORE(pg_true, 1, 1); + INCR_C_POINTER(0, v_size2); + INCR_C_POINTER(1, v_size2); } for (; i < v_m1; i += v_size) { + CREATE_A_POINTER(0, 0); + UPDATE_A_POINTER(v_size); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); @@ -568,9 +623,13 @@ CNAME(BLASLONG M, } VECTOR_STORE(pg_true, 0, 0); VECTOR_STORE(pg_true, 0, 1); + INCR_C_POINTER(0, v_size); + INCR_C_POINTER(1, v_size); } for (; i < M; i += v_size) { const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M)); + CREATE_A_POINTER(0, 0); + UPDATE_A_POINTER(0); BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); @@ -599,13 +658,26 @@ CNAME(BLASLONG M, } VECTOR_STORE(pg_tail, 0, 0); VECTOR_STORE(pg_tail, 0, 1); + INCR_C_POINTER(0, 0); + INCR_C_POINTER(1, 0); } + + UPDATE_B_POINTER(2); + RESET_A_POINTER(); + UPDATE_C_POINTER(2); } for (; j < N; j++) { + CREATE_C_POINTER(0, 0); + CREATE_B_POINTER(0, 0); + BLASLONG i = 0; for (; i < v_m2; i += v_size2) { + CREATE_A_POINTER(0, 0); + CREATE_A_POINTER(1, v_size); + UPDATE_A_POINTER(v_size2); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(1, 0); @@ -620,9 +692,13 @@ CNAME(BLASLONG M, } VECTOR_STORE(pg_true, 0, 0); VECTOR_STORE(pg_true, 1, 0); + INCR_C_POINTER(0, v_size2); } for (; i < v_m1; i += v_size) { + CREATE_A_POINTER(0, 0); + UPDATE_A_POINTER(v_size); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); @@ -633,9 +709,12 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); } VECTOR_STORE(pg_true, 0, 0); + INCR_C_POINTER(0, v_size); } for (; i < M; i += v_size) { const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M)); + CREATE_A_POINTER(0, 0); + UPDATE_A_POINTER(0); BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); @@ -647,11 +726,16 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); } VECTOR_STORE(pg_tail, 0, 0); + INCR_C_POINTER(0, 0); } + + UPDATE_B_POINTER(1); + RESET_A_POINTER(); + UPDATE_C_POINTER(1); } if (pack_b) free(packed_b); return 0; -} +} \ No newline at end of file diff --git a/kernel/arm64/dgemm_small_kernel_nt_sve.c b/kernel/arm64/dgemm_small_kernel_nt_sve.c index 982388287f..241d96a6c9 100644 --- a/kernel/arm64/dgemm_small_kernel_nt_sve.c +++ b/kernel/arm64/dgemm_small_kernel_nt_sve.c @@ -46,13 +46,27 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. }) #endif -#define A_ELEMENT_K(m, offset_k) A[(i + (m)) + (k + offset_k) * lda] +#define RESET_A_POINTER() a_offset = A; + +#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale; +#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale; +#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k) * lda) #define A_ELEMENT(m) A_ELEMENT_K(m, 0) -#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) * ldb + (j + (n))] +#define RESET_B_POINTER() b_offset = B; + +#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale; +#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale; +#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k) * ldb) #define B_ELEMENT(n) B_ELEMENT_K(n, 0) -#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc] +#define CREATE_C_POINTER(n, scale) FLOAT* c_offset##n = c_offset + scale * ldc; +#define INCR_C_POINTER(m, incr) // c_offset ## m += incr; +#define UPDATE_C_POINTER(scale) c_offset = c_offset + scale * ldc; +#define C_ELEMENT(m, n) *(c_offset##n + ((m * v_size) + i)) + +// #undef C_ELEMENT +// #define C_ELEMENT(m, n) C[(i+(m))+(j+(n))*ldc] #define PACK_ELEMENT_K(n, offset_k) packed_b[(k + offset_k) * 4 + n] #define PACK_ELEMENT(n) PACK_ELEMENT_K(n, 0) @@ -97,8 +111,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define BROADCAST_LOAD_B(n, offset_k) \ svfloat64_t b##s##n##_k##offset_k = svdup_f64(B_ELEMENT_K(n, offset_k)); #define VECTOR_LOAD_A(pg, m, offset_k) \ - svfloat64_t a##s##m##_k##offset_k = \ - svld1(pg, &A_ELEMENT_K(v_size * m, offset_k)); + svfloat64_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k)); #define QUADWORD_LOAD_B(n, offset_k) \ svfloat64_t b##s##n##_k##offset_k = \ svld1rq(pg_true, &B_ELEMENT_K(n, offset_k)); @@ -111,26 +124,23 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifdef B0 #define VECTOR_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); + svst1(pg, &C_ELEMENT(m, n), result##m##n); #define SCATTER_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - svst1_scatter_index( \ - pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n); + svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n); #else #define VECTOR_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ result##m##n = \ - svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(v_size * m, n)), beta_vec); \ - svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); + svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec); \ + svst1(pg, &C_ELEMENT(m, n), result##m##n); #define SCATTER_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - result##m##n = svmla_m( \ - pg, \ - result##m##n, \ - svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u64(0LL, ldc)), \ - beta_vec); \ - svst1_scatter_index( \ - pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n); + result##m##n = svmla_m(pg, \ + result##m##n, \ + svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec), \ + beta_vec); \ + svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n); #endif #ifndef LIKELY @@ -140,13 +150,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define LIKELY(x) (x) #endif #endif -#ifndef UNLIKELY -#ifdef __GNUC__ -#define UNLIKELY(x) __builtin_expect(!!(x), 0) -#else -#define UNLIKELY(x) (x) -#endif -#endif #ifdef B0 int @@ -176,8 +179,7 @@ CNAME(BLASLONG M, #endif { const uint64_t v_size = svcntd(); - const uint64_t v_size32 = v_size * 32; - const uint64_t v_size3 = v_size * 3; + const uint64_t v_size2 = v_size * 2; const svbool_t pg_true = svptrue_b64(); const svbool_t pg_quad = svwhilelt_b64(0, 2); const svfloat64_t alpha_vec = svdup_f64(alpha); @@ -186,14 +188,31 @@ CNAME(BLASLONG M, #endif const BLASLONG n4 = N & -4; const BLASLONG n2 = N & -2; - const BLASLONG v_m3 = M - (M % v_size3); + const BLASLONG v_m2 = M & -v_size2; const BLASLONG v_m1 = M & -v_size; + FLOAT* b_offset = B; + FLOAT* a_offset = A; + FLOAT* c_offset = C; + BLASLONG j = 0; for (; j < n4; j += 4) { + CREATE_C_POINTER(0, 0); + CREATE_C_POINTER(1, 1); + CREATE_C_POINTER(2, 2); + CREATE_C_POINTER(3, 3); + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + CREATE_B_POINTER(2, 2); + CREATE_B_POINTER(3, 3); + BLASLONG i = 0; - for (; i < v_m3; i += v_size3) { + for (; i < v_m2; i += v_size2) { + + CREATE_A_POINTER(0, 0); + CREATE_A_POINTER(1, v_size); + UPDATE_A_POINTER(v_size2); BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); @@ -204,10 +223,6 @@ CNAME(BLASLONG M, DECLARE_RESULT_VECTOR(1, 1); DECLARE_RESULT_VECTOR(1, 2); DECLARE_RESULT_VECTOR(1, 3); - DECLARE_RESULT_VECTOR(2, 0); - DECLARE_RESULT_VECTOR(2, 1); - DECLARE_RESULT_VECTOR(2, 2); - DECLARE_RESULT_VECTOR(2, 3); for (; k < K; k++) { @@ -223,11 +238,6 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0); UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0); - VECTOR_LOAD_A(pg_true, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 2, 2, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 3, 2, 1, 0); } VECTOR_STORE(pg_true, 0, 0); VECTOR_STORE(pg_true, 0, 1); @@ -237,13 +247,16 @@ CNAME(BLASLONG M, VECTOR_STORE(pg_true, 1, 1); VECTOR_STORE(pg_true, 1, 2); VECTOR_STORE(pg_true, 1, 3); - VECTOR_STORE(pg_true, 2, 0); - VECTOR_STORE(pg_true, 2, 1); - VECTOR_STORE(pg_true, 2, 2); - VECTOR_STORE(pg_true, 2, 3); + INCR_C_POINTER(0, v_size2); + INCR_C_POINTER(1, v_size2); + INCR_C_POINTER(2, v_size2); + INCR_C_POINTER(3, v_size2); } for (; i < v_m1; i += v_size) { + CREATE_A_POINTER(0, 0); + UPDATE_A_POINTER(v_size); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); @@ -264,9 +277,15 @@ CNAME(BLASLONG M, VECTOR_STORE(pg_true, 0, 1); VECTOR_STORE(pg_true, 0, 2); VECTOR_STORE(pg_true, 0, 3); + INCR_C_POINTER(0, v_size); + INCR_C_POINTER(1, v_size); + INCR_C_POINTER(2, v_size); + INCR_C_POINTER(3, v_size); } for (; i < M; i += v_size) { const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M)); + CREATE_A_POINTER(0, 0); + UPDATE_A_POINTER(0); BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); @@ -288,20 +307,35 @@ CNAME(BLASLONG M, VECTOR_STORE(pg_tail, 0, 1); VECTOR_STORE(pg_tail, 0, 2); VECTOR_STORE(pg_tail, 0, 3); + INCR_C_POINTER(0, 0); + INCR_C_POINTER(1, 0); + INCR_C_POINTER(2, 0); + INCR_C_POINTER(3, 0); } + + UPDATE_B_POINTER(4); + RESET_A_POINTER(); + UPDATE_C_POINTER(4); } for (; j < n2; j += 2) { + CREATE_C_POINTER(0, 0); + CREATE_C_POINTER(1, 1); + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + BLASLONG i = 0; - for (; i < v_m3; i += v_size3) { + for (; i < v_m2; i += v_size2) { + + CREATE_A_POINTER(0, 0); + CREATE_A_POINTER(1, v_size); + UPDATE_A_POINTER(v_size2); BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); DECLARE_RESULT_VECTOR(1, 0); DECLARE_RESULT_VECTOR(1, 1); - DECLARE_RESULT_VECTOR(2, 0); - DECLARE_RESULT_VECTOR(2, 1); for (; k < K; k++) { @@ -312,19 +346,19 @@ CNAME(BLASLONG M, VECTOR_LOAD_A(pg_true, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - VECTOR_LOAD_A(pg_true, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 1, 0, 1, 0); } VECTOR_STORE(pg_true, 0, 0); VECTOR_STORE(pg_true, 0, 1); VECTOR_STORE(pg_true, 1, 0); VECTOR_STORE(pg_true, 1, 1); - VECTOR_STORE(pg_true, 2, 0); - VECTOR_STORE(pg_true, 2, 1); + INCR_C_POINTER(0, v_size2); + INCR_C_POINTER(1, v_size2); } for (; i < v_m1; i += v_size) { + CREATE_A_POINTER(0, 0); + UPDATE_A_POINTER(v_size); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); @@ -338,9 +372,13 @@ CNAME(BLASLONG M, } VECTOR_STORE(pg_true, 0, 0); VECTOR_STORE(pg_true, 0, 1); + INCR_C_POINTER(0, v_size); + INCR_C_POINTER(1, v_size); } for (; i < M; i += v_size) { const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M)); + CREATE_A_POINTER(0, 0); + UPDATE_A_POINTER(0); BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); @@ -355,17 +393,29 @@ CNAME(BLASLONG M, } VECTOR_STORE(pg_tail, 0, 0); VECTOR_STORE(pg_tail, 0, 1); + INCR_C_POINTER(0, 0); + INCR_C_POINTER(1, 0); } + + UPDATE_B_POINTER(2); + RESET_A_POINTER(); + UPDATE_C_POINTER(2); } for (; j < N; j++) { + CREATE_C_POINTER(0, 0); + CREATE_B_POINTER(0, 0); + BLASLONG i = 0; - for (; i < v_m3; i += v_size3) { + for (; i < v_m2; i += v_size2) { + + CREATE_A_POINTER(0, 0); + CREATE_A_POINTER(1, v_size); + UPDATE_A_POINTER(v_size2); BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(1, 0); - DECLARE_RESULT_VECTOR(2, 0); for (; k < K; k++) { @@ -374,15 +424,16 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); VECTOR_LOAD_A(pg_true, 1, 0); UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); - VECTOR_LOAD_A(pg_true, 2, 0); - UPDATE_RESULT_VECTOR(pg_true, 2, 0, 0); } VECTOR_STORE(pg_true, 0, 0); VECTOR_STORE(pg_true, 1, 0); - VECTOR_STORE(pg_true, 2, 0); + INCR_C_POINTER(0, v_size2); } for (; i < v_m1; i += v_size) { + CREATE_A_POINTER(0, 0); + UPDATE_A_POINTER(v_size); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); @@ -393,9 +444,12 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); } VECTOR_STORE(pg_true, 0, 0); + INCR_C_POINTER(0, v_size); } for (; i < M; i += v_size) { const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M)); + CREATE_A_POINTER(0, 0); + UPDATE_A_POINTER(0); BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); @@ -407,8 +461,13 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); } VECTOR_STORE(pg_tail, 0, 0); + INCR_C_POINTER(0, 0); } + + UPDATE_B_POINTER(1); + RESET_A_POINTER(); + UPDATE_C_POINTER(1); } return 0; -} +} \ No newline at end of file diff --git a/kernel/arm64/dgemm_small_kernel_tn_sve.c b/kernel/arm64/dgemm_small_kernel_tn_sve.c index 7158851da3..b8783c1d53 100644 --- a/kernel/arm64/dgemm_small_kernel_tn_sve.c +++ b/kernel/arm64/dgemm_small_kernel_tn_sve.c @@ -46,15 +46,30 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. }) #endif -#define A_ELEMENT_K(m, offset_k) A[(i + (m)) * lda + (k + offset_k)] +#define RESET_A_POINTER() a_offset = A; + +#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale * lda; +#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale * lda; +#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k)) #define A_ELEMENT(m) A_ELEMENT_K(m, 0) -#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) + (j + (n)) * ldb] +#define RESET_B_POINTER() b_offset = B; + +#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale * ldb; +#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale * ldb; +#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k)) #define B_ELEMENT(n) B_ELEMENT_K(n, 0) -#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc] +#define CREATE_C_POINTER(m, scale) FLOAT* c_offset##m = c_offset + scale; +#define INCR_C_POINTER(m, incr) // c_offset ## m += incr * ldc; +#define UPDATE_C_POINTER(scale) c_offset += scale; +#define C_ELEMENT(m, n) \ + *(c_offset##m + ((j + n) * ldc)) // C[(i+(m))+(j+(n))*ldc] + +// #undef C_ELEMENT +// #define C_ELEMENT(m, n) C[(i+(m))+(j+(n))*ldc] -#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * 2 * v_size + m] +#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * v_size2 + m] #define PACK_ELEMENT(m) PACK_ELEMENT_K(m, 0) // ASIMD @@ -112,14 +127,13 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define BROADCAST_LOAD_B(n, offset_k) \ svfloat64_t b##s##n##_k##offset_k = svdup_f64(B_ELEMENT_K(n, offset_k)); #define VECTOR_LOAD_A(pg, m, offset_k) \ - svfloat64_t a##s##m##_k##offset_k = \ - svld1(pg, &A_ELEMENT_K(v_size * m, offset_k)); + svfloat64_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k)); #define QUADWORD_LOAD_B(n, offset_k) \ svfloat64_t b##s##n##_k##offset_k = \ svld1rq(pg_true, &B_ELEMENT_K(n, offset_k)); #define GATHER_LOAD_A(pg, m, offset_k) \ svfloat64_t a##s##m##_k##offset_k = \ - svld1_gather_index(pg, &A_ELEMENT_K(v_size * m, offset_k), lda_vec); + svld1_gather_index(pg, &A_ELEMENT_K(m, offset_k), lda_vec); #define PACK_A(m, offset_k) \ svst1(pg_first, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k); #define VECTOR_PACK_A(m, offset_k) \ @@ -143,26 +157,23 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifdef B0 #define VECTOR_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); + svst1(pg, &C_ELEMENT(m, n), result##m##n); #define SCATTER_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - svst1_scatter_index( \ - pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n); + svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n); #else #define VECTOR_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ result##m##n = \ - svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(v_size * m, n)), beta_vec); \ - svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); + svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec); \ + svst1(pg, &C_ELEMENT(m, n), result##m##n); #define SCATTER_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - result##m##n = svmla_m( \ - pg, \ - result##m##n, \ - svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u64(0LL, ldc)), \ - beta_vec); \ - svst1_scatter_index( \ - pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n); + result##m##n = svmla_m(pg, \ + result##m##n, \ + svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec), \ + beta_vec); \ + svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n); #endif #ifndef LIKELY @@ -172,13 +183,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define LIKELY(x) (x) #endif #endif -#ifndef UNLIKELY -#ifdef __GNUC__ -#define UNLIKELY(x) __builtin_expect(!!(x), 0) -#else -#define UNLIKELY(x) (x) -#endif -#endif #ifdef B0 int @@ -226,14 +230,29 @@ CNAME(BLASLONG M, const int pack_a = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0; FLOAT* packed_a = - (pack_a) ? packed_a = (FLOAT*)malloc(K * 2 * v_size * sizeof(FLOAT)) : NULL; + (pack_a) ? packed_a = (FLOAT*)malloc(K * v_size2 * sizeof(FLOAT)) : NULL; + + FLOAT* a_offset = A; + FLOAT* b_offset = B; + FLOAT* c_offset = C; BLASLONG i = 0; for (; i < v_m2; i += v_size2) { + CREATE_C_POINTER(0, 0); + CREATE_C_POINTER(1, v_size); + CREATE_A_POINTER(0, 0); + CREATE_A_POINTER(1, v_size); + BLASLONG j = 0; for (; j < n4; j += 4) { + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + CREATE_B_POINTER(2, 2); + CREATE_B_POINTER(3, 3); + UPDATE_B_POINTER(4); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); @@ -408,9 +427,15 @@ CNAME(BLASLONG M, VECTOR_STORE(pg_true, 1, 1); VECTOR_STORE(pg_true, 1, 2); VECTOR_STORE(pg_true, 1, 3); + INCR_C_POINTER(0, 4); + INCR_C_POINTER(1, 4); } for (; j < n2; j += 2) { + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + UPDATE_B_POINTER(2); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); @@ -484,9 +509,14 @@ CNAME(BLASLONG M, VECTOR_STORE(pg_true, 0, 1); VECTOR_STORE(pg_true, 1, 0); VECTOR_STORE(pg_true, 1, 1); + INCR_C_POINTER(0, 2); + INCR_C_POINTER(1, 2); } for (; j < N; j++) { + CREATE_B_POINTER(0, 0); + UPDATE_B_POINTER(1); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(1, 0); @@ -512,13 +542,28 @@ CNAME(BLASLONG M, } VECTOR_STORE(pg_true, 0, 0); VECTOR_STORE(pg_true, 1, 0); + INCR_C_POINTER(0, 1); + INCR_C_POINTER(1, 1); } + + UPDATE_A_POINTER(v_size2); + RESET_B_POINTER(); + UPDATE_C_POINTER(v_size2); } for (; i < v_m1; i += v_size) { + CREATE_C_POINTER(0, 0); + CREATE_A_POINTER(0, 0); + BLASLONG j = 0; for (; j < n4; j += 4) { + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + CREATE_B_POINTER(2, 2); + CREATE_B_POINTER(3, 3); + UPDATE_B_POINTER(4); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); @@ -562,9 +607,14 @@ CNAME(BLASLONG M, VECTOR_STORE(pg_true, 0, 1); VECTOR_STORE(pg_true, 0, 2); VECTOR_STORE(pg_true, 0, 3); + INCR_C_POINTER(0, 4); } for (; j < n2; j += 2) { + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + UPDATE_B_POINTER(2); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); @@ -592,9 +642,13 @@ CNAME(BLASLONG M, } VECTOR_STORE(pg_true, 0, 0); VECTOR_STORE(pg_true, 0, 1); + INCR_C_POINTER(0, 2); } for (; j < N; j++) { + CREATE_B_POINTER(0, 0); + UPDATE_B_POINTER(1); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); @@ -605,14 +659,27 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); } VECTOR_STORE(pg_true, 0, 0); + INCR_C_POINTER(0, 1); } + + UPDATE_A_POINTER(v_size); + RESET_B_POINTER(); + UPDATE_C_POINTER(v_size); } for (; i < M; i += v_size) { const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M)); + CREATE_C_POINTER(0, 0); + CREATE_A_POINTER(0, 0); BLASLONG j = 0; for (; j < n4; j += 4) { + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + CREATE_B_POINTER(2, 2); + CREATE_B_POINTER(3, 3); + UPDATE_B_POINTER(4); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); @@ -656,9 +723,14 @@ CNAME(BLASLONG M, VECTOR_STORE(pg_tail, 0, 1); VECTOR_STORE(pg_tail, 0, 2); VECTOR_STORE(pg_tail, 0, 3); + INCR_C_POINTER(0, 4); } for (; j < n2; j += 2) { + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + UPDATE_B_POINTER(2); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); @@ -686,9 +758,13 @@ CNAME(BLASLONG M, } VECTOR_STORE(pg_tail, 0, 0); VECTOR_STORE(pg_tail, 0, 1); + INCR_C_POINTER(0, 2); } for (; j < N; j++) { + CREATE_B_POINTER(0, 0); + UPDATE_B_POINTER(1); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); @@ -699,11 +775,16 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); } VECTOR_STORE(pg_tail, 0, 0); + INCR_C_POINTER(0, 1); } + + UPDATE_A_POINTER(0); + RESET_B_POINTER(); + UPDATE_C_POINTER(0); } if (pack_a) free(packed_a); return 0; -} +} \ No newline at end of file diff --git a/kernel/arm64/dgemm_small_kernel_tt_sve.c b/kernel/arm64/dgemm_small_kernel_tt_sve.c index 12fc0b59e4..aa5bf27512 100644 --- a/kernel/arm64/dgemm_small_kernel_tt_sve.c +++ b/kernel/arm64/dgemm_small_kernel_tt_sve.c @@ -46,15 +46,30 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. }) #endif -#define A_ELEMENT_K(m, offset_k) A[(i + (m)) * lda + (k + offset_k)] +#define RESET_A_POINTER() a_offset = A; + +#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale * lda; +#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale * lda; +#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k)) #define A_ELEMENT(m) A_ELEMENT_K(m, 0) -#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) * ldb + (j + (n))] +#define RESET_B_POINTER() b_offset = B; + +#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale; +#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale; +#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k) * ldb) #define B_ELEMENT(n) B_ELEMENT_K(n, 0) -#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc] +#define CREATE_C_POINTER(m, scale) FLOAT* c_offset##m = c_offset + scale; +#define INCR_C_POINTER(m, incr) // c_offset ## m += incr * ldc; +#define UPDATE_C_POINTER(scale) c_offset += scale; +#define C_ELEMENT(m, n) \ + *(c_offset##m + ((j + n) * ldc)) // C[(i+(m))+(j+(n))*ldc] + +// #undef C_ELEMENT +// #define C_ELEMENT(m, n) C[(i+(m))+(j+(n))*ldc] -#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * 2 * v_size + m] +#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * v_size2 + m] #define PACK_ELEMENT(m) PACK_ELEMENT_K(m, 0) // ASIMD @@ -103,14 +118,13 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define BROADCAST_LOAD_B(n, offset_k) \ svfloat64_t b##s##n##_k##offset_k = svdup_f64(B_ELEMENT_K(n, offset_k)); #define VECTOR_LOAD_A(pg, m, offset_k) \ - svfloat64_t a##s##m##_k##offset_k = \ - svld1(pg, &A_ELEMENT_K(v_size * m, offset_k)); + svfloat64_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k)); #define QUADWORD_LOAD_B(n, offset_k) \ svfloat64_t b##s##n##_k##offset_k = \ svld1rq(pg_true, &B_ELEMENT_K(n, offset_k)); #define GATHER_LOAD_A(pg, m, offset_k) \ svfloat64_t a##s##m##_k##offset_k = \ - svld1_gather_index(pg, &A_ELEMENT_K(v_size * m, offset_k), lda_vec); + svld1_gather_index(pg, &A_ELEMENT_K(m, offset_k), lda_vec); #define PACK_A(m, offset_k) \ svst1(pg_first, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k); #define VECTOR_PACK_A(m, offset_k) \ @@ -134,26 +148,23 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifdef B0 #define VECTOR_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); + svst1(pg, &C_ELEMENT(m, n), result##m##n); #define SCATTER_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - svst1_scatter_index( \ - pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n); + svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n); #else #define VECTOR_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ result##m##n = \ - svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(v_size * m, n)), beta_vec); \ - svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); + svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec); \ + svst1(pg, &C_ELEMENT(m, n), result##m##n); #define SCATTER_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - result##m##n = svmla_m( \ - pg, \ - result##m##n, \ - svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u64(0LL, ldc)), \ - beta_vec); \ - svst1_scatter_index( \ - pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n); + result##m##n = svmla_m(pg, \ + result##m##n, \ + svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec), \ + beta_vec); \ + svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n); #endif #ifndef LIKELY @@ -163,13 +174,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define LIKELY(x) (x) #endif #endif -#ifndef UNLIKELY -#ifdef __GNUC__ -#define UNLIKELY(x) __builtin_expect(!!(x), 0) -#else -#define UNLIKELY(x) (x) -#endif -#endif #ifdef B0 int @@ -216,14 +220,29 @@ CNAME(BLASLONG M, const int pack_a = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0; FLOAT* packed_a = - (pack_a) ? packed_a = (FLOAT*)malloc(K * 2 * v_size * sizeof(FLOAT)) : NULL; + (pack_a) ? packed_a = (FLOAT*)malloc(K * v_size2 * sizeof(FLOAT)) : NULL; + + FLOAT* a_offset = A; + FLOAT* b_offset = B; + FLOAT* c_offset = C; BLASLONG i = 0; for (; i < v_m2; i += v_size2) { + CREATE_C_POINTER(0, 0); + CREATE_C_POINTER(1, v_size); + CREATE_A_POINTER(0, 0); + CREATE_A_POINTER(1, v_size); + BLASLONG j = 0; for (; j < n4; j += 4) { + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + CREATE_B_POINTER(2, 2); + CREATE_B_POINTER(3, 3); + UPDATE_B_POINTER(4); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); @@ -295,9 +314,15 @@ CNAME(BLASLONG M, VECTOR_STORE(pg_true, 1, 1); VECTOR_STORE(pg_true, 1, 2); VECTOR_STORE(pg_true, 1, 3); + INCR_C_POINTER(0, 4); + INCR_C_POINTER(1, 4); } for (; j < n2; j += 2) { + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + UPDATE_B_POINTER(2); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); @@ -331,9 +356,14 @@ CNAME(BLASLONG M, VECTOR_STORE(pg_true, 0, 1); VECTOR_STORE(pg_true, 1, 0); VECTOR_STORE(pg_true, 1, 1); + INCR_C_POINTER(0, 2); + INCR_C_POINTER(1, 2); } for (; j < N; j++) { + CREATE_B_POINTER(0, 0); + UPDATE_B_POINTER(1); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(1, 0); @@ -359,13 +389,28 @@ CNAME(BLASLONG M, } VECTOR_STORE(pg_true, 0, 0); VECTOR_STORE(pg_true, 1, 0); + INCR_C_POINTER(0, 1); + INCR_C_POINTER(1, 1); } + + UPDATE_A_POINTER(v_size2); + RESET_B_POINTER(); + UPDATE_C_POINTER(v_size2); } for (; i < v_m1; i += v_size) { + CREATE_C_POINTER(0, 0); + CREATE_A_POINTER(0, 0); + BLASLONG j = 0; for (; j < n4; j += 4) { + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + CREATE_B_POINTER(2, 2); + CREATE_B_POINTER(3, 3); + UPDATE_B_POINTER(4); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); @@ -386,9 +431,14 @@ CNAME(BLASLONG M, VECTOR_STORE(pg_true, 0, 1); VECTOR_STORE(pg_true, 0, 2); VECTOR_STORE(pg_true, 0, 3); + INCR_C_POINTER(0, 4); } for (; j < n2; j += 2) { + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + UPDATE_B_POINTER(2); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); @@ -402,9 +452,13 @@ CNAME(BLASLONG M, } VECTOR_STORE(pg_true, 0, 0); VECTOR_STORE(pg_true, 0, 1); + INCR_C_POINTER(0, 2); } for (; j < N; j++) { + CREATE_B_POINTER(0, 0); + UPDATE_B_POINTER(1); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); @@ -415,14 +469,27 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); } VECTOR_STORE(pg_true, 0, 0); + INCR_C_POINTER(0, 1); } + + UPDATE_A_POINTER(v_size); + RESET_B_POINTER(); + UPDATE_C_POINTER(v_size); } for (; i < M; i += v_size) { const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M)); + CREATE_C_POINTER(0, 0); + CREATE_A_POINTER(0, 0); BLASLONG j = 0; for (; j < n4; j += 4) { + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + CREATE_B_POINTER(2, 2); + CREATE_B_POINTER(3, 3); + UPDATE_B_POINTER(4); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); @@ -443,9 +510,14 @@ CNAME(BLASLONG M, VECTOR_STORE(pg_tail, 0, 1); VECTOR_STORE(pg_tail, 0, 2); VECTOR_STORE(pg_tail, 0, 3); + INCR_C_POINTER(0, 4); } for (; j < n2; j += 2) { + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + UPDATE_B_POINTER(2); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); @@ -459,9 +531,13 @@ CNAME(BLASLONG M, } VECTOR_STORE(pg_tail, 0, 0); VECTOR_STORE(pg_tail, 0, 1); + INCR_C_POINTER(0, 2); } for (; j < N; j++) { + CREATE_B_POINTER(0, 0); + UPDATE_B_POINTER(1); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); @@ -472,11 +548,16 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); } VECTOR_STORE(pg_tail, 0, 0); + INCR_C_POINTER(0, 1); } + + UPDATE_A_POINTER(0); + RESET_B_POINTER(); + UPDATE_C_POINTER(0); } if (pack_a) free(packed_a); return 0; -} +} \ No newline at end of file diff --git a/kernel/arm64/gemm_small_kernel_permit_sve.c b/kernel/arm64/gemm_small_kernel_permit_sve.c index 9526dbbe22..c1275129d6 100644 --- a/kernel/arm64/gemm_small_kernel_permit_sve.c +++ b/kernel/arm64/gemm_small_kernel_permit_sve.c @@ -32,16 +32,14 @@ int CNAME(int transa, int transb, BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alph BLASLONG MNK = M * N * K; #if defined(DOUBLE) // dgemm - // TN prefers full copies much earlier - if (transa && !transb && MNK > 16*16*16) { - return 0; - } + if (MNK <= 64*64*64) + return 1; #else // sgemm - // TODO! + if (MNK <= 256*256*256) + return 1; #endif - if (MNK <= 64*64*64) - return 1; + return 0; } diff --git a/kernel/arm64/sgemm_small_kernel_nn_sve.c b/kernel/arm64/sgemm_small_kernel_nn_sve.c index 85c7cfa868..2e65e61ff3 100644 --- a/kernel/arm64/sgemm_small_kernel_nn_sve.c +++ b/kernel/arm64/sgemm_small_kernel_nn_sve.c @@ -46,15 +46,29 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. }) #endif -#define A_ELEMENT_K(m, offset_k) A[(i + (m)) + (k + offset_k) * lda] +#define RESET_A_POINTER() a_offset = A; + +#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale; +#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale; +#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k) * lda) #define A_ELEMENT(m) A_ELEMENT_K(m, 0) -#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) + (j + (n)) * ldb] +#define RESET_B_POINTER() b_offset = B; + +#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale * ldb; +#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale * ldb; +#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k)) #define B_ELEMENT(n) B_ELEMENT_K(n, 0) -#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc] +#define CREATE_C_POINTER(n, scale) FLOAT* c_offset##n = c_offset + scale * ldc; +#define INCR_C_POINTER(m, incr) // c_offset ## m += incr; +#define UPDATE_C_POINTER(scale) c_offset = c_offset + scale * ldc; +#define C_ELEMENT(m, n) *(c_offset##n + ((m * v_size) + i)) + +// #undef C_ELEMENT +// #define C_ELEMENT(m, n) C[(i+(m))+(j+(n))*ldc] -#define PACK_ELEMENT_K(n, offset_k) packed_b[(k + offset_k) * 8 + n] +#define PACK_ELEMENT_K(n, offset_k) packed_b[(k + offset_k) * 4 + n] #define PACK_ELEMENT(n) PACK_ELEMENT_K(n, 0) // ASIMD @@ -141,8 +155,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define BROADCAST_LOAD_B(n, offset_k) \ svfloat32_t b##s##n##_k##offset_k = svdup_f32(B_ELEMENT_K(n, offset_k)); #define VECTOR_LOAD_A(pg, m, offset_k) \ - svfloat32_t a##s##m##_k##offset_k = \ - svld1(pg, &A_ELEMENT_K(v_size * m, offset_k)); + svfloat32_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k)); #define QUADWORD_LOAD_B(n, offset_k) \ svfloat32_t b##s##n##_k##offset_k = \ svld1rq(pg_true, &B_ELEMENT_K(n, offset_k)); @@ -169,26 +182,23 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifdef B0 #define VECTOR_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); + svst1(pg, &C_ELEMENT(m, n), result##m##n); #define SCATTER_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - svst1_scatter_index( \ - pg, &C_ELEMENT(v_size* m, n), svindex_u32(0LL, ldc), result##m##n); + svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n); #else #define VECTOR_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ result##m##n = \ - svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(v_size * m, n)), beta_vec); \ - svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); + svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec); \ + svst1(pg, &C_ELEMENT(m, n), result##m##n); #define SCATTER_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - result##m##n = svmla_m( \ - pg, \ - result##m##n, \ - svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u32(0LL, ldc)), \ - beta_vec); \ - svst1_scatter_index( \ - pg, &C_ELEMENT(v_size* m, n), svindex_u32(0LL, ldc), result##m##n); + result##m##n = svmla_m(pg, \ + result##m##n, \ + svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec), \ + beta_vec); \ + svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n); #endif #ifndef LIKELY @@ -198,13 +208,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define LIKELY(x) (x) #endif #endif -#ifndef UNLIKELY -#ifdef __GNUC__ -#define UNLIKELY(x) __builtin_expect(!!(x), 0) -#else -#define UNLIKELY(x) (x) -#endif -#endif #ifdef B0 int @@ -234,7 +237,6 @@ CNAME(BLASLONG M, #endif { const uint64_t v_size = svcntw(); - const uint64_t v_size2 = v_size * 2; const svbool_t pg_true = svptrue_b32(); const svbool_t pg_quad = svwhilelt_b32(0, 4); const svbool_t pg_first = svwhilelt_b32(0, 1); @@ -242,39 +244,41 @@ CNAME(BLASLONG M, #ifndef B0 const svfloat32_t beta_vec = svdup_f32(beta); #endif - const BLASLONG n8 = N & -8; const BLASLONG n4 = N & -4; - const BLASLONG v_m2 = M & -v_size2; const BLASLONG v_m1 = M & -v_size; const BLASLONG k4 = K & -4; - const int pack_b = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0; + const int pack_b = M >= v_size && N >= 8 && K >= 8 ? 1 : 0; FLOAT* packed_b = - (pack_b) ? packed_b = (FLOAT*)malloc(K * 8 * sizeof(FLOAT)) : NULL; + (pack_b) ? packed_b = (FLOAT*)malloc(K * 4 * sizeof(FLOAT)) : NULL; + + FLOAT* b_offset = B; + FLOAT* a_offset = A; + FLOAT* c_offset = C; BLASLONG j = 0; - for (; j < n8; j += 8) { + for (; j < n4; j += 4) { + + CREATE_C_POINTER(0, 0); + CREATE_C_POINTER(1, 1); + CREATE_C_POINTER(2, 2); + CREATE_C_POINTER(3, 3); + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + CREATE_B_POINTER(2, 2); + CREATE_B_POINTER(3, 3); BLASLONG i = 0; - for (; i < v_m2; i += v_size2) { + for (; i < v_m1; i += v_size) { + + CREATE_A_POINTER(0, 0); + UPDATE_A_POINTER(v_size); BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); DECLARE_RESULT_VECTOR(0, 2); DECLARE_RESULT_VECTOR(0, 3); - DECLARE_RESULT_VECTOR(0, 4); - DECLARE_RESULT_VECTOR(0, 5); - DECLARE_RESULT_VECTOR(0, 6); - DECLARE_RESULT_VECTOR(0, 7); - DECLARE_RESULT_VECTOR(1, 0); - DECLARE_RESULT_VECTOR(1, 1); - DECLARE_RESULT_VECTOR(1, 2); - DECLARE_RESULT_VECTOR(1, 3); - DECLARE_RESULT_VECTOR(1, 4); - DECLARE_RESULT_VECTOR(1, 5); - DECLARE_RESULT_VECTOR(1, 6); - DECLARE_RESULT_VECTOR(1, 7); if (LIKELY(packed_b != NULL)) { if (i == 0) { @@ -310,68 +314,6 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); - VECTOR_LOAD_B_K4(4, 0); - VECTOR_LOAD_B_K4(5, 0); - VECTOR_LOAD_B_K4(6, 0); - VECTOR_LOAD_B_K4(7, 0); - TRANSPOSE_B4_K4(4, 5, 6, 7, 0, 1, 2, 3); - SCALE_B4_K4(4, 0, 1, 2, 3); - VECTOR_PACK_B4(4, 0); - VECTOR_PACK_B4(4, 1); - VECTOR_PACK_B4(4, 2); - VECTOR_PACK_B4(4, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 3); - VECTOR_LOAD_A(pg_true, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); - VECTOR_LOAD_A(pg_true, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 1); - VECTOR_LOAD_A(pg_true, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 2); - VECTOR_LOAD_A(pg_true, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 3); } for (; k < K; k++) { @@ -382,33 +324,12 @@ CNAME(BLASLONG M, BROADCAST_LOAD_B(1, 0); PACK_B(1, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); - VECTOR_LOAD_A(pg_true, 1, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); BROADCAST_LOAD_B(2, 0); PACK_B(2, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); BROADCAST_LOAD_B(3, 0); PACK_B(3, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); - BROADCAST_LOAD_B(4, 0); - PACK_B(4, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 4, 0); - BROADCAST_LOAD_B(5, 0); - PACK_B(5, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 5, 0); - BROADCAST_LOAD_B(6, 0); - PACK_B(6, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 6, 0); - BROADCAST_LOAD_B(7, 0); - PACK_B(7, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 7, 0); } } else { for (; k < K; k++) { @@ -419,20 +340,6 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - UNPACK_QUADWORD_B(4, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); - VECTOR_LOAD_A(pg_true, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); } } } else { @@ -464,64 +371,6 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); - VECTOR_LOAD_B_K4(4, 0); - VECTOR_LOAD_B_K4(5, 0); - VECTOR_LOAD_B_K4(6, 0); - VECTOR_LOAD_B_K4(7, 0); - TRANSPOSE_B4_K4(4, 5, 6, 7, 0, 1, 2, 3); - SCALE_B4_K4(4, 0, 1, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 3); - VECTOR_LOAD_A(pg_true, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); - VECTOR_LOAD_A(pg_true, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 1); - VECTOR_LOAD_A(pg_true, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 2); - VECTOR_LOAD_A(pg_true, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 3); } for (; k < K; k++) { @@ -530,167 +379,31 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); BROADCAST_LOAD_B(1, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); - VECTOR_LOAD_A(pg_true, 1, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); BROADCAST_LOAD_B(2, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); BROADCAST_LOAD_B(3, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); - BROADCAST_LOAD_B(4, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 4, 0); - BROADCAST_LOAD_B(5, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 5, 0); - BROADCAST_LOAD_B(6, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 6, 0); - BROADCAST_LOAD_B(7, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 7, 0); } } VECTOR_STORE(pg_true, 0, 0); VECTOR_STORE(pg_true, 0, 1); VECTOR_STORE(pg_true, 0, 2); VECTOR_STORE(pg_true, 0, 3); - VECTOR_STORE(pg_true, 0, 4); - VECTOR_STORE(pg_true, 0, 5); - VECTOR_STORE(pg_true, 0, 6); - VECTOR_STORE(pg_true, 0, 7); - VECTOR_STORE(pg_true, 1, 0); - VECTOR_STORE(pg_true, 1, 1); - VECTOR_STORE(pg_true, 1, 2); - VECTOR_STORE(pg_true, 1, 3); - VECTOR_STORE(pg_true, 1, 4); - VECTOR_STORE(pg_true, 1, 5); - VECTOR_STORE(pg_true, 1, 6); - VECTOR_STORE(pg_true, 1, 7); - } - for (; i < v_m1; i += v_size) { - - BLASLONG k = 0; - DECLARE_RESULT_VECTOR(0, 0); - DECLARE_RESULT_VECTOR(0, 1); - DECLARE_RESULT_VECTOR(0, 2); - DECLARE_RESULT_VECTOR(0, 3); - DECLARE_RESULT_VECTOR(0, 4); - DECLARE_RESULT_VECTOR(0, 5); - DECLARE_RESULT_VECTOR(0, 6); - DECLARE_RESULT_VECTOR(0, 7); - - if (LIKELY(packed_b != NULL)) { - for (; k < K; k++) { - - UNPACK_QUADWORD_B(0, 0); - VECTOR_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - UNPACK_QUADWORD_B(4, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); - } - } else { - for (; k < k4; k += 4) { - - VECTOR_LOAD_B_K4(0, 0); - VECTOR_LOAD_B_K4(1, 0); - VECTOR_LOAD_B_K4(2, 0); - VECTOR_LOAD_B_K4(3, 0); - TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); - SCALE_B4_K4(0, 0, 1, 2, 3); - VECTOR_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - VECTOR_LOAD_A(pg_true, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); - VECTOR_LOAD_A(pg_true, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); - VECTOR_LOAD_A(pg_true, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); - VECTOR_LOAD_B_K4(4, 0); - VECTOR_LOAD_B_K4(5, 0); - VECTOR_LOAD_B_K4(6, 0); - VECTOR_LOAD_B_K4(7, 0); - TRANSPOSE_B4_K4(4, 5, 6, 7, 0, 1, 2, 3); - SCALE_B4_K4(4, 0, 1, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 3); - } - for (; k < K; k++) { - - BROADCAST_LOAD_B(0, 0); - VECTOR_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); - BROADCAST_LOAD_B(1, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); - BROADCAST_LOAD_B(2, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); - BROADCAST_LOAD_B(3, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); - BROADCAST_LOAD_B(4, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0); - BROADCAST_LOAD_B(5, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0); - BROADCAST_LOAD_B(6, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0); - BROADCAST_LOAD_B(7, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0); - } - } - VECTOR_STORE(pg_true, 0, 0); - VECTOR_STORE(pg_true, 0, 1); - VECTOR_STORE(pg_true, 0, 2); - VECTOR_STORE(pg_true, 0, 3); - VECTOR_STORE(pg_true, 0, 4); - VECTOR_STORE(pg_true, 0, 5); - VECTOR_STORE(pg_true, 0, 6); - VECTOR_STORE(pg_true, 0, 7); + INCR_C_POINTER(0, v_size); + INCR_C_POINTER(1, v_size); + INCR_C_POINTER(2, v_size); + INCR_C_POINTER(3, v_size); } for (; i < M; i += v_size) { const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M)); + CREATE_A_POINTER(0, 0); + UPDATE_A_POINTER(0); BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); DECLARE_RESULT_VECTOR(0, 2); DECLARE_RESULT_VECTOR(0, 3); - DECLARE_RESULT_VECTOR(0, 4); - DECLARE_RESULT_VECTOR(0, 5); - DECLARE_RESULT_VECTOR(0, 6); - DECLARE_RESULT_VECTOR(0, 7); if (LIKELY(packed_b != NULL)) { for (; k < K; k++) { @@ -701,11 +414,6 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - UNPACK_QUADWORD_B(4, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); } } else { for (; k < k4; k += 4) { @@ -736,28 +444,6 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); - VECTOR_LOAD_B_K4(4, 0); - VECTOR_LOAD_B_K4(5, 0); - VECTOR_LOAD_B_K4(6, 0); - VECTOR_LOAD_B_K4(7, 0); - TRANSPOSE_B4_K4(4, 5, 6, 7, 0, 1, 2, 3); - SCALE_B4_K4(4, 0, 1, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 3); } for (; k < K; k++) { @@ -770,248 +456,33 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_tail, 0, 2, 0); BROADCAST_LOAD_B(3, 0); UPDATE_RESULT_VECTOR(pg_tail, 0, 3, 0); - BROADCAST_LOAD_B(4, 0); - UPDATE_RESULT_VECTOR(pg_tail, 0, 4, 0); - BROADCAST_LOAD_B(5, 0); - UPDATE_RESULT_VECTOR(pg_tail, 0, 5, 0); - BROADCAST_LOAD_B(6, 0); - UPDATE_RESULT_VECTOR(pg_tail, 0, 6, 0); - BROADCAST_LOAD_B(7, 0); - UPDATE_RESULT_VECTOR(pg_tail, 0, 7, 0); } } VECTOR_STORE(pg_tail, 0, 0); VECTOR_STORE(pg_tail, 0, 1); VECTOR_STORE(pg_tail, 0, 2); VECTOR_STORE(pg_tail, 0, 3); - VECTOR_STORE(pg_tail, 0, 4); - VECTOR_STORE(pg_tail, 0, 5); - VECTOR_STORE(pg_tail, 0, 6); - VECTOR_STORE(pg_tail, 0, 7); - } - } - for (; j < n4; j += 4) { - - BLASLONG i = 0; - for (; i < v_m2; i += v_size2) { - - BLASLONG k = 0; - DECLARE_RESULT_VECTOR(0, 0); - DECLARE_RESULT_VECTOR(0, 1); - DECLARE_RESULT_VECTOR(0, 2); - DECLARE_RESULT_VECTOR(0, 3); - DECLARE_RESULT_VECTOR(1, 0); - DECLARE_RESULT_VECTOR(1, 1); - DECLARE_RESULT_VECTOR(1, 2); - DECLARE_RESULT_VECTOR(1, 3); - - for (; k < k4; k += 4) { - - VECTOR_LOAD_B_K4(0, 0); - VECTOR_LOAD_B_K4(1, 0); - VECTOR_LOAD_B_K4(2, 0); - VECTOR_LOAD_B_K4(3, 0); - TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); - SCALE_B4_K4(0, 0, 1, 2, 3); - VECTOR_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - VECTOR_LOAD_A(pg_true, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); - VECTOR_LOAD_A(pg_true, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); - VECTOR_LOAD_A(pg_true, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); - VECTOR_LOAD_A(pg_true, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); - VECTOR_LOAD_A(pg_true, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1); - VECTOR_LOAD_A(pg_true, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2); - VECTOR_LOAD_A(pg_true, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3); - } - for (; k < K; k++) { - - BROADCAST_LOAD_B(0, 0); - VECTOR_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); - BROADCAST_LOAD_B(1, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); - VECTOR_LOAD_A(pg_true, 1, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); - BROADCAST_LOAD_B(2, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); - BROADCAST_LOAD_B(3, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); - } - VECTOR_STORE(pg_true, 0, 0); - VECTOR_STORE(pg_true, 0, 1); - VECTOR_STORE(pg_true, 0, 2); - VECTOR_STORE(pg_true, 0, 3); - VECTOR_STORE(pg_true, 1, 0); - VECTOR_STORE(pg_true, 1, 1); - VECTOR_STORE(pg_true, 1, 2); - VECTOR_STORE(pg_true, 1, 3); + INCR_C_POINTER(0, 0); + INCR_C_POINTER(1, 0); + INCR_C_POINTER(2, 0); + INCR_C_POINTER(3, 0); } - for (; i < v_m1; i += v_size) { - - BLASLONG k = 0; - DECLARE_RESULT_VECTOR(0, 0); - DECLARE_RESULT_VECTOR(0, 1); - DECLARE_RESULT_VECTOR(0, 2); - DECLARE_RESULT_VECTOR(0, 3); - - for (; k < k4; k += 4) { - - VECTOR_LOAD_B_K4(0, 0); - VECTOR_LOAD_B_K4(1, 0); - VECTOR_LOAD_B_K4(2, 0); - VECTOR_LOAD_B_K4(3, 0); - TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); - SCALE_B4_K4(0, 0, 1, 2, 3); - VECTOR_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - VECTOR_LOAD_A(pg_true, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); - VECTOR_LOAD_A(pg_true, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); - VECTOR_LOAD_A(pg_true, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); - } - for (; k < K; k++) { - BROADCAST_LOAD_B(0, 0); - VECTOR_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); - BROADCAST_LOAD_B(1, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); - BROADCAST_LOAD_B(2, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); - BROADCAST_LOAD_B(3, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); - } - VECTOR_STORE(pg_true, 0, 0); - VECTOR_STORE(pg_true, 0, 1); - VECTOR_STORE(pg_true, 0, 2); - VECTOR_STORE(pg_true, 0, 3); - } - for (; i < M; i += v_size) { - const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M)); - - BLASLONG k = 0; - DECLARE_RESULT_VECTOR(0, 0); - DECLARE_RESULT_VECTOR(0, 1); - DECLARE_RESULT_VECTOR(0, 2); - DECLARE_RESULT_VECTOR(0, 3); - - for (; k < k4; k += 4) { - - VECTOR_LOAD_B_K4(0, 0); - VECTOR_LOAD_B_K4(1, 0); - VECTOR_LOAD_B_K4(2, 0); - VECTOR_LOAD_B_K4(3, 0); - TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); - SCALE_B4_K4(0, 0, 1, 2, 3); - VECTOR_LOAD_A(pg_tail, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - VECTOR_LOAD_A(pg_tail, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); - VECTOR_LOAD_A(pg_tail, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); - VECTOR_LOAD_A(pg_tail, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); - } - for (; k < K; k++) { - - BROADCAST_LOAD_B(0, 0); - VECTOR_LOAD_A(pg_tail, 0, 0); - UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); - BROADCAST_LOAD_B(1, 0); - UPDATE_RESULT_VECTOR(pg_tail, 0, 1, 0); - BROADCAST_LOAD_B(2, 0); - UPDATE_RESULT_VECTOR(pg_tail, 0, 2, 0); - BROADCAST_LOAD_B(3, 0); - UPDATE_RESULT_VECTOR(pg_tail, 0, 3, 0); - } - VECTOR_STORE(pg_tail, 0, 0); - VECTOR_STORE(pg_tail, 0, 1); - VECTOR_STORE(pg_tail, 0, 2); - VECTOR_STORE(pg_tail, 0, 3); - } + UPDATE_B_POINTER(4); + RESET_A_POINTER(); + UPDATE_C_POINTER(4); } for (; j < N; j++) { - BLASLONG i = 0; - for (; i < v_m2; i += v_size2) { - - BLASLONG k = 0; - DECLARE_RESULT_VECTOR(0, 0); - DECLARE_RESULT_VECTOR(1, 0); - - for (; k < K; k++) { + CREATE_C_POINTER(0, 0); + CREATE_B_POINTER(0, 0); - BROADCAST_LOAD_B(0, 0); - VECTOR_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); - VECTOR_LOAD_A(pg_true, 1, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); - } - VECTOR_STORE(pg_true, 0, 0); - VECTOR_STORE(pg_true, 1, 0); - } + BLASLONG i = 0; for (; i < v_m1; i += v_size) { + CREATE_A_POINTER(0, 0); + UPDATE_A_POINTER(v_size); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); @@ -1022,9 +493,12 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); } VECTOR_STORE(pg_true, 0, 0); + INCR_C_POINTER(0, v_size); } for (; i < M; i += v_size) { const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M)); + CREATE_A_POINTER(0, 0); + UPDATE_A_POINTER(0); BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); @@ -1036,11 +510,16 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); } VECTOR_STORE(pg_tail, 0, 0); + INCR_C_POINTER(0, 0); } + + UPDATE_B_POINTER(1); + RESET_A_POINTER(); + UPDATE_C_POINTER(1); } if (pack_b) free(packed_b); return 0; -} +} \ No newline at end of file diff --git a/kernel/arm64/sgemm_small_kernel_nt_sve.c b/kernel/arm64/sgemm_small_kernel_nt_sve.c index 1c3d324d08..9f99c24224 100644 --- a/kernel/arm64/sgemm_small_kernel_nt_sve.c +++ b/kernel/arm64/sgemm_small_kernel_nt_sve.c @@ -46,15 +46,29 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. }) #endif -#define A_ELEMENT_K(m, offset_k) A[(i + (m)) + (k + offset_k) * lda] +#define RESET_A_POINTER() a_offset = A; + +#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale; +#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale; +#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k) * lda) #define A_ELEMENT(m) A_ELEMENT_K(m, 0) -#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) * ldb + (j + (n))] +#define RESET_B_POINTER() b_offset = B; + +#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale; +#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale; +#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k) * ldb) #define B_ELEMENT(n) B_ELEMENT_K(n, 0) -#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc] +#define CREATE_C_POINTER(n, scale) FLOAT* c_offset##n = c_offset + scale * ldc; +#define INCR_C_POINTER(m, incr) // c_offset ## m += incr; +#define UPDATE_C_POINTER(scale) c_offset = c_offset + scale * ldc; +#define C_ELEMENT(m, n) *(c_offset##n + ((m * v_size) + i)) -#define PACK_ELEMENT_K(n, offset_k) packed_b[(k + offset_k) * 8 + n] +// #undef C_ELEMENT +// #define C_ELEMENT(m, n) C[(i+(m))+(j+(n))*ldc] + +#define PACK_ELEMENT_K(n, offset_k) packed_b[(k + offset_k) * 4 + n] #define PACK_ELEMENT(n) PACK_ELEMENT_K(n, 0) // ASIMD @@ -113,8 +127,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define BROADCAST_LOAD_B(n, offset_k) \ svfloat32_t b##s##n##_k##offset_k = svdup_f32(B_ELEMENT_K(n, offset_k)); #define VECTOR_LOAD_A(pg, m, offset_k) \ - svfloat32_t a##s##m##_k##offset_k = \ - svld1(pg, &A_ELEMENT_K(v_size * m, offset_k)); + svfloat32_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k)); #define QUADWORD_LOAD_B(n, offset_k) \ svfloat32_t b##s##n##_k##offset_k = \ svld1rq(pg_true, &B_ELEMENT_K(n, offset_k)); @@ -141,26 +154,23 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifdef B0 #define VECTOR_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); + svst1(pg, &C_ELEMENT(m, n), result##m##n); #define SCATTER_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - svst1_scatter_index( \ - pg, &C_ELEMENT(v_size* m, n), svindex_u32(0LL, ldc), result##m##n); + svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n); #else #define VECTOR_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ result##m##n = \ - svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(v_size * m, n)), beta_vec); \ - svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); + svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec); \ + svst1(pg, &C_ELEMENT(m, n), result##m##n); #define SCATTER_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - result##m##n = svmla_m( \ - pg, \ - result##m##n, \ - svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u32(0LL, ldc)), \ - beta_vec); \ - svst1_scatter_index( \ - pg, &C_ELEMENT(v_size* m, n), svindex_u32(0LL, ldc), result##m##n); + result##m##n = svmla_m(pg, \ + result##m##n, \ + svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec), \ + beta_vec); \ + svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n); #endif #ifndef LIKELY @@ -170,13 +180,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define LIKELY(x) (x) #endif #endif -#ifndef UNLIKELY -#ifdef __GNUC__ -#define UNLIKELY(x) __builtin_expect(!!(x), 0) -#else -#define UNLIKELY(x) (x) -#endif -#endif #ifdef B0 int @@ -206,7 +209,6 @@ CNAME(BLASLONG M, #endif { const uint64_t v_size = svcntw(); - const uint64_t v_size3 = v_size * 3; const svbool_t pg_true = svptrue_b32(); const svbool_t pg_quad = svwhilelt_b32(0, 4); const svbool_t pg_first = svwhilelt_b32(0, 1); @@ -214,46 +216,40 @@ CNAME(BLASLONG M, #ifndef B0 const svfloat32_t beta_vec = svdup_f32(beta); #endif - const BLASLONG n8 = N & -8; const BLASLONG n4 = N & -4; - const BLASLONG v_m3 = M - (M % v_size3); const BLASLONG v_m1 = M & -v_size; - const int pack_b = M >= v_size3 && N >= 8 && K >= 8 ? 1 : 0; + const int pack_b = M >= v_size && N >= 8 && K >= 8 ? 1 : 0; FLOAT* packed_b = - (pack_b) ? packed_b = (FLOAT*)malloc(K * 8 * sizeof(FLOAT)) : NULL; + (pack_b) ? packed_b = (FLOAT*)malloc(K * 4 * sizeof(FLOAT)) : NULL; + + FLOAT* b_offset = B; + FLOAT* a_offset = A; + FLOAT* c_offset = C; BLASLONG j = 0; - for (; j < n8; j += 8) { + for (; j < n4; j += 4) { + + CREATE_C_POINTER(0, 0); + CREATE_C_POINTER(1, 1); + CREATE_C_POINTER(2, 2); + CREATE_C_POINTER(3, 3); + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + CREATE_B_POINTER(2, 2); + CREATE_B_POINTER(3, 3); BLASLONG i = 0; - for (; i < v_m3; i += v_size3) { + for (; i < v_m1; i += v_size) { + + CREATE_A_POINTER(0, 0); + UPDATE_A_POINTER(v_size); BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); DECLARE_RESULT_VECTOR(0, 2); DECLARE_RESULT_VECTOR(0, 3); - DECLARE_RESULT_VECTOR(0, 4); - DECLARE_RESULT_VECTOR(0, 5); - DECLARE_RESULT_VECTOR(0, 6); - DECLARE_RESULT_VECTOR(0, 7); - DECLARE_RESULT_VECTOR(1, 0); - DECLARE_RESULT_VECTOR(1, 1); - DECLARE_RESULT_VECTOR(1, 2); - DECLARE_RESULT_VECTOR(1, 3); - DECLARE_RESULT_VECTOR(1, 4); - DECLARE_RESULT_VECTOR(1, 5); - DECLARE_RESULT_VECTOR(1, 6); - DECLARE_RESULT_VECTOR(1, 7); - DECLARE_RESULT_VECTOR(2, 0); - DECLARE_RESULT_VECTOR(2, 1); - DECLARE_RESULT_VECTOR(2, 2); - DECLARE_RESULT_VECTOR(2, 3); - DECLARE_RESULT_VECTOR(2, 4); - DECLARE_RESULT_VECTOR(2, 5); - DECLARE_RESULT_VECTOR(2, 6); - DECLARE_RESULT_VECTOR(2, 7); if (LIKELY(packed_b != NULL)) { if (i == 0) { @@ -266,30 +262,6 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - QUADWORD_LOAD_B(4, 0); - QUADWORD_PACK_B(4, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); - VECTOR_LOAD_A(pg_true, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); - VECTOR_LOAD_A(pg_true, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 3, 0, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 7, 4, 3, 0); } } else { for (; k < K; k++) { @@ -300,29 +272,6 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - UNPACK_QUADWORD_B(4, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); - VECTOR_LOAD_A(pg_true, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); - VECTOR_LOAD_A(pg_true, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 3, 0, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 7, 4, 3, 0); } } } else { @@ -334,120 +283,27 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - QUADWORD_LOAD_B(4, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); - VECTOR_LOAD_A(pg_true, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); - VECTOR_LOAD_A(pg_true, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 3, 0, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 7, 4, 3, 0); - } - } - VECTOR_STORE(pg_true, 0, 0); - VECTOR_STORE(pg_true, 0, 1); - VECTOR_STORE(pg_true, 0, 2); - VECTOR_STORE(pg_true, 0, 3); - VECTOR_STORE(pg_true, 0, 4); - VECTOR_STORE(pg_true, 0, 5); - VECTOR_STORE(pg_true, 0, 6); - VECTOR_STORE(pg_true, 0, 7); - VECTOR_STORE(pg_true, 1, 0); - VECTOR_STORE(pg_true, 1, 1); - VECTOR_STORE(pg_true, 1, 2); - VECTOR_STORE(pg_true, 1, 3); - VECTOR_STORE(pg_true, 1, 4); - VECTOR_STORE(pg_true, 1, 5); - VECTOR_STORE(pg_true, 1, 6); - VECTOR_STORE(pg_true, 1, 7); - VECTOR_STORE(pg_true, 2, 0); - VECTOR_STORE(pg_true, 2, 1); - VECTOR_STORE(pg_true, 2, 2); - VECTOR_STORE(pg_true, 2, 3); - VECTOR_STORE(pg_true, 2, 4); - VECTOR_STORE(pg_true, 2, 5); - VECTOR_STORE(pg_true, 2, 6); - VECTOR_STORE(pg_true, 2, 7); - } - for (; i < v_m1; i += v_size) { - - BLASLONG k = 0; - DECLARE_RESULT_VECTOR(0, 0); - DECLARE_RESULT_VECTOR(0, 1); - DECLARE_RESULT_VECTOR(0, 2); - DECLARE_RESULT_VECTOR(0, 3); - DECLARE_RESULT_VECTOR(0, 4); - DECLARE_RESULT_VECTOR(0, 5); - DECLARE_RESULT_VECTOR(0, 6); - DECLARE_RESULT_VECTOR(0, 7); - - if (LIKELY(packed_b != NULL)) { - for (; k < K; k++) { - - UNPACK_QUADWORD_B(0, 0); - VECTOR_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - UNPACK_QUADWORD_B(4, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); - } - } else { - for (; k < K; k++) { - - QUADWORD_LOAD_B(0, 0); - VECTOR_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - QUADWORD_LOAD_B(4, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); } } VECTOR_STORE(pg_true, 0, 0); VECTOR_STORE(pg_true, 0, 1); VECTOR_STORE(pg_true, 0, 2); VECTOR_STORE(pg_true, 0, 3); - VECTOR_STORE(pg_true, 0, 4); - VECTOR_STORE(pg_true, 0, 5); - VECTOR_STORE(pg_true, 0, 6); - VECTOR_STORE(pg_true, 0, 7); + INCR_C_POINTER(0, v_size); + INCR_C_POINTER(1, v_size); + INCR_C_POINTER(2, v_size); + INCR_C_POINTER(3, v_size); } for (; i < M; i += v_size) { const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M)); + CREATE_A_POINTER(0, 0); + UPDATE_A_POINTER(0); BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); DECLARE_RESULT_VECTOR(0, 2); DECLARE_RESULT_VECTOR(0, 3); - DECLARE_RESULT_VECTOR(0, 4); - DECLARE_RESULT_VECTOR(0, 5); - DECLARE_RESULT_VECTOR(0, 6); - DECLARE_RESULT_VECTOR(0, 7); if (LIKELY(packed_b != NULL)) { for (; k < K; k++) { @@ -458,11 +314,6 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - UNPACK_QUADWORD_B(4, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); } } else { for (; k < K; k++) { @@ -473,146 +324,33 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - QUADWORD_LOAD_B(4, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); } } VECTOR_STORE(pg_tail, 0, 0); VECTOR_STORE(pg_tail, 0, 1); VECTOR_STORE(pg_tail, 0, 2); VECTOR_STORE(pg_tail, 0, 3); - VECTOR_STORE(pg_tail, 0, 4); - VECTOR_STORE(pg_tail, 0, 5); - VECTOR_STORE(pg_tail, 0, 6); - VECTOR_STORE(pg_tail, 0, 7); + INCR_C_POINTER(0, 0); + INCR_C_POINTER(1, 0); + INCR_C_POINTER(2, 0); + INCR_C_POINTER(3, 0); } - } - for (; j < n4; j += 4) { - - BLASLONG i = 0; - for (; i < v_m3; i += v_size3) { - - BLASLONG k = 0; - DECLARE_RESULT_VECTOR(0, 0); - DECLARE_RESULT_VECTOR(0, 1); - DECLARE_RESULT_VECTOR(0, 2); - DECLARE_RESULT_VECTOR(0, 3); - DECLARE_RESULT_VECTOR(1, 0); - DECLARE_RESULT_VECTOR(1, 1); - DECLARE_RESULT_VECTOR(1, 2); - DECLARE_RESULT_VECTOR(1, 3); - DECLARE_RESULT_VECTOR(2, 0); - DECLARE_RESULT_VECTOR(2, 1); - DECLARE_RESULT_VECTOR(2, 2); - DECLARE_RESULT_VECTOR(2, 3); - - for (; k < K; k++) { - QUADWORD_LOAD_B(0, 0); - VECTOR_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - VECTOR_LOAD_A(pg_true, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); - VECTOR_LOAD_A(pg_true, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(2, 3, 0, 3, 0); - } - VECTOR_STORE(pg_true, 0, 0); - VECTOR_STORE(pg_true, 0, 1); - VECTOR_STORE(pg_true, 0, 2); - VECTOR_STORE(pg_true, 0, 3); - VECTOR_STORE(pg_true, 1, 0); - VECTOR_STORE(pg_true, 1, 1); - VECTOR_STORE(pg_true, 1, 2); - VECTOR_STORE(pg_true, 1, 3); - VECTOR_STORE(pg_true, 2, 0); - VECTOR_STORE(pg_true, 2, 1); - VECTOR_STORE(pg_true, 2, 2); - VECTOR_STORE(pg_true, 2, 3); - } - for (; i < v_m1; i += v_size) { - - BLASLONG k = 0; - DECLARE_RESULT_VECTOR(0, 0); - DECLARE_RESULT_VECTOR(0, 1); - DECLARE_RESULT_VECTOR(0, 2); - DECLARE_RESULT_VECTOR(0, 3); - - for (; k < K; k++) { - - QUADWORD_LOAD_B(0, 0); - VECTOR_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - } - VECTOR_STORE(pg_true, 0, 0); - VECTOR_STORE(pg_true, 0, 1); - VECTOR_STORE(pg_true, 0, 2); - VECTOR_STORE(pg_true, 0, 3); - } - for (; i < M; i += v_size) { - const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M)); - - BLASLONG k = 0; - DECLARE_RESULT_VECTOR(0, 0); - DECLARE_RESULT_VECTOR(0, 1); - DECLARE_RESULT_VECTOR(0, 2); - DECLARE_RESULT_VECTOR(0, 3); - - for (; k < K; k++) { - - QUADWORD_LOAD_B(0, 0); - VECTOR_LOAD_A(pg_tail, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - } - VECTOR_STORE(pg_tail, 0, 0); - VECTOR_STORE(pg_tail, 0, 1); - VECTOR_STORE(pg_tail, 0, 2); - VECTOR_STORE(pg_tail, 0, 3); - } + UPDATE_B_POINTER(4); + RESET_A_POINTER(); + UPDATE_C_POINTER(4); } for (; j < N; j++) { - BLASLONG i = 0; - for (; i < v_m3; i += v_size3) { - - BLASLONG k = 0; - DECLARE_RESULT_VECTOR(0, 0); - DECLARE_RESULT_VECTOR(1, 0); - DECLARE_RESULT_VECTOR(2, 0); + CREATE_C_POINTER(0, 0); + CREATE_B_POINTER(0, 0); - for (; k < K; k++) { - - BROADCAST_LOAD_B(0, 0); - VECTOR_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); - VECTOR_LOAD_A(pg_true, 1, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); - VECTOR_LOAD_A(pg_true, 2, 0); - UPDATE_RESULT_VECTOR(pg_true, 2, 0, 0); - } - VECTOR_STORE(pg_true, 0, 0); - VECTOR_STORE(pg_true, 1, 0); - VECTOR_STORE(pg_true, 2, 0); - } + BLASLONG i = 0; for (; i < v_m1; i += v_size) { + CREATE_A_POINTER(0, 0); + UPDATE_A_POINTER(v_size); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); @@ -623,9 +361,12 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); } VECTOR_STORE(pg_true, 0, 0); + INCR_C_POINTER(0, v_size); } for (; i < M; i += v_size) { const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M)); + CREATE_A_POINTER(0, 0); + UPDATE_A_POINTER(0); BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); @@ -637,11 +378,16 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); } VECTOR_STORE(pg_tail, 0, 0); + INCR_C_POINTER(0, 0); } + + UPDATE_B_POINTER(1); + RESET_A_POINTER(); + UPDATE_C_POINTER(1); } if (pack_b) free(packed_b); return 0; -} +} \ No newline at end of file diff --git a/kernel/arm64/sgemm_small_kernel_tn_sve.c b/kernel/arm64/sgemm_small_kernel_tn_sve.c index 6fd3b12a67..9cbb60d40f 100644 --- a/kernel/arm64/sgemm_small_kernel_tn_sve.c +++ b/kernel/arm64/sgemm_small_kernel_tn_sve.c @@ -46,15 +46,30 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. }) #endif -#define A_ELEMENT_K(m, offset_k) A[(i + (m)) * lda + (k + offset_k)] +#define RESET_A_POINTER() a_offset = A; + +#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale * lda; +#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale * lda; +#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k)) #define A_ELEMENT(m) A_ELEMENT_K(m, 0) -#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) + (j + (n)) * ldb] +#define RESET_B_POINTER() b_offset = B; + +#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale * ldb; +#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale * ldb; +#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k)) #define B_ELEMENT(n) B_ELEMENT_K(n, 0) -#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc] +#define CREATE_C_POINTER(m, scale) FLOAT* c_offset##m = c_offset + scale; +#define INCR_C_POINTER(m, incr) // c_offset ## m += incr * ldc; +#define UPDATE_C_POINTER(scale) c_offset += scale; +#define C_ELEMENT(m, n) \ + *(c_offset##m + ((j + n) * ldc)) // C[(i+(m))+(j+(n))*ldc] + +// #undef C_ELEMENT +// #define C_ELEMENT(m, n) C[(i+(m))+(j+(n))*ldc] -#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * 2 * v_size + m] +#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * v_size + m] #define PACK_ELEMENT(m) PACK_ELEMENT_K(m, 0) // ASIMD @@ -65,36 +80,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. float32x4_t a##m##_k##offset_k = vld1q_dup_f32(&A_ELEMENT_K(m, offset_k)); #define LOAD_A1(m, offset_k) \ float32_t a##m##_k##offset_k = A_ELEMENT_K(m, offset_k); -#define VECTOR_LOAD_B_K4(n, offset_k) \ - float32x4_t b##k##n##_k##offset_k = vld1q_f32(&B_ELEMENT_K(n, offset_k)); -#define TRANSPOSE_B4_K4( \ - n0, n1, n2, n3, offset_k0, offset_k1, offset_k2, offset_k3) \ - float32x4_t b##t##n0##_k##offset_k0 = \ - vzip1q_f32(b##k##n0##_k##offset_k0, b##k##n1##_k##offset_k0); \ - float32x4_t b##t##n0##_k##offset_k1 = \ - vzip2q_f32(b##k##n0##_k##offset_k0, b##k##n1##_k##offset_k0); \ - float32x4_t b##t##n0##_k##offset_k2 = \ - vzip1q_f32(b##k##n2##_k##offset_k0, b##k##n3##_k##offset_k0); \ - float32x4_t b##t##n0##_k##offset_k3 = \ - vzip2q_f32(b##k##n2##_k##offset_k0, b##k##n3##_k##offset_k0); \ - float32x4_t b##n0##_k##offset_k0 = vreinterpretq_f32_f64( \ - vzip1q_f64(vreinterpretq_f64_f32(b##t##n0##_k##offset_k0), \ - vreinterpretq_f64_f32(b##t##n0##_k##offset_k2))); \ - float32x4_t b##n0##_k##offset_k1 = vreinterpretq_f32_f64( \ - vzip2q_f64(vreinterpretq_f64_f32(b##t##n0##_k##offset_k0), \ - vreinterpretq_f64_f32(b##t##n0##_k##offset_k2))); \ - float32x4_t b##n0##_k##offset_k2 = vreinterpretq_f32_f64( \ - vzip1q_f64(vreinterpretq_f64_f32(b##t##n0##_k##offset_k1), \ - vreinterpretq_f64_f32(b##t##n0##_k##offset_k3))); \ - float32x4_t b##n0##_k##offset_k3 = vreinterpretq_f32_f64( \ - vzip2q_f64(vreinterpretq_f64_f32(b##t##n0##_k##offset_k1), \ - vreinterpretq_f64_f32(b##t##n0##_k##offset_k3))); - -#define SCALE_B4_K4(n0, offset_k0, offset_k1, offset_k2, offset_k3) \ - svfloat32_t b##s##n0##_k##offset_k0 = svdup_neonq_f32(b##n0##_k##offset_k0); \ - svfloat32_t b##s##n0##_k##offset_k1 = svdup_neonq_f32(b##n0##_k##offset_k1); \ - svfloat32_t b##s##n0##_k##offset_k2 = svdup_neonq_f32(b##n0##_k##offset_k2); \ - svfloat32_t b##s##n0##_k##offset_k3 = svdup_neonq_f32(b##n0##_k##offset_k3); #define GATHER_LOAD_B4(n, offset_k) \ float32x4_t b##n##_k##offset_k = vdupq_n_f32(B_ELEMENT_K(n, offset_k)); \ b##n##_k##offset_k = \ @@ -105,8 +90,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. vsetq_lane_f32(B_ELEMENT_K(n + 3, offset_k), b##n##_k##offset_k, 3); #define VECTOR_UNPACK_B4(n, offset_k) \ float32x4_t b##n##_k##offset_k = vld1q_f32(&PACK_ELEMENT_K(n, offset_k)); -#define VECTOR_PACK_B4(n, offset_k) \ - vst1q_f32(&PACK_ELEMENT_K(n, offset_k), b##n##_k##offset_k); #define PACK_B0(n, offset_k) \ PACK_ELEMENT_K(n, offset_k) = vget_lane_f32(b##n##_k##offset_k, 0); #define UPDATE_RESULT_VECTOR4(m, n, offset_k) \ @@ -141,14 +124,10 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define BROADCAST_LOAD_B(n, offset_k) \ svfloat32_t b##s##n##_k##offset_k = svdup_f32(B_ELEMENT_K(n, offset_k)); #define VECTOR_LOAD_A(pg, m, offset_k) \ - svfloat32_t a##s##m##_k##offset_k = \ - svld1(pg, &A_ELEMENT_K(v_size * m, offset_k)); -#define QUADWORD_LOAD_B(n, offset_k) \ - svfloat32_t b##s##n##_k##offset_k = \ - svld1rq(pg_true, &B_ELEMENT_K(n, offset_k)); + svfloat32_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k)); #define GATHER_LOAD_A(pg, m, offset_k) \ svfloat32_t a##s##m##_k##offset_k = \ - svld1_gather_index(pg, &A_ELEMENT_K(v_size * m, offset_k), lda_vec); + svld1_gather_index(pg, &A_ELEMENT_K(m, offset_k), lda_vec); #define PACK_A(m, offset_k) \ svst1(pg_first, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k); #define VECTOR_PACK_A(m, offset_k) \ @@ -172,26 +151,23 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifdef B0 #define VECTOR_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); + svst1(pg, &C_ELEMENT(m, n), result##m##n); #define SCATTER_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - svst1_scatter_index( \ - pg, &C_ELEMENT(v_size* m, n), svindex_u32(0LL, ldc), result##m##n); + svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n); #else #define VECTOR_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ result##m##n = \ - svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(v_size * m, n)), beta_vec); \ - svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n); + svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec); \ + svst1(pg, &C_ELEMENT(m, n), result##m##n); #define SCATTER_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - result##m##n = svmla_m( \ - pg, \ - result##m##n, \ - svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u32(0LL, ldc)), \ - beta_vec); \ - svst1_scatter_index( \ - pg, &C_ELEMENT(v_size* m, n), svindex_u32(0LL, ldc), result##m##n); + result##m##n = svmla_m(pg, \ + result##m##n, \ + svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec), \ + beta_vec); \ + svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n); #endif #ifndef LIKELY @@ -201,18 +177,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define LIKELY(x) (x) #endif #endif -#ifndef UNLIKELY -#ifdef __GNUC__ -#define UNLIKELY(x) __builtin_expect(!!(x), 0) -#else -#define UNLIKELY(x) (x) -#endif -#endif - - -#define GATHER_LOAD_A64(pg, m, offset_k) \ - svfloat64_t a##t##m##_k##offset_k = \ - svld1_gather_offset(pg, (double *)&A_ELEMENT_K(v64_size * m, offset_k), lda_vec64); #ifdef B0 int @@ -242,8 +206,6 @@ CNAME(BLASLONG M, #endif { const uint64_t v_size = svcntw(); - const uint64_t v_size2 = v_size * 2; - const uint64_t v64_size = v_size / 2; const svbool_t pg_true = svptrue_b32(); const svbool_t pg_quad = svwhilelt_b32(0, 4); const svbool_t pg_first = svwhilelt_b32(0, 1); @@ -252,173 +214,41 @@ CNAME(BLASLONG M, const svfloat32_t beta_vec = svdup_f32(beta); #endif const svuint32_t lda_vec = svindex_u32(0LL, lda); - const svuint64_t lda_vec64 = svmul_m(pg_true, svindex_u64(0,sizeof(FLOAT)), lda); - const BLASLONG v_m2 = M & -v_size2; const BLASLONG v_m1 = M & -v_size; - const BLASLONG n8 = N & -8; const BLASLONG n4 = N & -4; - const BLASLONG k4 = K & -4; - const int pack_a = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0; + const int pack_a = M >= v_size && N >= 8 && K >= 8 ? 1 : 0; FLOAT* packed_a = - (pack_a) ? packed_a = (FLOAT*)malloc(K * 2 * v_size * sizeof(FLOAT)) : NULL; + (pack_a) ? packed_a = (FLOAT*)malloc(K * v_size * sizeof(FLOAT)) : NULL; + + FLOAT* a_offset = A; + FLOAT* b_offset = B; + FLOAT* c_offset = C; BLASLONG i = 0; - for (; i < v_m2; i += v_size2) { + for (; i < v_m1; i += v_size) { + + CREATE_C_POINTER(0, 0); + CREATE_A_POINTER(0, 0); BLASLONG j = 0; - for (; j < n8; j += 8) { + for (; j < n4; j += 4) { + + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + CREATE_B_POINTER(2, 2); + CREATE_B_POINTER(3, 3); + UPDATE_B_POINTER(4); BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); DECLARE_RESULT_VECTOR(0, 2); DECLARE_RESULT_VECTOR(0, 3); - DECLARE_RESULT_VECTOR(0, 4); - DECLARE_RESULT_VECTOR(0, 5); - DECLARE_RESULT_VECTOR(0, 6); - DECLARE_RESULT_VECTOR(0, 7); - DECLARE_RESULT_VECTOR(1, 0); - DECLARE_RESULT_VECTOR(1, 1); - DECLARE_RESULT_VECTOR(1, 2); - DECLARE_RESULT_VECTOR(1, 3); - DECLARE_RESULT_VECTOR(1, 4); - DECLARE_RESULT_VECTOR(1, 5); - DECLARE_RESULT_VECTOR(1, 6); - DECLARE_RESULT_VECTOR(1, 7); if (LIKELY(packed_a != NULL)) { if (j == 0) { - for (; k < k4; k += 4) { - - VECTOR_LOAD_B_K4(0, 0); - VECTOR_LOAD_B_K4(1, 0); - VECTOR_LOAD_B_K4(2, 0); - VECTOR_LOAD_B_K4(3, 0); - TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); - SCALE_B4_K4(0, 0, 1, 2, 3); - - GATHER_LOAD_A64(pg_true, 0, 0); - GATHER_LOAD_A64(pg_true, 1, 0); - svfloat32_t as0_k0 = svuzp1(svreinterpret_f32(at0_k0), svreinterpret_f32(at1_k0)); - svfloat32_t as0_k1 = svuzp2(svreinterpret_f32(at0_k0), svreinterpret_f32(at1_k0)); - VECTOR_PACK_A(0, 0); - VECTOR_PACK_A(0, 1); - - // GATHER_LOAD_A(pg_true, 0, 0); - // VECTOR_PACK_A(0, 0); - // GATHER_LOAD_A(pg_true, 0, 1); - // VECTOR_PACK_A(0, 1); - - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); - - GATHER_LOAD_A64(pg_true, 0, 2); - GATHER_LOAD_A64(pg_true, 1, 2); - svfloat32_t as0_k2 = svuzp1(svreinterpret_f32(at0_k2), svreinterpret_f32(at1_k2)); - svfloat32_t as0_k3 = svuzp2(svreinterpret_f32(at0_k2), svreinterpret_f32(at1_k2)); - VECTOR_PACK_A(0, 2); - VECTOR_PACK_A(0, 3); - - // GATHER_LOAD_A(pg_true, 0, 2); - // VECTOR_PACK_A(0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); - // GATHER_LOAD_A(pg_true, 0, 3); - // VECTOR_PACK_A(0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); - VECTOR_LOAD_B_K4(4, 0); - VECTOR_LOAD_B_K4(5, 0); - VECTOR_LOAD_B_K4(6, 0); - VECTOR_LOAD_B_K4(7, 0); - TRANSPOSE_B4_K4(4, 5, 6, 7, 0, 1, 2, 3); - SCALE_B4_K4(4, 0, 1, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 3); - - GATHER_LOAD_A64(pg_true, 2, 0); - GATHER_LOAD_A64(pg_true, 3, 0); - svfloat32_t as1_k0 = svuzp1(svreinterpret_f32(at2_k0), svreinterpret_f32(at3_k0)); - svfloat32_t as1_k1 = svuzp2(svreinterpret_f32(at2_k0), svreinterpret_f32(at3_k0)); - VECTOR_PACK_A(1, 0); - VECTOR_PACK_A(1, 1); - - // GATHER_LOAD_A(pg_true, 1, 0); - // VECTOR_PACK_A(1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); - // GATHER_LOAD_A(pg_true, 1, 1); - // VECTOR_PACK_A(1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 1); - - // 64-bit load x2 then unzip into 32-bit - GATHER_LOAD_A64(pg_true, 2, 2); - GATHER_LOAD_A64(pg_true, 3, 2); - svfloat32_t as1_k2 = svuzp1(svreinterpret_f32(at2_k2), svreinterpret_f32(at3_k2)); - svfloat32_t as1_k3 = svuzp2(svreinterpret_f32(at2_k2), svreinterpret_f32(at3_k2)); - VECTOR_PACK_A(1, 2); - VECTOR_PACK_A(1, 3); - - // GATHER_LOAD_A(pg_true, 1, 2); - // VECTOR_PACK_A(1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 2); - // GATHER_LOAD_A(pg_true, 1, 3); - // VECTOR_PACK_A(1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 3); - } for (; k < K; k++) { BROADCAST_LOAD_B(0, 0); @@ -427,117 +257,12 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); BROADCAST_LOAD_B(1, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); - GATHER_LOAD_A(pg_true, 1, 0); - VECTOR_PACK_A(1, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); BROADCAST_LOAD_B(2, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); BROADCAST_LOAD_B(3, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); - BROADCAST_LOAD_B(4, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 4, 0); - BROADCAST_LOAD_B(5, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 5, 0); - BROADCAST_LOAD_B(6, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 6, 0); - BROADCAST_LOAD_B(7, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 7, 0); } } else { - for (; k < k4; k += 4) { - - VECTOR_LOAD_B_K4(0, 0); - VECTOR_LOAD_B_K4(1, 0); - VECTOR_LOAD_B_K4(2, 0); - VECTOR_LOAD_B_K4(3, 0); - TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); - SCALE_B4_K4(0, 0, 1, 2, 3); - UNPACK_VECTOR_A(0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - UNPACK_VECTOR_A(0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); - UNPACK_VECTOR_A(0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); - UNPACK_VECTOR_A(0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); - VECTOR_LOAD_B_K4(4, 0); - VECTOR_LOAD_B_K4(5, 0); - VECTOR_LOAD_B_K4(6, 0); - VECTOR_LOAD_B_K4(7, 0); - TRANSPOSE_B4_K4(4, 5, 6, 7, 0, 1, 2, 3); - SCALE_B4_K4(4, 0, 1, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 3); - UNPACK_VECTOR_A(1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); - UNPACK_VECTOR_A(1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 1); - UNPACK_VECTOR_A(1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 2); - UNPACK_VECTOR_A(1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 3); - } for (; k < K; k++) { BROADCAST_LOAD_B(0, 0); @@ -545,293 +270,13 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); BROADCAST_LOAD_B(1, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); - UNPACK_VECTOR_A(1, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); BROADCAST_LOAD_B(2, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); BROADCAST_LOAD_B(3, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); - BROADCAST_LOAD_B(4, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 4, 0); - BROADCAST_LOAD_B(5, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 5, 0); - BROADCAST_LOAD_B(6, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 6, 0); - BROADCAST_LOAD_B(7, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 7, 0); } } } else { - for (; k < k4; k += 4) { - - VECTOR_LOAD_B_K4(0, 0); - VECTOR_LOAD_B_K4(1, 0); - VECTOR_LOAD_B_K4(2, 0); - VECTOR_LOAD_B_K4(3, 0); - TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); - SCALE_B4_K4(0, 0, 1, 2, 3); - GATHER_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - GATHER_LOAD_A(pg_true, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); - GATHER_LOAD_A(pg_true, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); - GATHER_LOAD_A(pg_true, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); - VECTOR_LOAD_B_K4(4, 0); - VECTOR_LOAD_B_K4(5, 0); - VECTOR_LOAD_B_K4(6, 0); - VECTOR_LOAD_B_K4(7, 0); - TRANSPOSE_B4_K4(4, 5, 6, 7, 0, 1, 2, 3); - SCALE_B4_K4(4, 0, 1, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 3); - GATHER_LOAD_A(pg_true, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); - GATHER_LOAD_A(pg_true, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 1); - GATHER_LOAD_A(pg_true, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 2); - GATHER_LOAD_A(pg_true, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 3); - } - for (; k < K; k++) { - - BROADCAST_LOAD_B(0, 0); - GATHER_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); - BROADCAST_LOAD_B(1, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); - GATHER_LOAD_A(pg_true, 1, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); - BROADCAST_LOAD_B(2, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); - BROADCAST_LOAD_B(3, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); - BROADCAST_LOAD_B(4, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 4, 0); - BROADCAST_LOAD_B(5, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 5, 0); - BROADCAST_LOAD_B(6, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 6, 0); - BROADCAST_LOAD_B(7, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 7, 0); - } - } - VECTOR_STORE(pg_true, 0, 0); - VECTOR_STORE(pg_true, 0, 1); - VECTOR_STORE(pg_true, 0, 2); - VECTOR_STORE(pg_true, 0, 3); - VECTOR_STORE(pg_true, 0, 4); - VECTOR_STORE(pg_true, 0, 5); - VECTOR_STORE(pg_true, 0, 6); - VECTOR_STORE(pg_true, 0, 7); - VECTOR_STORE(pg_true, 1, 0); - VECTOR_STORE(pg_true, 1, 1); - VECTOR_STORE(pg_true, 1, 2); - VECTOR_STORE(pg_true, 1, 3); - VECTOR_STORE(pg_true, 1, 4); - VECTOR_STORE(pg_true, 1, 5); - VECTOR_STORE(pg_true, 1, 6); - VECTOR_STORE(pg_true, 1, 7); - } - for (; j < n4; j += 4) { - - BLASLONG k = 0; - DECLARE_RESULT_VECTOR(0, 0); - DECLARE_RESULT_VECTOR(0, 1); - DECLARE_RESULT_VECTOR(0, 2); - DECLARE_RESULT_VECTOR(0, 3); - DECLARE_RESULT_VECTOR(1, 0); - DECLARE_RESULT_VECTOR(1, 1); - DECLARE_RESULT_VECTOR(1, 2); - DECLARE_RESULT_VECTOR(1, 3); - - if (LIKELY(packed_a != NULL)) { - for (; k < k4; k += 4) { - - VECTOR_LOAD_B_K4(0, 0); - VECTOR_LOAD_B_K4(1, 0); - VECTOR_LOAD_B_K4(2, 0); - VECTOR_LOAD_B_K4(3, 0); - TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); - SCALE_B4_K4(0, 0, 1, 2, 3); - UNPACK_VECTOR_A(0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - UNPACK_VECTOR_A(0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); - UNPACK_VECTOR_A(0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); - UNPACK_VECTOR_A(0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); - UNPACK_VECTOR_A(1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); - UNPACK_VECTOR_A(1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1); - UNPACK_VECTOR_A(1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2); - UNPACK_VECTOR_A(1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3); - } - for (; k < K; k++) { - - BROADCAST_LOAD_B(0, 0); - UNPACK_VECTOR_A(0, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); - BROADCAST_LOAD_B(1, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); - UNPACK_VECTOR_A(1, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); - BROADCAST_LOAD_B(2, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); - BROADCAST_LOAD_B(3, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); - } - } else { - for (; k < k4; k += 4) { - - VECTOR_LOAD_B_K4(0, 0); - VECTOR_LOAD_B_K4(1, 0); - VECTOR_LOAD_B_K4(2, 0); - VECTOR_LOAD_B_K4(3, 0); - TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); - SCALE_B4_K4(0, 0, 1, 2, 3); - GATHER_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - GATHER_LOAD_A(pg_true, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); - GATHER_LOAD_A(pg_true, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); - GATHER_LOAD_A(pg_true, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); - GATHER_LOAD_A(pg_true, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); - GATHER_LOAD_A(pg_true, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1); - GATHER_LOAD_A(pg_true, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2); - GATHER_LOAD_A(pg_true, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3); - } for (; k < K; k++) { BROADCAST_LOAD_B(0, 0); @@ -839,31 +284,25 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); BROADCAST_LOAD_B(1, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); - GATHER_LOAD_A(pg_true, 1, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); BROADCAST_LOAD_B(2, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); BROADCAST_LOAD_B(3, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); } } VECTOR_STORE(pg_true, 0, 0); VECTOR_STORE(pg_true, 0, 1); VECTOR_STORE(pg_true, 0, 2); VECTOR_STORE(pg_true, 0, 3); - VECTOR_STORE(pg_true, 1, 0); - VECTOR_STORE(pg_true, 1, 1); - VECTOR_STORE(pg_true, 1, 2); - VECTOR_STORE(pg_true, 1, 3); + INCR_C_POINTER(0, 4); } for (; j < N; j++) { + CREATE_B_POINTER(0, 0); + UPDATE_B_POINTER(1); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); - DECLARE_RESULT_VECTOR(1, 0); if (LIKELY(packed_a != NULL)) { for (; k < K; k++) { @@ -871,8 +310,6 @@ CNAME(BLASLONG M, BROADCAST_LOAD_B(0, 0); UNPACK_VECTOR_A(0, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); - UNPACK_VECTOR_A(1, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); } } else { for (; k < K; k++) { @@ -880,334 +317,36 @@ CNAME(BLASLONG M, BROADCAST_LOAD_B(0, 0); GATHER_LOAD_A(pg_true, 0, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); - GATHER_LOAD_A(pg_true, 1, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); } } VECTOR_STORE(pg_true, 0, 0); - VECTOR_STORE(pg_true, 1, 0); + INCR_C_POINTER(0, 1); } - } - for (; i < v_m1; i += v_size) { - - BLASLONG j = 0; - for (; j < n8; j += 8) { - BLASLONG k = 0; - DECLARE_RESULT_VECTOR(0, 0); - DECLARE_RESULT_VECTOR(0, 1); - DECLARE_RESULT_VECTOR(0, 2); - DECLARE_RESULT_VECTOR(0, 3); - DECLARE_RESULT_VECTOR(0, 4); - DECLARE_RESULT_VECTOR(0, 5); - DECLARE_RESULT_VECTOR(0, 6); - DECLARE_RESULT_VECTOR(0, 7); - - for (; k < k4; k += 4) { - - VECTOR_LOAD_B_K4(0, 0); - VECTOR_LOAD_B_K4(1, 0); - VECTOR_LOAD_B_K4(2, 0); - VECTOR_LOAD_B_K4(3, 0); - TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); - SCALE_B4_K4(0, 0, 1, 2, 3); - - GATHER_LOAD_A64(pg_true, 0, 0); - GATHER_LOAD_A64(pg_true, 1, 0); - svfloat32_t as0_k0 = svuzp1(svreinterpret_f32(at0_k0), svreinterpret_f32(at1_k0)); - svfloat32_t as0_k1 = svuzp2(svreinterpret_f32(at0_k0), svreinterpret_f32(at1_k0)); - - // GATHER_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - // GATHER_LOAD_A(pg_true, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); - - GATHER_LOAD_A64(pg_true, 0, 2); - GATHER_LOAD_A64(pg_true, 1, 2); - svfloat32_t as0_k2 = svuzp1(svreinterpret_f32(at0_k2), svreinterpret_f32(at1_k2)); - svfloat32_t as0_k3 = svuzp2(svreinterpret_f32(at0_k2), svreinterpret_f32(at1_k2)); - - // GATHER_LOAD_A(pg_true, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); - // GATHER_LOAD_A(pg_true, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); - VECTOR_LOAD_B_K4(4, 0); - VECTOR_LOAD_B_K4(5, 0); - VECTOR_LOAD_B_K4(6, 0); - VECTOR_LOAD_B_K4(7, 0); - TRANSPOSE_B4_K4(4, 5, 6, 7, 0, 1, 2, 3); - SCALE_B4_K4(4, 0, 1, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 3); - } - for (; k < K; k++) { - - BROADCAST_LOAD_B(0, 0); - GATHER_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); - BROADCAST_LOAD_B(1, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); - BROADCAST_LOAD_B(2, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); - BROADCAST_LOAD_B(3, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); - BROADCAST_LOAD_B(4, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0); - BROADCAST_LOAD_B(5, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0); - BROADCAST_LOAD_B(6, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0); - BROADCAST_LOAD_B(7, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0); - } - VECTOR_STORE(pg_true, 0, 0); - VECTOR_STORE(pg_true, 0, 1); - VECTOR_STORE(pg_true, 0, 2); - VECTOR_STORE(pg_true, 0, 3); - VECTOR_STORE(pg_true, 0, 4); - VECTOR_STORE(pg_true, 0, 5); - VECTOR_STORE(pg_true, 0, 6); - VECTOR_STORE(pg_true, 0, 7); - } - for (; j < n4; j += 4) { - - BLASLONG k = 0; - DECLARE_RESULT_VECTOR(0, 0); - DECLARE_RESULT_VECTOR(0, 1); - DECLARE_RESULT_VECTOR(0, 2); - DECLARE_RESULT_VECTOR(0, 3); - - for (; k < k4; k += 4) { - - VECTOR_LOAD_B_K4(0, 0); - VECTOR_LOAD_B_K4(1, 0); - VECTOR_LOAD_B_K4(2, 0); - VECTOR_LOAD_B_K4(3, 0); - TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); - SCALE_B4_K4(0, 0, 1, 2, 3); - - GATHER_LOAD_A64(pg_true, 0, 0); - GATHER_LOAD_A64(pg_true, 1, 0); - svfloat32_t as0_k0 = svuzp1(svreinterpret_f32(at0_k0), svreinterpret_f32(at1_k0)); - svfloat32_t as0_k1 = svuzp2(svreinterpret_f32(at0_k0), svreinterpret_f32(at1_k0)); - - // GATHER_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - // GATHER_LOAD_A(pg_true, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); - - GATHER_LOAD_A64(pg_true, 0, 2); - GATHER_LOAD_A64(pg_true, 1, 2); - svfloat32_t as0_k2 = svuzp1(svreinterpret_f32(at0_k2), svreinterpret_f32(at1_k2)); - svfloat32_t as0_k3 = svuzp2(svreinterpret_f32(at0_k2), svreinterpret_f32(at1_k2)); - - // GATHER_LOAD_A(pg_true, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); - // GATHER_LOAD_A(pg_true, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); - } - for (; k < K; k++) { - - BROADCAST_LOAD_B(0, 0); - GATHER_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); - BROADCAST_LOAD_B(1, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); - BROADCAST_LOAD_B(2, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); - BROADCAST_LOAD_B(3, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); - } - VECTOR_STORE(pg_true, 0, 0); - VECTOR_STORE(pg_true, 0, 1); - VECTOR_STORE(pg_true, 0, 2); - VECTOR_STORE(pg_true, 0, 3); - } - for (; j < N; j++) { - - BLASLONG k = 0; - DECLARE_RESULT_VECTOR(0, 0); - - for (; k < K; k++) { - - BROADCAST_LOAD_B(0, 0); - GATHER_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); - } - VECTOR_STORE(pg_true, 0, 0); - } + UPDATE_A_POINTER(v_size); + RESET_B_POINTER(); + UPDATE_C_POINTER(v_size); } for (; i < M; i += v_size) { const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M)); + CREATE_C_POINTER(0, 0); + CREATE_A_POINTER(0, 0); BLASLONG j = 0; - for (; j < n8; j += 8) { - - BLASLONG k = 0; - DECLARE_RESULT_VECTOR(0, 0); - DECLARE_RESULT_VECTOR(0, 1); - DECLARE_RESULT_VECTOR(0, 2); - DECLARE_RESULT_VECTOR(0, 3); - DECLARE_RESULT_VECTOR(0, 4); - DECLARE_RESULT_VECTOR(0, 5); - DECLARE_RESULT_VECTOR(0, 6); - DECLARE_RESULT_VECTOR(0, 7); - - for (; k < k4; k += 4) { - - VECTOR_LOAD_B_K4(0, 0); - VECTOR_LOAD_B_K4(1, 0); - VECTOR_LOAD_B_K4(2, 0); - VECTOR_LOAD_B_K4(3, 0); - TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); - SCALE_B4_K4(0, 0, 1, 2, 3); - GATHER_LOAD_A(pg_tail, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - GATHER_LOAD_A(pg_tail, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); - GATHER_LOAD_A(pg_tail, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); - GATHER_LOAD_A(pg_tail, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); - VECTOR_LOAD_B_K4(4, 0); - VECTOR_LOAD_B_K4(5, 0); - VECTOR_LOAD_B_K4(6, 0); - VECTOR_LOAD_B_K4(7, 0); - TRANSPOSE_B4_K4(4, 5, 6, 7, 0, 1, 2, 3); - SCALE_B4_K4(4, 0, 1, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 3); - } - for (; k < K; k++) { - - BROADCAST_LOAD_B(0, 0); - GATHER_LOAD_A(pg_tail, 0, 0); - UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); - BROADCAST_LOAD_B(1, 0); - UPDATE_RESULT_VECTOR(pg_tail, 0, 1, 0); - BROADCAST_LOAD_B(2, 0); - UPDATE_RESULT_VECTOR(pg_tail, 0, 2, 0); - BROADCAST_LOAD_B(3, 0); - UPDATE_RESULT_VECTOR(pg_tail, 0, 3, 0); - BROADCAST_LOAD_B(4, 0); - UPDATE_RESULT_VECTOR(pg_tail, 0, 4, 0); - BROADCAST_LOAD_B(5, 0); - UPDATE_RESULT_VECTOR(pg_tail, 0, 5, 0); - BROADCAST_LOAD_B(6, 0); - UPDATE_RESULT_VECTOR(pg_tail, 0, 6, 0); - BROADCAST_LOAD_B(7, 0); - UPDATE_RESULT_VECTOR(pg_tail, 0, 7, 0); - } - VECTOR_STORE(pg_tail, 0, 0); - VECTOR_STORE(pg_tail, 0, 1); - VECTOR_STORE(pg_tail, 0, 2); - VECTOR_STORE(pg_tail, 0, 3); - VECTOR_STORE(pg_tail, 0, 4); - VECTOR_STORE(pg_tail, 0, 5); - VECTOR_STORE(pg_tail, 0, 6); - VECTOR_STORE(pg_tail, 0, 7); - } for (; j < n4; j += 4) { + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + CREATE_B_POINTER(2, 2); + CREATE_B_POINTER(3, 3); + UPDATE_B_POINTER(4); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); DECLARE_RESULT_VECTOR(0, 2); DECLARE_RESULT_VECTOR(0, 3); - for (; k < k4; k += 4) { - - VECTOR_LOAD_B_K4(0, 0); - VECTOR_LOAD_B_K4(1, 0); - VECTOR_LOAD_B_K4(2, 0); - VECTOR_LOAD_B_K4(3, 0); - TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); - SCALE_B4_K4(0, 0, 1, 2, 3); - GATHER_LOAD_A(pg_tail, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - GATHER_LOAD_A(pg_tail, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); - GATHER_LOAD_A(pg_tail, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); - GATHER_LOAD_A(pg_tail, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); - } for (; k < K; k++) { BROADCAST_LOAD_B(0, 0); @@ -1224,9 +363,13 @@ CNAME(BLASLONG M, VECTOR_STORE(pg_tail, 0, 1); VECTOR_STORE(pg_tail, 0, 2); VECTOR_STORE(pg_tail, 0, 3); + INCR_C_POINTER(0, 4); } for (; j < N; j++) { + CREATE_B_POINTER(0, 0); + UPDATE_B_POINTER(1); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); @@ -1237,11 +380,16 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); } VECTOR_STORE(pg_tail, 0, 0); + INCR_C_POINTER(0, 1); } + + UPDATE_A_POINTER(0); + RESET_B_POINTER(); + UPDATE_C_POINTER(0); } if (pack_a) free(packed_a); return 0; -} +} \ No newline at end of file diff --git a/kernel/arm64/sgemm_small_kernel_tt_sve.c b/kernel/arm64/sgemm_small_kernel_tt_sve.c index 894e7fd46a..dd9840c378 100644 --- a/kernel/arm64/sgemm_small_kernel_tt_sve.c +++ b/kernel/arm64/sgemm_small_kernel_tt_sve.c @@ -46,15 +46,30 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. }) #endif -#define A_ELEMENT_K(m, offset_k) A[(i + (m)) * lda + (k + offset_k)] +#define RESET_A_POINTER() a_offset = A; + +#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale * lda; +#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale * lda; +#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k)) #define A_ELEMENT(m) A_ELEMENT_K(m, 0) -#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) * ldb + (j + (n))] +#define RESET_B_POINTER() b_offset = B; + +#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale; +#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale; +#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k) * ldb) #define B_ELEMENT(n) B_ELEMENT_K(n, 0) -#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc] +#define CREATE_C_POINTER(m, scale) FLOAT* c_offset##m = c_offset + scale; +#define INCR_C_POINTER(m, incr) // c_offset ## m += incr * ldc; +#define UPDATE_C_POINTER(scale) c_offset += scale; +#define C_ELEMENT(m, n) \ + *(c_offset##m + ((j + n) * ldc)) // C[(i+(m))+(j+(n))*ldc] -#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * v_size2 + m] +// #undef C_ELEMENT +// #define C_ELEMENT(m, n) C[(i+(m))+(j+(n))*ldc] + +#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * v_size + m] #define PACK_ELEMENT(m) PACK_ELEMENT_K(m, 0) // ASIMD @@ -106,22 +121,23 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. svfloat32_t a##s##m##_k##offset_k = svdup_f32(A_ELEMENT_K(m, offset_k)); #define BROADCAST_LOAD_B(n, offset_k) \ svfloat32_t b##s##n##_k##offset_k = svdup_f32(B_ELEMENT_K(n, offset_k)); - +#define VECTOR_LOAD_A(pg, m, offset_k) \ + svfloat32_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k)); #define QUADWORD_LOAD_B(n, offset_k) \ svfloat32_t b##s##n##_k##offset_k = \ svld1rq(pg_true, &B_ELEMENT_K(n, offset_k)); -#define GATHER_LOAD_A(pg, offset_m, m, offset_k) \ +#define GATHER_LOAD_A(pg, m, offset_k) \ svfloat32_t a##s##m##_k##offset_k = \ - svld1_gather_index(pg, &A_ELEMENT_K(offset_m, offset_k), lda_vec); + svld1_gather_index(pg, &A_ELEMENT_K(m, offset_k), lda_vec); #define PACK_A(m, offset_k) \ svst1(pg_first, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k); -#define VECTOR_PACK_A(offset_m, m, offset_k) \ +#define VECTOR_PACK_A(m, offset_k) \ svst1(pg_true, &PACK_ELEMENT_K(m* v_size, offset_k), a##s##m##_k##offset_k); #define QUADWORD_PACK_A(m, offset_k) \ svst1(pg_quad, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k); -#define UNPACK_VECTOR_A(offset_m, m, offset_k) \ +#define UNPACK_VECTOR_A(m, offset_k) \ svfloat32_t a##s##m##_k##offset_k = \ - svld1(pg_true, &PACK_ELEMENT_K(offset_m, offset_k)); + svld1(pg_true, &PACK_ELEMENT_K(m * v_size, offset_k)); #define UNPACK_BROADCAST_A(m, offset_k) \ svfloat32_t a##s##m##_k##offset_k = svdup_f32(PACK_ELEMENT_K(m, offset_k)); #define UNPACK_QUADWORD_A(m, offset_k) \ @@ -134,28 +150,25 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. result##m##n = svmla_lane( \ result##m##n, a##s##m##_k##offset_k, b##s##outer##_k##offset_k, lane); #ifdef B0 -#define VECTOR_STORE(pg, offset_m, m, n) \ +#define VECTOR_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - svst1(pg, &C_ELEMENT(offset_m, n), result##m##n); + svst1(pg, &C_ELEMENT(m, n), result##m##n); #define SCATTER_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - svst1_scatter_index( \ - pg, &C_ELEMENT(v_size* m, n), svindex_u32(0LL, ldc), result##m##n); + svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n); #else -#define VECTOR_STORE(pg, offset_m, m, n) \ +#define VECTOR_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ result##m##n = \ - svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(offset_m, n)), beta_vec); \ - svst1(pg, &C_ELEMENT(offset_m, n), result##m##n); + svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec); \ + svst1(pg, &C_ELEMENT(m, n), result##m##n); #define SCATTER_STORE(pg, m, n) \ result##m##n = svmul_m(pg, result##m##n, alpha_vec); \ - result##m##n = svmla_m( \ - pg, \ - result##m##n, \ - svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u32(0LL, ldc)), \ - beta_vec); \ - svst1_scatter_index( \ - pg, &C_ELEMENT(v_size* m, n), svindex_u32(0LL, ldc), result##m##n); + result##m##n = svmla_m(pg, \ + result##m##n, \ + svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec), \ + beta_vec); \ + svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n); #endif #ifndef LIKELY @@ -165,13 +178,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define LIKELY(x) (x) #endif #endif -#ifndef UNLIKELY -#ifdef __GNUC__ -#define UNLIKELY(x) __builtin_expect(!!(x), 0) -#else -#define UNLIKELY(x) (x) -#endif -#endif #ifdef B0 int @@ -201,337 +207,132 @@ CNAME(BLASLONG M, #endif { const uint64_t v_size = svcntw(); - const uint64_t v_size2 = v_size * 2; const svbool_t pg_true = svptrue_b32(); + const svbool_t pg_quad = svwhilelt_b32(0, 4); + const svbool_t pg_first = svwhilelt_b32(0, 1); const svfloat32_t alpha_vec = svdup_f32(alpha); #ifndef B0 const svfloat32_t beta_vec = svdup_f32(beta); #endif const svuint32_t lda_vec = svindex_u32(0LL, lda); - const BLASLONG v_m2 = M & -v_size2; const BLASLONG v_m1 = M & -v_size; - const BLASLONG n8 = N & -8; const BLASLONG n4 = N & -4; - const int pack_a = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0; + const int pack_a = M >= v_size && N >= 8 && K >= 8 ? 1 : 0; FLOAT* packed_a = - (pack_a) ? packed_a = (FLOAT*)malloc(K * 2 * v_size * sizeof(FLOAT)) : NULL; + (pack_a) ? packed_a = (FLOAT*)malloc(K * v_size * sizeof(FLOAT)) : NULL; + + FLOAT* a_offset = A; + FLOAT* b_offset = B; + FLOAT* c_offset = C; BLASLONG i = 0; - for (; i < v_m2; i += v_size2) { + for (; i < v_m1; i += v_size) { + + CREATE_C_POINTER(0, 0); + CREATE_A_POINTER(0, 0); BLASLONG j = 0; - for (; j < n8; j += 8) { + for (; j < n4; j += 4) { + + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + CREATE_B_POINTER(2, 2); + CREATE_B_POINTER(3, 3); + UPDATE_B_POINTER(4); BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); DECLARE_RESULT_VECTOR(0, 2); DECLARE_RESULT_VECTOR(0, 3); - DECLARE_RESULT_VECTOR(0, 4); - DECLARE_RESULT_VECTOR(0, 5); - DECLARE_RESULT_VECTOR(0, 6); - DECLARE_RESULT_VECTOR(0, 7); - DECLARE_RESULT_VECTOR(1, 0); - DECLARE_RESULT_VECTOR(1, 1); - DECLARE_RESULT_VECTOR(1, 2); - DECLARE_RESULT_VECTOR(1, 3); - DECLARE_RESULT_VECTOR(1, 4); - DECLARE_RESULT_VECTOR(1, 5); - DECLARE_RESULT_VECTOR(1, 6); - DECLARE_RESULT_VECTOR(1, 7); if (LIKELY(packed_a != NULL)) { if (j == 0) { for (; k < K; k++) { QUADWORD_LOAD_B(0, 0); - GATHER_LOAD_A(pg_true, 0, 0, 0); - VECTOR_PACK_A(0, 0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + VECTOR_PACK_A(0, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - QUADWORD_LOAD_B(4, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); - GATHER_LOAD_A(pg_true, v_size, 1, 0); - VECTOR_PACK_A(v_size, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); } } else { for (; k < K; k++) { QUADWORD_LOAD_B(0, 0); - UNPACK_VECTOR_A(0, 0, 0); + UNPACK_VECTOR_A(0, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - QUADWORD_LOAD_B(4, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); - UNPACK_VECTOR_A(v_size, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); } } } else { for (; k < K; k++) { QUADWORD_LOAD_B(0, 0); - GATHER_LOAD_A(pg_true, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - QUADWORD_LOAD_B(4, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); - GATHER_LOAD_A(pg_true, v_size, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); - } - } - VECTOR_STORE(pg_true, 0, 0, 0); - VECTOR_STORE(pg_true, 0, 0, 1); - VECTOR_STORE(pg_true, 0, 0, 2); - VECTOR_STORE(pg_true, 0, 0, 3); - VECTOR_STORE(pg_true, 0, 0, 4); - VECTOR_STORE(pg_true, 0, 0, 5); - VECTOR_STORE(pg_true, 0, 0, 6); - VECTOR_STORE(pg_true, 0, 0, 7); - VECTOR_STORE(pg_true, v_size, 1, 0); - VECTOR_STORE(pg_true, v_size, 1, 1); - VECTOR_STORE(pg_true, v_size, 1, 2); - VECTOR_STORE(pg_true, v_size, 1, 3); - VECTOR_STORE(pg_true, v_size, 1, 4); - VECTOR_STORE(pg_true, v_size, 1, 5); - VECTOR_STORE(pg_true, v_size, 1, 6); - VECTOR_STORE(pg_true, v_size, 1, 7); - } - for (; j < n4; j += 4) { - - BLASLONG k = 0; - DECLARE_RESULT_VECTOR(0, 0); - DECLARE_RESULT_VECTOR(0, 1); - DECLARE_RESULT_VECTOR(0, 2); - DECLARE_RESULT_VECTOR(0, 3); - DECLARE_RESULT_VECTOR(1, 0); - DECLARE_RESULT_VECTOR(1, 1); - DECLARE_RESULT_VECTOR(1, 2); - DECLARE_RESULT_VECTOR(1, 3); - - if (LIKELY(packed_a != NULL)) { - for (; k < K; k++) { - - QUADWORD_LOAD_B(0, 0); - UNPACK_VECTOR_A(0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - UNPACK_VECTOR_A(v_size, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); - } - } else { - for (; k < K; k++) { - - QUADWORD_LOAD_B(0, 0); - GATHER_LOAD_A(pg_true, 0, 0, 0); + GATHER_LOAD_A(pg_true, 0, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - GATHER_LOAD_A(pg_true, v_size, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); } } - VECTOR_STORE(pg_true, 0, 0, 0); - VECTOR_STORE(pg_true, 0, 0, 1); - VECTOR_STORE(pg_true, 0, 0, 2); - VECTOR_STORE(pg_true, 0, 0, 3); - VECTOR_STORE(pg_true, v_size, 1, 0); - VECTOR_STORE(pg_true, v_size, 1, 1); - VECTOR_STORE(pg_true, v_size, 1, 2); - VECTOR_STORE(pg_true, v_size, 1, 3); + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + INCR_C_POINTER(0, 4); } for (; j < N; j++) { + CREATE_B_POINTER(0, 0); + UPDATE_B_POINTER(1); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); - DECLARE_RESULT_VECTOR(1, 0); if (LIKELY(packed_a != NULL)) { for (; k < K; k++) { BROADCAST_LOAD_B(0, 0); - UNPACK_VECTOR_A(0, 0, 0); + UNPACK_VECTOR_A(0, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); - UNPACK_VECTOR_A(v_size, 1, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); } } else { for (; k < K; k++) { BROADCAST_LOAD_B(0, 0); - GATHER_LOAD_A(pg_true, 0, 0, 0); + GATHER_LOAD_A(pg_true, 0, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); - GATHER_LOAD_A(pg_true, v_size, 1, 0); - UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); } } - VECTOR_STORE(pg_true, 0, 0, 0); - VECTOR_STORE(pg_true, v_size, 1, 0); + VECTOR_STORE(pg_true, 0, 0); + INCR_C_POINTER(0, 1); } - } - for (; i < v_m1; i += v_size) { - - BLASLONG j = 0; - for (; j < n8; j += 8) { - - BLASLONG k = 0; - DECLARE_RESULT_VECTOR(0, 0); - DECLARE_RESULT_VECTOR(0, 1); - DECLARE_RESULT_VECTOR(0, 2); - DECLARE_RESULT_VECTOR(0, 3); - DECLARE_RESULT_VECTOR(0, 4); - DECLARE_RESULT_VECTOR(0, 5); - DECLARE_RESULT_VECTOR(0, 6); - DECLARE_RESULT_VECTOR(0, 7); - - for (; k < K; k++) { - QUADWORD_LOAD_B(0, 0); - GATHER_LOAD_A(pg_true, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - QUADWORD_LOAD_B(4, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); - } - VECTOR_STORE(pg_true, 0, 0, 0); - VECTOR_STORE(pg_true, 0, 0, 1); - VECTOR_STORE(pg_true, 0, 0, 2); - VECTOR_STORE(pg_true, 0, 0, 3); - VECTOR_STORE(pg_true, 0, 0, 4); - VECTOR_STORE(pg_true, 0, 0, 5); - VECTOR_STORE(pg_true, 0, 0, 6); - VECTOR_STORE(pg_true, 0, 0, 7); - } - for (; j < n4; j += 4) { - - BLASLONG k = 0; - DECLARE_RESULT_VECTOR(0, 0); - DECLARE_RESULT_VECTOR(0, 1); - DECLARE_RESULT_VECTOR(0, 2); - DECLARE_RESULT_VECTOR(0, 3); - - for (; k < K; k++) { - - QUADWORD_LOAD_B(0, 0); - GATHER_LOAD_A(pg_true, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - } - VECTOR_STORE(pg_true, 0, 0, 0); - VECTOR_STORE(pg_true, 0, 0, 1); - VECTOR_STORE(pg_true, 0, 0, 2); - VECTOR_STORE(pg_true, 0, 0, 3); - } - for (; j < N; j++) { - - BLASLONG k = 0; - DECLARE_RESULT_VECTOR(0, 0); - - for (; k < K; k++) { - - BROADCAST_LOAD_B(0, 0); - GATHER_LOAD_A(pg_true, 0, 0, 0); - UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); - } - VECTOR_STORE(pg_true, 0, 0, 0); - } + UPDATE_A_POINTER(v_size); + RESET_B_POINTER(); + UPDATE_C_POINTER(v_size); } for (; i < M; i += v_size) { const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M)); + CREATE_C_POINTER(0, 0); + CREATE_A_POINTER(0, 0); BLASLONG j = 0; - for (; j < n8; j += 8) { - - BLASLONG k = 0; - DECLARE_RESULT_VECTOR(0, 0); - DECLARE_RESULT_VECTOR(0, 1); - DECLARE_RESULT_VECTOR(0, 2); - DECLARE_RESULT_VECTOR(0, 3); - DECLARE_RESULT_VECTOR(0, 4); - DECLARE_RESULT_VECTOR(0, 5); - DECLARE_RESULT_VECTOR(0, 6); - DECLARE_RESULT_VECTOR(0, 7); - - for (; k < K; k++) { - - QUADWORD_LOAD_B(0, 0); - GATHER_LOAD_A(pg_tail, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); - QUADWORD_LOAD_B(4, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); - } - VECTOR_STORE(pg_tail, 0, 0, 0); - VECTOR_STORE(pg_tail, 0, 0, 1); - VECTOR_STORE(pg_tail, 0, 0, 2); - VECTOR_STORE(pg_tail, 0, 0, 3); - VECTOR_STORE(pg_tail, 0, 0, 4); - VECTOR_STORE(pg_tail, 0, 0, 5); - VECTOR_STORE(pg_tail, 0, 0, 6); - VECTOR_STORE(pg_tail, 0, 0, 7); - } for (; j < n4; j += 4) { + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + CREATE_B_POINTER(2, 2); + CREATE_B_POINTER(3, 3); + UPDATE_B_POINTER(4); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); @@ -541,34 +342,43 @@ CNAME(BLASLONG M, for (; k < K; k++) { QUADWORD_LOAD_B(0, 0); - GATHER_LOAD_A(pg_tail, 0, 0, 0); + GATHER_LOAD_A(pg_tail, 0, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); } - VECTOR_STORE(pg_tail, 0, 0, 0); - VECTOR_STORE(pg_tail, 0, 0, 1); - VECTOR_STORE(pg_tail, 0, 0, 2); - VECTOR_STORE(pg_tail, 0, 0, 3); + VECTOR_STORE(pg_tail, 0, 0); + VECTOR_STORE(pg_tail, 0, 1); + VECTOR_STORE(pg_tail, 0, 2); + VECTOR_STORE(pg_tail, 0, 3); + INCR_C_POINTER(0, 4); } for (; j < N; j++) { + CREATE_B_POINTER(0, 0); + UPDATE_B_POINTER(1); + BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); for (; k < K; k++) { BROADCAST_LOAD_B(0, 0); - GATHER_LOAD_A(pg_tail, 0, 0, 0); + GATHER_LOAD_A(pg_tail, 0, 0); UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); } - VECTOR_STORE(pg_tail, 0, 0, 0); + VECTOR_STORE(pg_tail, 0, 0); + INCR_C_POINTER(0, 1); } + + UPDATE_A_POINTER(0); + RESET_B_POINTER(); + UPDATE_C_POINTER(0); } if (pack_a) free(packed_a); return 0; -} +} \ No newline at end of file From 37a854718b1d2423271356cef67c12be1bd1a842 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Mon, 24 Jun 2024 14:33:06 +0300 Subject: [PATCH 154/267] BENCH: sync codspeed-benchmarks with BLAS-benchmarks --- benchmark/pybench/benchmarks/bench_blas.py | 160 ++++++++++++-------- benchmark/pybench/openblas_wrap/__init__.py | 27 +--- 2 files changed, 105 insertions(+), 82 deletions(-) diff --git a/benchmark/pybench/benchmarks/bench_blas.py b/benchmark/pybench/benchmarks/bench_blas.py index 064be1ead9..96002a202b 100644 --- a/benchmark/pybench/benchmarks/bench_blas.py +++ b/benchmark/pybench/benchmarks/bench_blas.py @@ -1,66 +1,76 @@ import pytest import numpy as np -from openblas_wrap import ( - # level 1 - dnrm2, ddot, daxpy, - # level 3 - dgemm, dsyrk, - # lapack - dgesv, # linalg.solve - dgesdd, dgesdd_lwork, # linalg.svd - dsyev, dsyev_lwork, # linalg.eigh -) +import openblas_wrap as ow + +dtype_map = { + 's': np.float32, + 'd': np.float64, + 'c': np.complex64, + 'z': np.complex128, + 'dz': np.complex128, +} + # ### BLAS level 1 ### # dnrm2 -dnrm2_sizes = [100, 1000] +dnrm2_sizes = [100, 200, 400, 600, 800, 1000] -def run_dnrm2(n, x, incx): - res = dnrm2(x, n, incx=incx) +def run_dnrm2(n, x, incx, func): + res = func(x, n, incx=incx) return res +@pytest.mark.parametrize('variant', ['d', 'dz']) @pytest.mark.parametrize('n', dnrm2_sizes) -def test_nrm2(benchmark, n): +def test_nrm2(benchmark, n, variant): rndm = np.random.RandomState(1234) - x = np.array(rndm.uniform(size=(n,)), dtype=float) - result = benchmark(run_dnrm2, n, x, 1) + dtyp = dtype_map[variant] + + x = np.array(rndm.uniform(size=(n,)), dtype=dtyp) + nrm2 = ow.get_func('nrm2', variant) + result = benchmark(run_dnrm2, n, x, 1, nrm2) # ddot -ddot_sizes = [100, 1000] +ddot_sizes = [100, 200, 400, 600, 800, 1000] -def run_ddot(x, y,): - res = ddot(x, y) +def run_ddot(x, y, func): + res = func(x, y) return res @pytest.mark.parametrize('n', ddot_sizes) def test_dot(benchmark, n): rndm = np.random.RandomState(1234) + x = np.array(rndm.uniform(size=(n,)), dtype=float) y = np.array(rndm.uniform(size=(n,)), dtype=float) - result = benchmark(run_ddot, x, y) + dot = ow.get_func('dot', 'd') + result = benchmark(run_ddot, x, y, dot) # daxpy -daxpy_sizes = [100, 1000] +daxpy_sizes = [100, 200, 400, 600, 800, 1000] -def run_daxpy(x, y,): - res = daxpy(x, y, a=2.0) +def run_daxpy(x, y, func): + res = func(x, y, a=2.0) return res +@pytest.mark.parametrize('variant', ['s', 'd', 'c', 'z']) @pytest.mark.parametrize('n', daxpy_sizes) -def test_daxpy(benchmark, n): +def test_daxpy(benchmark, n, variant): rndm = np.random.RandomState(1234) - x = np.array(rndm.uniform(size=(n,)), dtype=float) - y = np.array(rndm.uniform(size=(n,)), dtype=float) - result = benchmark(run_daxpy, x, y) + dtyp = dtype_map[variant] + + x = np.array(rndm.uniform(size=(n,)), dtype=dtyp) + y = np.array(rndm.uniform(size=(n,)), dtype=dtyp) + axpy = ow.get_func('axpy', variant) + result = benchmark(run_daxpy, x, y, axpy) @@ -69,40 +79,46 @@ def test_daxpy(benchmark, n): # dgemm -gemm_sizes = [100, 1000] +gemm_sizes = [100, 200, 400, 600, 800, 1000] -def run_gemm(a, b, c): +def run_gemm(a, b, c, func): alpha = 1.0 - res = dgemm(alpha, a, b, c=c, overwrite_c=True) + res = func(alpha, a, b, c=c, overwrite_c=True) return res +@pytest.mark.parametrize('variant', ['s', 'd', 'c', 'z']) @pytest.mark.parametrize('n', gemm_sizes) -def test_gemm(benchmark, n): +def test_gemm(benchmark, n, variant): rndm = np.random.RandomState(1234) - a = np.array(rndm.uniform(size=(n, n)), dtype=float, order='F') - b = np.array(rndm.uniform(size=(n, n)), dtype=float, order='F') - c = np.empty((n, n), dtype=float, order='F') - result = benchmark(run_gemm, a, b, c) + dtyp = dtype_map[variant] + a = np.array(rndm.uniform(size=(n, n)), dtype=dtyp, order='F') + b = np.array(rndm.uniform(size=(n, n)), dtype=dtyp, order='F') + c = np.empty((n, n), dtype=dtyp, order='F') + gemm = ow.get_func('gemm', variant) + result = benchmark(run_gemm, a, b, c, gemm) assert result is c # dsyrk -syrk_sizes = [100, 1000] +syrk_sizes = [100, 200, 400, 600, 800, 1000] -def run_syrk(a, c): - res = dsyrk(1.0, a, c=c, overwrite_c=True) +def run_syrk(a, c, func): + res = func(1.0, a, c=c, overwrite_c=True) return res +@pytest.mark.parametrize('variant', ['s', 'd', 'c', 'z']) @pytest.mark.parametrize('n', syrk_sizes) -def test_syrk(benchmark, n): +def test_syrk(benchmark, n, variant): rndm = np.random.RandomState(1234) - a = np.array(rndm.uniform(size=(n, n)), dtype=float, order='F') - c = np.empty((n, n), dtype=float, order='F') - result = benchmark(run_syrk, a, c) + dtyp = dtype_map[variant] + a = np.array(rndm.uniform(size=(n, n)), dtype=dtyp, order='F') + c = np.empty((n, n), dtype=dtyp, order='F') + syrk = ow.get_func('syrk', variant) + result = benchmark(run_syrk, a, c, syrk) assert result is c @@ -110,21 +126,25 @@ def test_syrk(benchmark, n): # linalg.solve -gesv_sizes = [100, 1000] +gesv_sizes = [100, 200, 400, 600, 800, 1000] -def run_gesv(a, b): - res = dgesv(a, b, overwrite_a=True, overwrite_b=True) +def run_gesv(a, b, func): + res = func(a, b, overwrite_a=True, overwrite_b=True) return res +@pytest.mark.parametrize('variant', ['s', 'd', 'c', 'z']) @pytest.mark.parametrize('n', gesv_sizes) -def test_gesv(benchmark, n): +def test_gesv(benchmark, n, variant): rndm = np.random.RandomState(1234) - a = (np.array(rndm.uniform(size=(n, n)), dtype=float, order='F') + - np.eye(n, order='F')) - b = np.array(rndm.uniform(size=(n, 1)), order='F') - lu, piv, x, info = benchmark(run_gesv, a, b) + dtyp = dtype_map[variant] + + a = (np.array(rndm.uniform(size=(n, n)), dtype=dtyp, order='F') + + np.eye(n, dtype=dtyp, order='F')) + b = np.array(rndm.uniform(size=(n, 1)), dtype=dtyp, order='F') + gesv = ow.get_func('gesv', variant) + lu, piv, x, info = benchmark(run_gesv, a, b, gesv) assert lu is a assert x is b assert info == 0 @@ -135,49 +155,63 @@ def test_gesv(benchmark, n): gesdd_sizes = [(100, 5), (1000, 222)] -def run_gesdd(a, lwork): - res = dgesdd(a, lwork=lwork, full_matrices=False, overwrite_a=False) +def run_gesdd(a, lwork, func): + res = func(a, lwork=lwork, full_matrices=False, overwrite_a=False) return res +@pytest.mark.parametrize('variant', ['s', 'd']) @pytest.mark.parametrize('mn', gesdd_sizes) -def test_gesdd(benchmark, mn): +def test_gesdd(benchmark, mn, variant): m, n = mn rndm = np.random.RandomState(1234) - a = np.array(rndm.uniform(size=(m, n)), dtype=float, order='F') + dtyp = dtype_map[variant] + + a = np.array(rndm.uniform(size=(m, n)), dtype=dtyp, order='F') + + gesdd_lwork = ow.get_func('gesdd_lwork', variant) - lwork, info = dgesdd_lwork(m, n) + lwork, info = gesdd_lwork(m, n) lwork = int(lwork) assert info == 0 - u, s, vt, info = benchmark(run_gesdd, a, lwork) + gesdd = ow.get_func('gesdd', variant) + u, s, vt, info = benchmark(run_gesdd, a, lwork, gesdd) assert info == 0 - np.testing.assert_allclose(u @ np.diag(s) @ vt, a, atol=1e-13) + + atol = {'s': 1e-5, 'd': 1e-13} + + np.testing.assert_allclose(u @ np.diag(s) @ vt, a, atol=atol[variant]) # linalg.eigh -syev_sizes = [50, 200] +syev_sizes = [50, 64, 128, 200] -def run_syev(a, lwork): - res = dsyev(a, lwork=lwork, overwrite_a=True) +def run_syev(a, lwork, func): + res = func(a, lwork=lwork, overwrite_a=True) return res +@pytest.mark.parametrize('variant', ['s', 'd']) @pytest.mark.parametrize('n', syev_sizes) -def test_syev(benchmark, n): +def test_syev(benchmark, n, variant): rndm = np.random.RandomState(1234) + dtyp = dtype_map[variant] + a = rndm.uniform(size=(n, n)) - a = np.asarray(a + a.T, dtype=float, order='F') + a = np.asarray(a + a.T, dtype=dtyp, order='F') a_ = a.copy() + dsyev_lwork = ow.get_func('syev_lwork', variant) lwork, info = dsyev_lwork(n) lwork = int(lwork) assert info == 0 - w, v, info = benchmark(run_syev, a, lwork) + syev = ow.get_func('syev', variant) + w, v, info = benchmark(run_syev, a, lwork, syev) assert info == 0 assert a is v # overwrite_a=True diff --git a/benchmark/pybench/openblas_wrap/__init__.py b/benchmark/pybench/openblas_wrap/__init__.py index 06e16a665a..9babb1917e 100644 --- a/benchmark/pybench/openblas_wrap/__init__.py +++ b/benchmark/pybench/openblas_wrap/__init__.py @@ -6,23 +6,12 @@ __version__ = "0.1" -#from scipy.linalg.blas import ( -from ._flapack import ( - # level 1 - dnrm2 as dnrm2, - ddot as ddot, - daxpy as daxpy, - # level 3 - dgemm as dgemm, - dsyrk as dsyrk, -) +from . import _flapack + +PREFIX = '' + + +def get_func(name, variant): + """get_func('gesv', 'c') -> cgesv etc.""" + return getattr(_flapack, PREFIX + variant + name) -#from scipy.linalg.lapack import ( -from openblas_wrap._flapack import ( - # linalg.solve - dgesv as dgesv, - # linalg.svd - dgesdd as dgesdd, dgesdd_lwork as dgesdd_lwork, - # linalg.eigh - dsyev as dsyev, dsyev_lwork as dsyev_lwork -) From 400cf9f63d404006dc8fdc00ba031c4ead5a6b9e Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Mon, 24 Jun 2024 16:47:20 +0300 Subject: [PATCH 155/267] restore the problem sizes for codspeed benchmarks --- benchmark/pybench/benchmarks/bench_blas.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/benchmark/pybench/benchmarks/bench_blas.py b/benchmark/pybench/benchmarks/bench_blas.py index 96002a202b..cface1685f 100644 --- a/benchmark/pybench/benchmarks/bench_blas.py +++ b/benchmark/pybench/benchmarks/bench_blas.py @@ -15,7 +15,7 @@ # dnrm2 -dnrm2_sizes = [100, 200, 400, 600, 800, 1000] +dnrm2_sizes = [100, 1000] def run_dnrm2(n, x, incx, func): res = func(x, n, incx=incx) @@ -35,7 +35,7 @@ def test_nrm2(benchmark, n, variant): # ddot -ddot_sizes = [100, 200, 400, 600, 800, 1000] +ddot_sizes = [100, 1000] def run_ddot(x, y, func): res = func(x, y) @@ -54,7 +54,7 @@ def test_dot(benchmark, n): # daxpy -daxpy_sizes = [100, 200, 400, 600, 800, 1000] +daxpy_sizes = [100, 1000] def run_daxpy(x, y, func): res = func(x, y, a=2.0) @@ -79,7 +79,7 @@ def test_daxpy(benchmark, n, variant): # dgemm -gemm_sizes = [100, 200, 400, 600, 800, 1000] +gemm_sizes = [100, 1000] def run_gemm(a, b, c, func): alpha = 1.0 @@ -102,7 +102,7 @@ def test_gemm(benchmark, n, variant): # dsyrk -syrk_sizes = [100, 200, 400, 600, 800, 1000] +syrk_sizes = [100, 1000] def run_syrk(a, c, func): @@ -126,7 +126,7 @@ def test_syrk(benchmark, n, variant): # linalg.solve -gesv_sizes = [100, 200, 400, 600, 800, 1000] +gesv_sizes = [100, 1000] def run_gesv(a, b, func): @@ -187,7 +187,7 @@ def test_gesdd(benchmark, mn, variant): # linalg.eigh -syev_sizes = [50, 64, 128, 200] +syev_sizes = [50, 200] def run_syev(a, lwork, func): From 11a0c56166d837698f019a6485f7159946194eaf Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Thu, 27 Jun 2024 11:09:51 +0300 Subject: [PATCH 156/267] BENCH: add BLAS level 2 gemv and gbmv --- benchmark/pybench/benchmarks/bench_blas.py | 56 ++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/benchmark/pybench/benchmarks/bench_blas.py b/benchmark/pybench/benchmarks/bench_blas.py index cface1685f..70ea030731 100644 --- a/benchmark/pybench/benchmarks/bench_blas.py +++ b/benchmark/pybench/benchmarks/bench_blas.py @@ -73,6 +73,62 @@ def test_daxpy(benchmark, n, variant): result = benchmark(run_daxpy, x, y, axpy) +# ### BLAS level 2 ### + +gemv_sizes = [100, 1000] + +def run_gemv(a, x, y, func): + res = func(1.0, a, x, y=y, overwrite_y=True) + return res + + +@pytest.mark.parametrize('variant', ['s', 'd', 'c', 'z']) +@pytest.mark.parametrize('n', gemv_sizes) +def test_dgemv(benchmark, n, variant): + rndm = np.random.RandomState(1234) + dtyp = dtype_map[variant] + + x = np.array(rndm.uniform(size=(n,)), dtype=dtyp) + y = np.empty(n, dtype=dtyp) + + a = np.array(rndm.uniform(size=(n,n)), dtype=dtyp) + x = np.array(rndm.uniform(size=(n,)), dtype=dtyp) + y = np.zeros(n, dtype=dtyp) + + gemv = ow.get_func('gemv', variant) + result = benchmark(run_gemv, a, x, y, gemv) + + assert result is y + + +# dgbmv + +dgbmv_sizes = [100, 1000] + +def run_gbmv(m, n, kl, ku, a, x, y, func): + res = func(m, n, kl, ku, 1.0, a, x, y=y, overwrite_y=True) + return res + + + +@pytest.mark.parametrize('variant', ['s', 'd', 'c', 'z']) +@pytest.mark.parametrize('n', dgbmv_sizes) +@pytest.mark.parametrize('kl', [1]) +def test_dgbmv(benchmark, n, kl, variant): + rndm = np.random.RandomState(1234) + dtyp = dtype_map[variant] + + x = np.array(rndm.uniform(size=(n,)), dtype=dtyp) + y = np.empty(n, dtype=dtyp) + + m = n + + a = rndm.uniform(size=(2*kl + 1, n)) + a = np.array(a, dtype=dtyp, order='F') + + gbmv = ow.get_func('gbmv', variant) + result = benchmark(run_gbmv, m, n, kl, kl, a, x, y, gbmv) + assert result is y # ### BLAS level 3 ### From c1019d583248c0ffbf85a33cd7eaa232acacfca9 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 27 Jun 2024 10:58:59 +0200 Subject: [PATCH 157/267] Handle INF and NAN in inputs --- kernel/riscv64/scal.c | 5 ++++- kernel/riscv64/scal_vector.c | 4 ++-- kernel/riscv64/zscal.c | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/kernel/riscv64/scal.c b/kernel/riscv64/scal.c index 4ef49e2934..6c713aa18c 100644 --- a/kernel/riscv64/scal.c +++ b/kernel/riscv64/scal.c @@ -48,7 +48,10 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS { if ( da == 0.0 ) - x[i]=0.0; + if (isfinite(x[i])) + x[i]=0.0; + else + x[i]=NAN; else x[i] = da * x[i] ; diff --git a/kernel/riscv64/scal_vector.c b/kernel/riscv64/scal_vector.c index 8fa9315f6b..a1ba41c4f6 100644 --- a/kernel/riscv64/scal_vector.c +++ b/kernel/riscv64/scal_vector.c @@ -71,7 +71,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS FLOAT_V_T v0, v1; unsigned int gvl = 0; if(inc_x == 1){ - if(da == 0.0){ + if (0){ //if(da == 0.0){ memset(&x[0], 0, n * sizeof(FLOAT)); }else{ gvl = VSETVL(n); @@ -96,7 +96,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS } } }else{ - if(da == 0.0){ + if (0) { //if(da == 0.0){ BLASLONG stride_x = inc_x * sizeof(FLOAT); BLASLONG ix = 0; gvl = VSETVL(n); diff --git a/kernel/riscv64/zscal.c b/kernel/riscv64/zscal.c index c4855f73ea..8499145f47 100644 --- a/kernel/riscv64/zscal.c +++ b/kernel/riscv64/zscal.c @@ -61,9 +61,9 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r,FLOAT da_i, F { temp = - da_i * x[ip+1] ; if (isnan(x[ip]) || isinf(x[ip])) temp = NAN; - if (!isinf(x[ip+1])) + if (!isinf(x[ip+1])) x[ip+1] = da_i * x[ip] ; - else x[ip+1] = NAN; + else x[ip+1] = NAN; } } else From 28fb95d0beb201e26517d8030c5ed259cc3ca75a Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Thu, 27 Jun 2024 12:38:47 +0300 Subject: [PATCH 158/267] BENCH: actually add gemv/gbmv f2py wrappers --- .../pybench/openblas_wrap/blas_lapack.pyf.src | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/benchmark/pybench/openblas_wrap/blas_lapack.pyf.src b/benchmark/pybench/openblas_wrap/blas_lapack.pyf.src index d2d94baa0e..1ee1d3c30b 100644 --- a/benchmark/pybench/openblas_wrap/blas_lapack.pyf.src +++ b/benchmark/pybench/openblas_wrap/blas_lapack.pyf.src @@ -111,6 +111,97 @@ function nrm2(n,x,offx,incx) result(n2) end function nrm2 + +! +! Level 2 BLAS +! + + +subroutine gemv(m,n,alpha,a,x,beta,y,offx,incx,offy,incy,trans,rows,cols,ly) + ! Computes a matrix-vector product using a general matrix + ! + ! y = gemv(alpha,a,x,beta=0,y=0,offx=0,incx=1,offy=0,incy=0,trans=0) + ! Calculate y <- alpha * op(A) * x + beta * y + + callstatement (*f2py_func)((trans?(trans==2?"C":"T"):"N"),&m,&n,&alpha,a,&m, & + x+offx,&incx,&beta,y+offy,&incy) + callprotoargument char*,F_INT*,F_INT*,*,*,F_INT*,*,F_INT*,*, & + *,F_INT* + + integer optional, intent(in), check(trans>=0 && trans <=2) :: trans = 0 + integer optional, intent(in), check(incx>0||incx<0) :: incx = 1 + integer optional, intent(in), check(incy>0||incy<0) :: incy = 1 + intent(in) :: alpha + intent(in), optional :: beta = <0.0,\0,(0.0\,0.0),\2> + + dimension(*), intent(in) :: x + dimension(ly), intent(in,copy,out), depend(ly),optional :: y + integer intent(hide), depend(incy,rows,offy) :: ly = & + (y_capi==Py_None?1+offy+(rows-1)*abs(incy):-1) + dimension(m,n), intent(in) :: a + integer depend(a), intent(hide):: m = shape(a,0) + integer depend(a), intent(hide):: n = shape(a,1) + + integer optional, intent(in) :: offx=0 + integer optional, intent(in) :: offy=0 + check(offx>=0 && offxoffx+(cols-1)*abs(incx)) :: x + depend(offx,cols,incx) :: x + + check(offy>=0 && offyoffy+(rows-1)*abs(incy)) :: y + depend(offy,rows,incy) :: y + + integer depend(m,n,trans), intent(hide) :: rows = (trans?n:m) + integer depend(m,n,trans), intent(hide) :: cols = (trans?m:n) + +end subroutine gemv + + +subroutine gbmv(m,n,kl,ku,alpha,a,lda,x,incx,offx,beta,y,incy,offy,trans,ly) + ! Performs one of the matrix-vector operations + ! + ! y := alpha*A*x + beta*y, or y := alpha*A**T*x + beta*y, + ! or y := alpha*A**H*x + beta*y, + ! + ! where alpha and beta are scalars, x and y are vectors and A is an + ! m by n band matrix, with kl sub-diagonals and ku super-diagonals. + + callstatement (*f2py_func)((trans?(trans==2?"C":"T"):"N"),&m,&n,&kl,&ku,&alpha,a,&lda,x+offx,&incx,&beta,y+offy,&incy) + callprotoargument char*,F_INT*,F_INT*,F_INT*,F_INT*,*,*,F_INT*,*,F_INT*,*,*,F_INT* + + integer optional,intent(in),check(trans>=0 && trans <=2) :: trans = 0 + integer intent(in), depend(ku,kl),check(m>=ku+kl+1) :: m + integer intent(in),check(n>=0&&n==shape(a,1)),depend(a) :: n + integer intent(in),check(kl>=0) :: kl + integer intent(in),check(ku>=0) :: ku + integer intent(hide),depend(a) :: lda = MAX(shape(a,0),1) + integer optional, intent(in),check(incx>0||incx<0) :: incx = 1 + integer optional, intent(in),check(incy>0||incy<0) :: incy = 1 + integer intent(hide),depend(m,n,incy,offy,trans) :: ly = & + (y_capi==Py_None?1+offy+(trans==0?m-1:n-1)*abs(incy):-1) + integer optional, intent(in) :: offx=0 + integer optional, intent(in) :: offy=0 + + intent(in) :: alpha + intent(in),optional :: beta = <0.0,\0,(0.0\,0.0),\2> + + dimension(lda,n),intent(in) :: a + + dimension(ly), intent(in,out,copy,out=yout),depend(ly),optional :: y + check(offy>=0 && offyoffy+(trans==0?m-1:n-1)*abs(incy)) :: y + depend(offy,n,incy) :: y + + dimension(*), intent(in) :: x + check(offx>=0 && offxoffx+(trans==0?n-1:m-1)*abs(incx)) :: x + depend(offx,n,incx) :: x + +end subroutine gbmv + + + ! ! Level 3 BLAS ! From 2a5fe97e3b01c13682764ec61e297a6252265887 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 27 Jun 2024 16:21:57 +0200 Subject: [PATCH 159/267] temporarily(?) disable the alpha=0 branch as it does not handle INF,NAN --- kernel/power/scal.S | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/power/scal.S b/kernel/power/scal.S index 19fdd32aba..7d3e232450 100644 --- a/kernel/power/scal.S +++ b/kernel/power/scal.S @@ -84,8 +84,9 @@ cmpwi cr0, N, 0 blelr- cr0 - fcmpu cr0, FZERO, ALPHA - bne- cr0, LL(A1I1) +// fcmpu cr0, FZERO, ALPHA +// bne- cr0, LL(A1I1) + b LL(A1I1) cmpwi cr0, INCX, SIZE bne- cr0, LL(A0IN) From f3c364c2cc71b80584baa70e1ba5e24bd70d8435 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 27 Jun 2024 22:18:27 +0200 Subject: [PATCH 160/267] temporarily(?) disable the alpha=0 branch as it fails to handle INF,NAN --- kernel/sparc/scal.S | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kernel/sparc/scal.S b/kernel/sparc/scal.S index 36d9ce2abd..fd61c82018 100644 --- a/kernel/sparc/scal.S +++ b/kernel/sparc/scal.S @@ -120,8 +120,10 @@ FCLR(29) - FCMP ALPHA, FZERO - fbne .LL100 +// FCMP ALPHA, FZERO +// fbne .LL100 + b .LL100 + sll INCX, BASE_SHIFT, INCX cmp INCX, SIZE From 1b52e3dcfafc6f8a6602cf4cb50550ebebfd173a Mon Sep 17 00:00:00 2001 From: Pol Dellaiera Date: Fri, 28 Jun 2024 08:33:38 +0200 Subject: [PATCH 161/267] openblas: fix `BUFFERSIZE` value --- common_x86_64.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common_x86_64.h b/common_x86_64.h index dda168d6cb..21cd198f34 100644 --- a/common_x86_64.h +++ b/common_x86_64.h @@ -253,7 +253,7 @@ static __inline unsigned int blas_quickdivide(unsigned int x, unsigned int y){ #ifndef BUFFERSIZE #define BUFFER_SIZE (32 << 22) #else -#define BUFFER_SIZE (32 << BUFFERSIZE) +#define BUFFER_SIZE (32UL << BUFFERSIZE) #endif #define SEEK_ADDRESS From e8da541fb4af0ddca9ed95d0240a5a40529fee79 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Fri, 28 Jun 2024 09:48:00 -0500 Subject: [PATCH 162/267] fix regex for renaming callbacks --- Makefile.install | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile.install b/Makefile.install index b2bc5aa412..00266b26e3 100644 --- a/Makefile.install +++ b/Makefile.install @@ -72,18 +72,18 @@ ifndef NO_CBLAS @echo Generating cblas.h in $(DESTDIR)$(OPENBLAS_INCLUDE_DIR) @cp cblas.h cblas.tmp ifdef SYMBOLPREFIX - @sed 's/cblas[^( ]*/$(SYMBOLPREFIX)&/g' cblas.tmp > cblas.tmp2 - @sed 's/openblas[^( ]*/$(SYMBOLPREFIX)&/g' cblas.tmp2 > cblas.tmp + @sed 's/cblas[^() ]*/$(SYMBOLPREFIX)&/g' cblas.tmp > cblas.tmp2 + @sed 's/openblas[^() ]*/$(SYMBOLPREFIX)&/g' cblas.tmp2 > cblas.tmp #change back any openblas_complex_float and double that got hit @sed 's/$(SYMBOLPREFIX)openblas_complex_/openblas_complex_/g' cblas.tmp > cblas.tmp2 - @sed 's/goto[^( ]*/$(SYMBOLPREFIX)&/g' cblas.tmp2 > cblas.tmp + @sed 's/goto[^() ]*/$(SYMBOLPREFIX)&/g' cblas.tmp2 > cblas.tmp endif ifdef SYMBOLSUFFIX - @sed 's/cblas[^( ]*/&$(SYMBOLSUFFIX)/g' cblas.tmp > cblas.tmp2 - @sed 's/openblas[^( ]*/&$(SYMBOLSUFFIX)/g' cblas.tmp2 > cblas.tmp + @sed 's/cblas[^() ]*/&$(SYMBOLSUFFIX)/g' cblas.tmp > cblas.tmp2 + @sed 's/openblas[^() ]*/&$(SYMBOLSUFFIX)/g' cblas.tmp2 > cblas.tmp #change back any openblas_complex_float and double that got hit @sed 's/\(openblas_complex_\)\([^ ]*\)$(SYMBOLSUFFIX)/\1\2 /g' cblas.tmp > cblas.tmp2 - @sed 's/goto[^( ]*/&$(SYMBOLSUFFIX)/g' cblas.tmp2 > cblas.tmp + @sed 's/goto[^() ]*/&$(SYMBOLSUFFIX)/g' cblas.tmp2 > cblas.tmp endif @sed 's/common/openblas_config/g' cblas.tmp > "$(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/cblas.h" endif From 017a26863a8e2e742fe3d44ac54dc817e600abb3 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Fri, 28 Jun 2024 20:12:24 +0200 Subject: [PATCH 163/267] Update GitHub Actions used in docs.yml - Updates to latest versions for all actions - Pin actions not maintained by GitHub under `github.com/action` to exact commit hashes. This is good practice for security, since tags can be moved and then arbitray new code may execute. Since updating commit hashes is slightly more hassle, the trade-off made here is to trust GitHub, but not third-party actions. This should also solve the warnings about Node.js 16 actions being deprecated that is currently visible on the log page when docs.yml is executed. --- .github/workflows/docs.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 203097812f..8f594cdd18 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -9,15 +9,15 @@ jobs: if: "github.repository == 'OpenMathLib/OpenBLAS'" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: "3.10" - run: pip install mkdocs mkdocs-material # mkdocs gh-deploy command only builds to the top-level, hence building then deploying ourselves - run: mkdocs build - name: Deploy docs - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 if: ${{ github.ref == 'refs/heads/develop' }} with: github_token: ${{ secrets.GITHUB_TOKEN }} From 97df47644d3f51239b5dc5e5625beabdd1ff6ff9 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Sat, 29 Jun 2024 09:21:50 +0200 Subject: [PATCH 164/267] Improve html theme: dark mode, nicer color scheme, icons for github/license, etc. --- mkdocs.yml | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index 74a18a5bb8..374b03e398 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,17 +1,44 @@ site_name: OpenBLAS site_url: https://openblas.net/docs/ +repo_url: https://github.com/OpenMathLib/OpenBLAS +copyright: Copyright © 2012- OpenBLAS contributors + theme: name: material logo: logo.svg favicon: logo.svg + features: + - header.autohide palette: - primary: grey + # Palette toggle for dark mode + - scheme: slate + primary: blue grey + toggle: + icon: material/brightness-4 + name: Switch to light mode + + # Palette toggle for light mode + - scheme: default + primary: blue grey + toggle: + icon: material/brightness-7 + name: Switch to dark mode + +plugins: + - search + - git-revision-date-localized: + enable_creation_date: true + markdown_extensions: - admonition - pymdownx.details - pymdownx.superfences + - footnotes + - pymdownx.tabbed: + alternate_style: true - toc: toc_depth: 4 + nav: - index.md - install.md @@ -23,3 +50,10 @@ nav: - ci.md - about.md - faq.md + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/OpenMathLib/OpenBLAS + - icon: material/license + link: https://github.com/OpenMathLib/OpenBLAS/LICENSE From 3a8e72c0e6a13a3ad2ad7361b138b246dd81a368 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Sat, 29 Jun 2024 09:52:30 +0200 Subject: [PATCH 165/267] docs: improve the "About" documentation page --- docs/about.md | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/docs/about.md b/docs/about.md index 95acdccc8c..fc10aa171c 100644 --- a/docs/about.md +++ b/docs/about.md @@ -2,25 +2,44 @@ We have a [GitHub discussions](https://github.com/OpenMathLib/OpenBLAS/discussions/) forum to discuss usage and development of OpenBLAS. We also have a [Google group for *users*](https://groups.google.com/forum/#!forum/openblas-users) and a [Google group for *development of*](https://groups.google.com/forum/#!forum/openblas-dev) OpenBLAS. -## Donations - -You can read OpenBLAS statement of receipts and disbursement and cash balance on [google doc](https://docs.google.com/spreadsheet/ccc?key=0AghkTjXe2lDndE1UZml0dGpaUzJmZGhvenBZd1F2R1E&usp=sharing). A backer list is available [on GitHub](https://github.com/OpenMathLib/OpenBLAS/blob/develop/BACKERS.md). - -We welcome the hardware donation, including the latest CPU and boards. - ## Acknowledgements -This work is partially supported by +This work was or is partially supported by the following grants, contracts and institutions: + * Research and Development of Compiler System and Toolchain for Domestic CPU, National S&T Major Projects: Core Electronic Devices, High-end General Chips and Fundamental Software (No.2009ZX01036-001-002) * National High-tech R&D Program of China (Grant No.2012AA010903) +* Chan Zuckerberg Initiative's Essential Open Source Software for Science program: + * Cycle 1 grant: [Strengthening NumPy's foundations - growing beyond code](https://figshare.com/articles/journal_contribution/Proposal_NumPy_OpenBLAS_for_Chan_Zuckerberg_Initiative_EOSS_2019_round_1/10302167) (2019-2020) + * Cycle 3 grant: [Improving usability and sustainability for NumPy and OpenBLAS](https://chanzuckerberg.com/eoss/proposals/improving-usability-and-sustainability-for-numpy-and-openblas/) (2020-2021) +* Sovereign Tech Fund funding: [Keeping high performance linear algebra computation accessible and open for all](https://www.sovereigntechfund.de/tech/openblas) (2023-2024) + +Over the course of OpenBLAS development, a number of donations were received. +You can read OpenBLAS's statement of receipts and disbursement and cash balance in +[this Google doc](https://docs.google.com/spreadsheet/ccc?key=0AghkTjXe2lDndE1UZml0dGpaUzJmZGhvenBZd1F2R1E&usp=sharing) (covers 2013-2016). +A list of backers is available [in BACKERS.md](https://github.com/OpenMathLib/OpenBLAS/blob/develop/BACKERS.md) in the main repo. + +### Donations + +We welcome hardware donations, including the latest CPUs and motherboards. + ## Users of OpenBLAS -* Julia - a high-level, high-performance dynamic programming language for technical computing
-* Ceemple v1.0.3 (C++ technical computing environment), including OpenBLAS, Qt, Boost, OpenCV and others. The only solution with immediate-recompilation of C++ code. Available from Ceemple C++ Technical Computing. -* [netlib-java](https://github.com/fommil/netlib-java) and various upstream libraries, allowing OpenBLAS to be used from languages on the Java Virtual Machine. +Prominent open source users of OpenBLAS include: + +* [Julia](https://julialang.org) - a high-level, high-performance dynamic programming language for technical computing +* [NumPy](https://numpy.org) - the fundamental package for scientific computing with Python +* [SciPy](https://scipy.org) - fundamental algorithms for scientific computing in Python +* [R](https://www.r-project.org/) - a free software environment for statistical computing and graphics +* [OpenCV](https://opencv.org/) - the world's biggest computer vision library + +OpenBLAS is packaged in most major Linux distros, as well as general and +numerical computing-focused packaging ecosystems like Nix, Homebrew, Spack and +conda-forge. + +OpenBLAS is used directly by libraries written in C, C++ and Fortran (and +probably other languages), and directly by end users in those languages. - ## Publications From 237c2c4130cea13054e80f95344c8899ac33de79 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Sat, 29 Jun 2024 09:56:30 +0200 Subject: [PATCH 166/267] docs: fix footnote rendering on "Redistributing OpenBLAS" page --- docs/distributing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/distributing.md b/docs/distributing.md index ff481b048d..98b390a9ff 100644 --- a/docs/distributing.md +++ b/docs/distributing.md @@ -49,7 +49,7 @@ settings): to provide an ILP64 interface build as well, use a symbol suffix to avoid symbol name clashes (see the next section). -[^1] All major distributions do include LAPACK as of mid 2023 as far as we +[^1]: All major distributions do include LAPACK as of mid 2023 as far as we know. Older versions of Arch Linux did not, and that was known to cause problems. From c1b9bb8519fd5910903a6b133bb0f1b31c73c195 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Sat, 29 Jun 2024 11:38:46 +0200 Subject: [PATCH 167/267] docs: improvements to the User Manual --- docs/user_manual.md | 206 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 166 insertions(+), 40 deletions(-) diff --git a/docs/user_manual.md b/docs/user_manual.md index 4d5fa9eaa1..c403ab2b29 100644 --- a/docs/user_manual.md +++ b/docs/user_manual.md @@ -1,70 +1,174 @@ -## Compile the library + +This user manual covers compiling OpenBLAS itself, linking your code to OpenBLAS, +example code to use the C (CBLAS) and Fortran (BLAS) APIs, and some troubleshooting +tips. Compiling OpenBLAS is optional, since you may be able to install with a +package manager. + +!!! Note BLAS API reference documentation + + The OpenBLAS documentation does not contain API reference documentation for + BLAS or LAPACK, since these are standardized APIs, the documentation for + which can be found in other places. If you want to understand every BLAS + function and definition, we recommend reading the + [Intel MKL reference manual](https://software.intel.com/en-us/intel-mkl/documentation) + or the [Netlib BLAS documentation](http://netlib.org/blas/). + + OpenBLAS does contain a limited number of functions that are non-standard, + these are documented at [OpenBLAS extension functions](extensions.md). + + +## Compiling OpenBLAS + ### Normal compile - * type `make` to detect the CPU automatically. - or - * type `make TARGET=xxx` to set target CPU, e.g. `make TARGET=NEHALEM`. The full target list is in file TargetList.txt. -### Cross compile -Please set `CC` and `FC` with the cross toolchains. Then, set `HOSTCC` with your host C compiler. At last, set `TARGET` explicitly. +The default way to build and install OpenBLAS from source is with Make: +``` +make # add `-j4` to compile in parallel with 4 processes +make install +``` -Examples: +By default, the CPU architecture is detected automatically when invoking +`make`, and the build is optimized for the detected CPU. To override the +autodetection, use the `TARGET` flag: -* On x86 box, compile the library for ARM Cortex-A9 linux. +``` +# `make TARGET=xxx` sets target CPU: e.g. for an Intel Nehalem CPU: +make TARGET=NEHALEM +``` +The full list of known target CPU architectures can be found in +`TargetList.txt` in the root of the repository. -Install only gnueabihf versions. Please check https://github.com/xianyi/OpenBLAS/issues/936#issuecomment-237596847 +### Cross compile - make CC=arm-linux-gnueabihf-gcc FC=arm-linux-gnueabihf-gfortran HOSTCC=gcc TARGET=CORTEXA9 +For a basic cross-compilation with Make, three steps need to be taken: -* On X86 box, compile this library for loongson3a CPU. +- Set the `CC` and `FC` environment variables to select the cross toolchains + for C and Fortran. +- Set the `HOSTCC` environment variable to select the host C compiler (i.e. the + regular C compiler for the machine on which you are invoking the build). +- Set `TARGET` explicitly to the CPU architecture on which the produced + OpenBLAS binaries will be used. +#### Cross-compilation examples + +Compile the library for ARM Cortex-A9 linux on an x86-64 machine +_(note: install only `gnueabihf` versions of the cross toolchain - see +[this issue comment](https://github.com/OpenMathLib/OpenBLAS/issues/936#issuecomment-237596847) +for why_): ``` -make BINARY=64 CC=mips64el-unknown-linux-gnu-gcc FC=mips64el-unknown-linux-gnu-gfortran HOSTCC=gcc TARGET=LOONGSON3A +make CC=arm-linux-gnueabihf-gcc FC=arm-linux-gnueabihf-gfortran HOSTCC=gcc TARGET=CORTEXA9 ``` -* On X86 box, compile this library for loongson3a CPU with loongcc (based on Open64) compiler. +Compile OpenBLAS for a loongson3a CPU on an x86-64 machine: +``` +make BINARY=64 CC=mips64el-unknown-linux-gnu-gcc FC=mips64el-unknown-linux-gnu-gfortran HOSTCC=gcc TARGET=LOONGSON3A +``` +Compile OpenBLAS for loongson3a CPU with the `loongcc` (based on Open64) compiler on an x86-64 machine: ``` make CC=loongcc FC=loongf95 HOSTCC=gcc TARGET=LOONGSON3A CROSS=1 CROSS_SUFFIX=mips64el-st-linux-gnu- NO_LAPACKE=1 NO_SHARED=1 BINARY=32 ``` -### Debug version +### Building a debug version - make DEBUG=1 +Add `DEBUG=1` to your build command, e.g.: +``` +make DEBUG=1 +``` -### Install to the directory (optional) +### Install to a specific directory -Example: +!!! note - make install PREFIX=your_installation_directory + Installing to a directory is optional; it is also possible to use the shared or static + libraries directly from the build directory. -The default directory is /opt/OpenBLAS. Note that any flags passed to `make` during build should also be passed to `make install` to circumvent any install errors, i.e. some headers not being copied over correctly. +Use `make install` with the `PREFIX` flag to install to a specific directory: -For more information, please read [Installation Guide](install.md). +``` +make install PREFIX=/path/to/installation/directory +``` + +The default directory is `/opt/OpenBLAS`. + +!!! important + + Note that any flags passed to `make` during build should also be passed to + `make install` to circumvent any install errors, i.e. some headers not + being copied over correctly. -## Link the library +For more detailed information on building/installing from source, please read +the [Installation Guide](install.md). -* Link shared library +## Linking to OpenBLAS + +OpenBLAS can be used as a shared or a static library. + +### Link a shared library + +The shared library is normally called `libopenblas.so`, but not that the name +may be different as a result of build flags used or naming choices by a distro +packager (see [distributing.md] for details). To link a shared library named +`libopenblas.so`, the flag `-lopenblas` is needed. To find the OpenBLAS headers, +a `-I/path/to/includedir` is needed. And unless the library is installed in a +directory that the linker searches by default, also `-L` and `-Wl,-rpath` flags +are needed. For a source file `test.c` (e.g., the example code under _Call +CBLAS interface_ further down), the shared library can then be linked with: ``` gcc -o test test.c -I/your_path/OpenBLAS/include/ -L/your_path/OpenBLAS/lib -Wl,-rpath,/your_path/OpenBLAS/lib -lopenblas ``` -The `-Wl,-rpath,/your_path/OpenBLAS/lib` option to linker can be omitted if you ran `ldconfig` to update linker cache, put `/your_path/OpenBLAS/lib` in `/etc/ld.so.conf` or a file in `/etc/ld.so.conf.d`, or installed OpenBLAS in a location that is part of the `ld.so` default search path (usually /lib,/usr/lib and /usr/local/lib). Alternatively, you can set the environment variable LD_LIBRARY_PATH to point to the folder that contains libopenblas.so. Otherwise, linking at runtime will fail with a message like `cannot open shared object file: no such file or directory` +The `-Wl,-rpath,/your_path/OpenBLAS/lib` linker flag can be omitted if you +ran `ldconfig` to update linker cache, put `/your_path/OpenBLAS/lib` in +`/etc/ld.so.conf` or a file in `/etc/ld.so.conf.d`, or installed OpenBLAS in a +location that is part of the `ld.so` default search path (usually `/lib`, +`/usr/lib` and `/usr/local/lib`). Alternatively, you can set the environment +variable `LD_LIBRARY_PATH` to point to the folder that contains `libopenblas.so`. +Otherwise, the build may succeed but at runtime loading the library will fail +with a message like: +``` +cannot open shared object file: no such file or directory +``` -If the library is multithreaded, please add `-lpthread`. If the library contains LAPACK functions, please add `-lgfortran` or other Fortran libs, although if you only make calls to LAPACKE routines, i.e. your code has `#include "lapacke.h"` and makes calls to methods like `LAPACKE_dgeqrf`, `-lgfortran` is not needed. +More flags may be needed, depending on how OpenBLAS was built: -* Link static library +- If `libopenblas` is multi-threaded, please add `-lpthread`. +- If the library contains LAPACK functions (usually also true), please add + `-lgfortran` (other Fortran libraries may also be needed, e.g. `-lquadmath`). + Note that if you only make calls to LAPACKE routines, i.e. your code has + `#include "lapacke.h"` and makes calls to methods like `LAPACKE_dgeqrf`, + then `-lgfortran` is not needed. +!!! tip Use pkg-config + + Usually a pkg-config file (e.g., `openblas.pc`) is installed together + with a `libopenblas` shared library. pkg-config is a tool that will + tell you the exact flags needed for linking. For example: + + ``` + $ pkg-config --cflags openblas + -I/usr/local/include + $ pkg-config --libs openblas + -L/usr/local/lib -lopenblas + ``` + +### Link a static library + +Linking a static library is simpler - add the path to the static OpenBLAS +library to the compile command: ``` gcc -o test test.c /your/path/libopenblas.a ``` -You can download `test.c` from https://gist.github.com/xianyi/5780018 ## Code examples ### Call CBLAS interface -This example shows calling cblas_dgemm in C. https://gist.github.com/xianyi/6930656 + +This example shows calling `cblas_dgemm` in C: + + ```c #include #include @@ -83,14 +187,17 @@ void main() } ``` +To compile this file, save it as `test_cblas_dgemm.c` and then run: ``` -gcc -o test_cblas_open test_cblas_dgemm.c -I /your_path/OpenBLAS/include/ -L/your_path/OpenBLAS/lib -lopenblas -lpthread -lgfortran +gcc -o test_cblas_open test_cblas_dgemm.c -I/your_path/OpenBLAS/include/ -L/your_path/OpenBLAS/lib -lopenblas -lpthread -lgfortran ``` +will result in a `test_cblas_open` executable. ### Call BLAS Fortran interface -This example shows calling dgemm Fortran interface in C. https://gist.github.com/xianyi/5780018 +This example shows calling the `dgemm` Fortran interface in C: + ```c #include "stdio.h" #include "stdlib.h" @@ -158,22 +265,41 @@ int main(int argc, char* argv[]) } ``` +To compile this file, save it as `time_dgemm.c` and then run: ``` gcc -o time_dgemm time_dgemm.c /your/path/libopenblas.a -lpthread -./time_dgemm ``` +You can then run it as: `./time_dgemm `, with `m`, `n`, and `k` input +parameters to the `time_dgemm` executable. -## Troubleshooting +!!! note -* Please read [Faq](faq.md) at first. -* Please use gcc version 4.6 and above to compile Sandy Bridge AVX kernels on Linux/MingW/BSD. -* Please use Clang version 3.1 and above to compile the library on Sandy Bridge microarchitecture. The Clang 3.0 will generate the wrong AVX binary code. -* The number of CPUs/Cores should less than or equal to 256. On Linux x86_64(amd64), there is experimental support for up to 1024 CPUs/Cores and 128 numa nodes if you build the library with BIGNUMA=1. -* OpenBLAS does not set processor affinity by default. On Linux, you can enable processor affinity by commenting the line NO_AFFINITY=1 in Makefile.rule. But this may cause [the conflict with R parallel](https://stat.ethz.ch/pipermail/r-sig-hpc/2012-April/001348.html). -* On Loongson 3A. make test would be failed because of pthread_create error. The error code is EAGAIN. However, it will be OK when you run the same testcase on shell. + When calling the Fortran interface from C, you have to deal with symbol name + differences caused by compiler conventions. That is why the `dgemm_` function + call in the example above has a trailing underscore. This is what it looks like + when using `gcc`/`gfortran`, however such details may change for different + compilers. Hence it requires extra support code. The CBLAS interface may be + more portable when writing C code. -## BLAS reference manual + When writing code that needs to be portable and work across different + platforms and compilers, the above code example is not recommended for + usage. Instead, we advise looking at how OpenBLAS (or BLAS in general, since + this problem isn't specific to OpenBLAS) functions are called in widely + used projects like Julia, SciPy, or R. -If you want to understand every BLAS function and definition, please read [Intel MKL reference manual](https://software.intel.com/en-us/intel-mkl/documentation) or [netlib.org](http://netlib.org/blas/) -Here are [OpenBLAS extension functions](extensions.md) +## Troubleshooting + +* Please read the [FAQ](faq.md) first, your problem may be described there. +* Please ensure you are using a recent enough compiler, that supports the + features your CPU provides (example: GCC versions before 4.6 were known to + not support AVX kernels, and before 6.1 AVX512CD kernels). +* The number of CPU cores supported by default is <=256. On Linux x86-64, there + is experimental support for up to 1024 cores and 128 NUMA nodes if you build + the library with `BIGNUMA=1`. +* OpenBLAS does not set processor affinity by default. On Linux, you can enable + processor affinity by commenting out the line `NO_AFFINITY=1` in + `Makefile.rule`. +* On Loongson 3A, `make test` is known to fail with a `pthread_create` error + and an `EAGAIN` error code. However, it will be OK when you run the same + testcase in a shell. From a8e1ff84ce22ae8d158a752d8649ff4a03019749 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Sun, 30 Jun 2024 16:27:54 +0200 Subject: [PATCH 168/267] docs: improve the "Build system" page --- docs/build_system.md | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/docs/build_system.md b/docs/build_system.md index 98808fdfab..3de2205807 100644 --- a/docs/build_system.md +++ b/docs/build_system.md @@ -1,3 +1,8 @@ +This page describes the Make-based build, which is the default/authoritative +build method. Note that the OpenBLAS repository also supports building with +CMake (not described here) - that generally works and is tested, however there +may be small differences between the Make and CMake builds. + !!! warning This page is made by someone who is not the developer and should not be considered as an official documentation of the build system. For getting the full picture, it is best to read the Makefiles and understand them yourself. @@ -95,10 +100,21 @@ NUM_PARALLEL - define this to the number of OpenMP instances that your code m ``` -OpenBLAS uses a fixed set of memory buffers internally, used for communicating and compiling partial results from individual threads. -For efficiency, the management array structure for these buffers is sized at build time - this makes it necessary to know in advance how -many threads need to be supported on the target system(s). -With OpenMP, there is an additional level of complexity as there may be calls originating from a parallel region in the calling program. If OpenBLAS gets called from a single parallel region, it runs single-threaded automatically to avoid overloading the system by fanning out its own set of threads. -In the case that an OpenMP program makes multiple calls from independent regions or instances in parallel, this default serialization is not -sufficient as the additional caller(s) would compete for the original set of buffers already in use by the first call. -So if multiple OpenMP runtimes call into OpenBLAS at the same time, then only one of them will be able to make progress while all the rest of them spin-wait for the one available buffer. Setting NUM_PARALLEL to the upper bound on the number of OpenMP runtimes that you can have in a process ensures that there are a sufficient number of buffer sets available \ No newline at end of file +OpenBLAS uses a fixed set of memory buffers internally, used for communicating +and compiling partial results from individual threads. For efficiency, the +management array structure for these buffers is sized at build time - this +makes it necessary to know in advance how many threads need to be supported on +the target system(s). + +With OpenMP, there is an additional level of complexity as there may be calls +originating from a parallel region in the calling program. If OpenBLAS gets +called from a single parallel region, it runs single-threaded automatically to +avoid overloading the system by fanning out its own set of threads. In the case +that an OpenMP program makes multiple calls from independent regions or +instances in parallel, this default serialization is not sufficient as the +additional caller(s) would compete for the original set of buffers already in +use by the first call. So if multiple OpenMP runtimes call into OpenBLAS at the +same time, then only one of them will be able to make progress while all the +rest of them spin-wait for the one available buffer. Setting `NUM_PARALLEL` to +the upper bound on the number of OpenMP runtimes that you can have in a process +ensures that there are a sufficient number of buffer sets available. From 3eba16c583df433aa66dba8f2c616a69cbb8bea6 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Sun, 30 Jun 2024 17:50:01 +0200 Subject: [PATCH 169/267] docs: improve the Developer manual --- docs/developers.md | 145 ++++++++++++++++++++++++++++++--------------- 1 file changed, 97 insertions(+), 48 deletions(-) diff --git a/docs/developers.md b/docs/developers.md index 08443b7e47..b2c62eeb68 100644 --- a/docs/developers.md +++ b/docs/developers.md @@ -1,6 +1,6 @@ # Developer manual -## Source codes Layout +## Source code layout ``` OpenBLAS/ @@ -51,8 +51,7 @@ OpenBLAS/ ``` -A call tree for `dgemm` is as following. - +A call tree for `dgemm` looks as follows: ``` interface/gemm.c │ @@ -61,10 +60,9 @@ driver/level3/level3.c gemm assembly kernels at kernel/ ``` -To find the kernel currently used for a particular supported cpu, please check the corresponding `kernel/$(ARCH)/KERNEL.$(CPU)` file. - -Here is an example for `kernel/x86_64/KERNEL.HASWELL` +To find the kernel currently used for a particular supported CPU, please check the corresponding `kernel/$(ARCH)/KERNEL.$(CPU)` file. +Here is an example for `kernel/x86_64/KERNEL.HASWELL`: ``` ... DTRMMKERNEL = dtrmm_kernel_4x8_haswell.c @@ -73,71 +71,122 @@ DGEMMKERNEL = dgemm_kernel_4x8_haswell.S ``` According to the above `KERNEL.HASWELL`, OpenBLAS Haswell dgemm kernel file is `dgemm_kernel_4x8_haswell.S`. + ## Optimizing GEMM for a given hardware -Read the Goto paper to understand the algorithm. +!!! abstract "Read the Goto paper to understand the algorithm" + + Goto, Kazushige; van de Geijn, Robert A. (2008). + ["Anatomy of High-Performance Matrix Multiplication"](http://delivery.acm.org/10.1145/1360000/1356053/a12-goto.pdf?ip=155.68.162.54&id=1356053&acc=ACTIVE%20SERVICE&key=A79D83B43E50B5B8%2EF070BBE7E45C3F17%2E4D4702B0C3E38B35%2E4D4702B0C3E38B35&__acm__=1517932837_edfe766f1e295d9a7830812371e1d173). + ACM Transactions on Mathematical Software 34 (3): Article 12 + + (The above link is available only to ACM members, but this and many related + papers is also available on [the pages of van de Geijn's FLAME project](http://www.cs.utexas.edu/~flame/web/FLAMEPublications.html)) + +The `driver/level3/level3.c` is the implementation of Goto's algorithm. +Meanwhile, you can look at `kernel/generic/gemmkernel_2x2.c`, which is a naive +`2x2` register blocking `gemm` kernel in C. Then: + +* Write optimized assembly kernels. Consider instruction pipeline, available registers, memory/cache access. +* Tune cache block sizes (`Mc`, `Kc`, and `Nc`) + +Note that not all of the CPU-specific parameters in `param.h` are actively used in algorithms. +`DNUMOPT` only appears as a scale factor in profiling output of the level3 `syrk` interface code, +while its counterpart `SNUMOPT` (aliased as `NUMOPT` in `common.h`) is not used anywhere at all. -Goto, Kazushige; van de Geijn, Robert A. (2008). ["Anatomy of High-Performance Matrix Multiplication"](http://delivery.acm.org/10.1145/1360000/1356053/a12-goto.pdf?ip=155.68.162.54&id=1356053&acc=ACTIVE%20SERVICE&key=A79D83B43E50B5B8%2EF070BBE7E45C3F17%2E4D4702B0C3E38B35%2E4D4702B0C3E38B35&__acm__=1517932837_edfe766f1e295d9a7830812371e1d173). ACM Transactions on Mathematical Software 34 (3): Article 12 -(The above link is available only to ACM members, but this and many related papers is also available on the pages -of van de Geijn's FLAME project, http://www.cs.utexas.edu/~flame/web/FLAMEPublications.html ) +`SYMV_P` is only used in the generic kernels for the `symv` and `chemv`/`zhemv` functions - +at least some of those are usually overridden by CPU-specific implementations, so if you start +by cloning the existing implementation for a related CPU you need to check its `KERNEL` file +to see if tuning `SYMV_P` would have any effect at all. -The `driver/level3/level3.c` is the implementation of Goto's algorithm. Meanwhile, you can look at `kernel/generic/gemmkernel_2x2.c`, which is a naive `2x2` register blocking gemm kernel in C. +`GEMV_UNROLL` is only used by some older x86-64 kernels, so not all sections in `param.h` define it. +Similarly, not all of the CPU parameters like L2 or L3 cache sizes are necessarily used in current +kernels for a given model - by all indications the CPU identification code was imported from some +other project originally. -Then, -* Write optimized assembly kernels. consider instruction pipeline, available registers, memory/cache accessing -* Tuning cache block size, `Mc`, `Kc`, and `Nc` -Note that not all of the cpu-specific parameters in param.h are actively used in algorithms. DNUMOPT only appears as a scale factor in profiling output of the level3 syrk interface code, while its counterpart SNUMOPT (aliased as NUMOPT in common.h) is not used anywhere at all. -SYMV_P is only used in the generic kernels for the symv and chemv/zhemv functions - at least some of those are usually overridden by cpu-specific implementations, so if you start by cloning the existing implementation for a related cpu you need to check its KERNEL file to see if tuning SYMV_P would have any effect at all. -GEMV_UNROLL is only used by some older x86_64 kernels, so not all sections in param.h define it. -Similarly, not all of the cpu parameters like L2 or L3 cache sizes are necessarily used in current kernels for a given model - by all indications the cpu identification code was imported from some other project originally. +## Running OpenBLAS tests -## Run OpenBLAS Test +We use tests for Netlib BLAS, CBLAS, and LAPACK. In addition, we use +OpenBLAS-specific regression tests. They can be run with Make: -We use netlib blas test, cblas test, and LAPACK test. Meanwhile, we use [BLAS-Tester](https://github.com/xianyi/BLAS-Tester), a modified test tool from ATLAS. +* `make -C test` for BLAS tests +* `make -C ctest` for CBLAS tests +* `make -C utest` for OpenBLAS regression tests +* `make lapack-test` for LAPACK tests -* Run `test` and `ctest` at OpenBLAS. e.g. `make test` or `make ctest`. -* Run regression test `utest` at OpenBLAS. -* Run LAPACK test. e.g. `make lapack-test`. -* Clone [BLAS-Tester](https://github.com/xianyi/BLAS-Tester), which can compare the OpenBLAS result with netlib reference BLAS. +We also use the [BLAS-Tester](https://github.com/xianyi/BLAS-Tester) tests for regression testing. +It is basically the ATLAS test suite adapted for building with OpenBLAS. + +The project makes use of several Continuous Integration (CI) services +conveniently interfaced with GitHub to automatically run tests on a number of +platforms and build configurations. + +Also note that the test suites included with "numerically heavy" projects like +Julia, NumPy, SciPy, Octave or QuantumEspresso can be used for regression +testing, when those projects are built such that they use OpenBLAS. -The project makes use of several Continuous Integration (CI) services conveniently interfaced with github to automatically check compilability on a number of platforms. -Lastly, the testsuites included with "numerically heavy" projects like Julia, NumPy, Octave or QuantumEspresso can be used for regression testing. ## Benchmarking -Several simple C benchmarks for performance testing individual BLAS functions are available in the `benchmark` folder, and its `scripts` subdirectory contains corresponding versions for Python, Octave and R. -Other options include +A number of benchmarking methods are used by OpenBLAS: + +- Several simple C benchmarks for performance testing individual BLAS functions + are available in the `benchmark` folder. They can be run locally through the + `Makefile` in that directory. And the `benchmark/scripts` subdirectory + contains similar benchmarks that use OpenBLAS via NumPy, SciPy, Octave and R. +- On pull requests, a representative set of functions is tested for performance + regressions with Codspeed; results can be viewed at + [https://codspeed.io/OpenMathLib/OpenBLAS](https://codspeed.io/OpenMathLib/OpenBLAS). +- The [OpenMathLib/BLAS-Benchmarks](https://github.com/OpenMathLib/BLAS-Benchmarks) repository + contains an [Airspeed Velocity](https://github.com/airspeed-velocity/asv/)-based benchmark + suite which is run on several CPU architectures in cron jobs. Results are published + to a dashboard: [http://www.openmathlib.org/BLAS-Benchmarks/](http://www.openmathlib.org/BLAS-Benchmarks/). + +Benchmarking code for BLAS libraries, and specific performance analysis results, can be found +in a number of places. For example: -* https://github.com/RoyiAvital/MatlabJuliaMatrixOperationsBenchmark (various matrix operations in Julia and Matlab) -* https://github.com/mmperf/mmperf/ (single-core matrix multiplication) +* [MatlabJuliaMatrixOperationsBenchmark](https://github.com/RoyiAvital/MatlabJuliaMatrixOperationsBenchmark) + (various matrix operations in Julia and Matlab) +* [mmperf/mmperf](https://github.com/mmperf/mmperf/) (single-core matrix multiplication) -## Adding autodetection support for a new revision or variant of a supported cpu -Especially relevant for x86_64, a new cpu model may be a "refresh" (die shrink and/or different number of cores) within an existing -model family without significant changes to its instruction set. (e.g. Intel Skylake, Kaby Lake etc. still are fundamentally Haswell, -low end Goldmont etc. are Nehalem). In this case, compilation with the appropriate older TARGET will already lead to a satisfactory build. +## Adding autodetection support for a new revision or variant of a supported CPU + +Especially relevant for x86-64, a new CPU model may be a "refresh" (die shrink and/or different number of cores) within an existing +model family without significant changes to its instruction set (e.g., Intel Skylake and Kaby Lake still are fundamentally the same architecture as Haswell, +low end Goldmont etc. are Nehalem). In this case, compilation with the appropriate older `TARGET` will already lead to a satisfactory build. To achieve autodetection of the new model, its CPUID (or an equivalent identifier) needs to be added in the `cpuid_.c` -relevant for its general architecture, with the returned name for the new type set appropriately. For x86 which has the most complex -cpuid file, there are two functions that need to be edited - get_cpuname() to return e.g. CPUTYPE_HASWELL and get_corename() for the (broader) -core family returning e.g. CORE_HASWELL. (This information ends up in the Makefile.conf and config.h files generated by `getarch`. Failure to -set either will typically lead to a missing definition of the GEMM_UNROLL parameters later in the build, as `getarch_2nd` will be unable to -find a matching parameter section in param.h.) +relevant for its general architecture, with the returned name for the new type set appropriately. For x86, which has the most complex +`cpuid` file, there are two functions that need to be edited: `get_cpuname()` to return, e.g., `CPUTYPE_HASWELL` and `get_corename()` for the (broader) +core family returning, e.g., `CORE_HASWELL`.[^1] + +[^1]: + This information ends up in the `Makefile.conf` and `config.h` files generated by `getarch`. Failure to + set either will typically lead to a missing definition of the `GEMM_UNROLL` parameters later in the build, + as `getarch_2nd` will be unable to find a matching parameter section in `param.h`. -For architectures where "DYNAMIC_ARCH" builds are supported, a similar but simpler code section for the corresponding runtime detection of the cpu exists in `driver/others/dynamic.c` (for x86) and `driver/others/dynamic_.c` for other architectures. +For architectures where `DYNAMIC_ARCH` builds are supported, a similar but simpler code section for the corresponding +runtime detection of the CPU exists in `driver/others/dynamic.c` (for x86), and `driver/others/dynamic_.c` for other architectures. Note that for x86 the CPUID is compared after splitting it into its family, extended family, model and extended model parts, so the single decimal -number returned by Linux in /proc/cpuinfo for the model has to be converted back to hexadecimal before splitting into its constituent -digits, e.g. 142 = 8E , translates to extended model 8, model 14. +number returned by Linux in `/proc/cpuinfo` for the model has to be converted back to hexadecimal before splitting into its constituent +digits. For example, `142 == 8E` translates to extended model 8, model 14. -## Adding dedicated support for a new cpu model -Usually it will be possible to start from an existing model, clone its KERNEL configuration file to the new name to use for this TARGET and eventually replace individual kernels with versions better suited for peculiarities of the new cpu model. In addition, it is necessary to add -(or clone at first) the corresponding section of GEMM_UNROLL parameters in the toplevel param.h, and possibly to add definitions such as USE_TRMM -(governing whether TRMM functions use the respective GEMM kernel or a separate source file) to the Makefiles (and CMakeLists.txt) in the kernel -directory. The new cpu name needs to be added to TargetLists.txt and the cpu autodetection code used by the `getarch` helper program - contained in +## Adding dedicated support for a new CPU model + +Usually it will be possible to start from an existing model, clone its `KERNEL` configuration file to the new name to use for this +`TARGET` and eventually replace individual kernels with versions better suited for peculiarities of the new CPU model. +In addition, it is necessary to add (or clone at first) the corresponding section of `GEMM_UNROLL` parameters in the top-level `param.h`, +and possibly to add definitions such as `USE_TRMM` (governing whether `TRMM` functions use the respective `GEMM` kernel or a separate source file) +to the `Makefile`s (and `CMakeLists.txt`) in the kernel directory. The new CPU name needs to be added to `TargetList.txt`, +and the CPU auto-detection code used by the `getarch` helper program - contained in the `cpuid_.c` file amended to include the CPUID (or equivalent) information processing required (see preceding section). + ## Adding support for an entirely new architecture -This endeavour is best started by cloning the entire support structure for 32bit ARM, and within that the ARMV5 cpu in particular as this is implemented through plain C kernels only. An example providing a convenient "shopping list" can be seen in pull request #1526. +This endeavour is best started by cloning the entire support structure for 32-bit ARM, and within that the ARMv5 CPU in particular, +as this is implemented through plain C kernels only. An example providing a convenient "shopping list" can be seen in pull request +[#1526](https://github.com/OpenMathLib/OpenBLAS/pull/1526). From ca9a0c28e8c667be80b56f2d924caa0bcb8abe46 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Sun, 30 Jun 2024 17:58:36 +0200 Subject: [PATCH 170/267] docs: improve extensions page --- docs/extensions.md | 45 ++++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/docs/extensions.md b/docs/extensions.md index 3502f6487a..483b009289 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -1,4 +1,9 @@ -* BLAS-like extensions +OpenBLAS for the most part contains implementations of the reference (Netlib) +BLAS, CBLAS, LAPACK and LAPACKE interfaces. A few OpenBLAS-specific functions +are also provided however, which mostly can be seen as "BLAS extensions". +This page documents those non-standard APIs. + +## BLAS-like extensions | Routine | Data Types | Description | | ------------- |:------------- | :---------------| @@ -9,20 +14,26 @@ | ?geadd | s,d,c,z | matrix add | | ?gemmt | s,d,c,z | gemm but only a triangular part updated| -* BLAS-like and Conversion functions for bfloat16 (available when OpenBLAS was compiled with BUILD_BFLOAT16=1) - * `void cblas_sbstobf16` converts a float array to an array of bfloat16 values by rounding - * `void cblas_sbdtobf16` converts a double array to an array of bfloat16 values by rounding - * `void cblas_sbf16tos` converts a bfloat16 array to an array of floats - * `void cblas_dbf16tod` converts a bfloat16 array to an array of doubles - * `float cblas_sbdot` computes the dot product of two bfloat16 arrays - * `void cblas_sbgemv` performs the matrix-vector operations of GEMV with the input matrix and X vector as bfloat16 - * `void cblas_sbgemm` performs the matrix-matrix operations of GEMM with both input arrays containing bfloat16 - -* Utility functions - * openblas_get_num_threads - * openblas_set_num_threads - * `int openblas_get_num_procs(void)` returns the number of processors available on the system (may include "hyperthreading cores") - * `int openblas_get_parallel(void)` returns 0 for sequential use, 1 for platform-based threading and 2 for OpenMP-based threading - * `char * openblas_get_config()` returns the options OpenBLAS was built with, something like `NO_LAPACKE DYNAMIC_ARCH NO_AFFINITY Haswell` - * `int openblas_set_affinity(int thread_index, size_t cpusetsize, cpu_set_t *cpuset)` sets the cpu affinity mask of the given thread to the provided cpuset. (Only available under Linux, with semantics identical to pthread_setaffinity_np) + +## bfloat16 functionality + +BLAS-like and conversion functions for `bfloat16` (available when OpenBLAS was compiled with `BUILD_BFLOAT16=1`): + +* `void cblas_sbstobf16` converts a float array to an array of bfloat16 values by rounding +* `void cblas_sbdtobf16` converts a double array to an array of bfloat16 values by rounding +* `void cblas_sbf16tos` converts a bfloat16 array to an array of floats +* `void cblas_dbf16tod` converts a bfloat16 array to an array of doubles +* `float cblas_sbdot` computes the dot product of two bfloat16 arrays +* `void cblas_sbgemv` performs the matrix-vector operations of GEMV with the input matrix and X vector as bfloat16 +* `void cblas_sbgemm` performs the matrix-matrix operations of GEMM with both input arrays containing bfloat16 + +## Utility functions + +* `openblas_get_num_threads` +* `openblas_set_num_threads` +* `int openblas_get_num_procs(void)` returns the number of processors available on the system (may include "hyperthreading cores") +* `int openblas_get_parallel(void)` returns 0 for sequential use, 1 for platform-based threading and 2 for OpenMP-based threading +* `char * openblas_get_config()` returns the options OpenBLAS was built with, something like `NO_LAPACKE DYNAMIC_ARCH NO_AFFINITY Haswell` +* `int openblas_set_affinity(int thread_index, size_t cpusetsize, cpu_set_t *cpuset)` sets the CPU affinity mask of the given thread + to the provided cpuset. Only available on Linux, with semantics identical to `pthread_setaffinity_np`. From d0b9948b23a51cd57f6e857d4fbd59db9a16c05a Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sun, 30 Jun 2024 19:31:15 +0200 Subject: [PATCH 171/267] Guard against invalid thread_status.queue --- driver/others/blas_server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver/others/blas_server.c b/driver/others/blas_server.c index 80c28ebda9..64130b0dd0 100644 --- a/driver/others/blas_server.c +++ b/driver/others/blas_server.c @@ -418,7 +418,7 @@ blas_queue_t *tscq; tscq = atomic_load_queue(&thread_status[cpu].queue); - while(!tscq) { + while(!tscq || tscq == 0x1) { YIELDING; if ((unsigned int)rpcc() - last_tick > thread_timeout) { From c1c0dbfd60cb65a0917f869a82e7141c1ea8a26c Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Tue, 2 Jul 2024 14:05:47 +0200 Subject: [PATCH 172/267] docs: address review comments on PR 4774 --- docs/about.md | 3 ++- docs/user_manual.md | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/about.md b/docs/about.md index fc10aa171c..dea1adab19 100644 --- a/docs/about.md +++ b/docs/about.md @@ -8,6 +8,7 @@ This work was or is partially supported by the following grants, contracts and i * Research and Development of Compiler System and Toolchain for Domestic CPU, National S&T Major Projects: Core Electronic Devices, High-end General Chips and Fundamental Software (No.2009ZX01036-001-002) * National High-tech R&D Program of China (Grant No.2012AA010903) +* [PerfXLab](http://www.perfxlab.com/) * Chan Zuckerberg Initiative's Essential Open Source Software for Science program: * Cycle 1 grant: [Strengthening NumPy's foundations - growing beyond code](https://figshare.com/articles/journal_contribution/Proposal_NumPy_OpenBLAS_for_Chan_Zuckerberg_Initiative_EOSS_2019_round_1/10302167) (2019-2020) * Cycle 3 grant: [Improving usability and sustainability for NumPy and OpenBLAS](https://chanzuckerberg.com/eoss/proposals/improving-usability-and-sustainability-for-numpy-and-openblas/) (2020-2021) @@ -23,7 +24,7 @@ A list of backers is available [in BACKERS.md](https://github.com/OpenMathLib/Op We welcome hardware donations, including the latest CPUs and motherboards. -## Users of OpenBLAS +## Open source users of OpenBLAS Prominent open source users of OpenBLAS include: diff --git a/docs/user_manual.md b/docs/user_manual.md index c403ab2b29..7abdcf0b85 100644 --- a/docs/user_manual.md +++ b/docs/user_manual.md @@ -9,9 +9,9 @@ package manager. The OpenBLAS documentation does not contain API reference documentation for BLAS or LAPACK, since these are standardized APIs, the documentation for which can be found in other places. If you want to understand every BLAS - function and definition, we recommend reading the - [Intel MKL reference manual](https://software.intel.com/en-us/intel-mkl/documentation) - or the [Netlib BLAS documentation](http://netlib.org/blas/). + and LAPACK function and definition, we recommend reading the + [Netlib BLAS ](http://netlib.org/blas/) and [Netlib LAPACK](http://netlib.org/lapack/) + documentation. OpenBLAS does contain a limited number of functions that are non-standard, these are documented at [OpenBLAS extension functions](extensions.md). From 5b385fd453a8f1e00602d6c3ffd5bd3d6c5c6dda Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Tue, 2 Jul 2024 19:23:42 +0300 Subject: [PATCH 173/267] WIP: fish out the gesdd failure? --- .github/workflows/codspeed-bench.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/codspeed-bench.yml b/.github/workflows/codspeed-bench.yml index 04befefa9e..25e196ef2a 100644 --- a/.github/workflows/codspeed-bench.yml +++ b/.github/workflows/codspeed-bench.yml @@ -139,6 +139,13 @@ jobs: cd build/openblas_wrap python -c'import _flapack; print(dir(_flapack))' + - name: Run benchmarks under pytest-benchmark + run: | + cd benchmark/pybench + pip install pytest-benchmark + export PYTHONPATH=$PWD/build-install/lib/python${{matrix.pyver}}/site-packages/ + OPENBLAS_NUM_THREADS=1 pytest benchmarks/bench_blas.py -k 'gesdd' + - name: Run benchmarks uses: CodSpeedHQ/action@v2 with: From 9d0abe2d26744e40e23345bd679ed28185fe1aea Mon Sep 17 00:00:00 2001 From: Jaap Aarts Date: Wed, 3 Jul 2024 00:16:12 +0200 Subject: [PATCH 174/267] Add support for RISCV64_GENERIC in cmake --- cmake/prebuild.cmake | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmake/prebuild.cmake b/cmake/prebuild.cmake index 6d98ed32ef..e64352f4ae 100644 --- a/cmake/prebuild.cmake +++ b/cmake/prebuild.cmake @@ -1309,6 +1309,15 @@ endif () "#define DTB_DEFAULT_ENTRIES 128\n" "#define DTB_SIZE 4096\n" "#define L2_ASSOCIATIVE 8\n") + elseif ("${TCORE}" STREQUAL "RISCV64_GENERIC") + file(APPEND ${TARGET_CONF_TEMP} + "#define L1_DATA_SIZE 32768\n" + "#define L1_DATA_LINESIZE 32\n" + "#define L2_SIZE 1048576\n" + "#define L2_LINESIZE 32 \n" + "#define DTB_DEFAULT_ENTRIES 128\n" + "#define DTB_SIZE 4096\n" + "#define L2_ASSOCIATIVE 4\n") endif() set(SBGEMM_UNROLL_M 8) set(SBGEMM_UNROLL_N 4) From cd3c167c2858903f347a66ce7c19894c1f7cefba Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Wed, 3 Jul 2024 10:58:30 +0300 Subject: [PATCH 175/267] ignore sgesdd failure on codspeed In https://github.com/OpenMathLib/OpenBLAS/issues/4776 we're hitting ** On entry to SLASCL parameter number 4 had an illegal value on codspeed, but not outside (either locally or on github runners) --- benchmark/pybench/benchmarks/bench_blas.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/benchmark/pybench/benchmarks/bench_blas.py b/benchmark/pybench/benchmarks/bench_blas.py index 70ea030731..628c0cb2a6 100644 --- a/benchmark/pybench/benchmarks/bench_blas.py +++ b/benchmark/pybench/benchmarks/bench_blas.py @@ -234,11 +234,14 @@ def test_gesdd(benchmark, mn, variant): gesdd = ow.get_func('gesdd', variant) u, s, vt, info = benchmark(run_gesdd, a, lwork, gesdd) - assert info == 0 - - atol = {'s': 1e-5, 'd': 1e-13} - - np.testing.assert_allclose(u @ np.diag(s) @ vt, a, atol=atol[variant]) + if variant != 's': + # On entry to SLASCL parameter number 4 had an illegal value + # under codspeed (cannot repro locally or on CI w/o codspeed) + # https://github.com/OpenMathLib/OpenBLAS/issues/4776 + assert info == 0 + + atol = {'s': 1e-5, 'd': 1e-13} + np.testing.assert_allclose(u @ np.diag(s) @ vt, a, atol=atol[variant]) # linalg.eigh From 74f059a3cec06591eaca8e2dbd180df3f6649d88 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 3 Jul 2024 13:24:02 +0200 Subject: [PATCH 176/267] Update OSX jobs to use the macos-12 image --- azure-pipelines.yml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7f82a461dc..f539b05948 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -141,7 +141,7 @@ jobs: - job: OSX_OpenMP pool: - vmImage: 'macOS-11' + vmImage: 'macOS-12' steps: - script: | brew update @@ -151,7 +151,7 @@ jobs: - job: OSX_GCC_Nothreads pool: - vmImage: 'macOS-11' + vmImage: 'macOS-12' steps: - script: | brew update @@ -195,7 +195,7 @@ jobs: - job: OSX_dynarch_cmake pool: - vmImage: 'macOS-11' + vmImage: 'macOS-12' variables: LD_LIBRARY_PATH: /usr/local/opt/llvm/lib LIBRARY_PATH: /usr/local/opt/llvm/lib @@ -203,7 +203,7 @@ jobs: - script: | mkdir build cd build - cmake -DTARGET=CORE2 -DDYNAMIC_ARCH=1 -DDYNAMIC_LIST='NEHALEM HASWELL SKYLAKEX' -DCMAKE_C_COMPILER=gcc-10 -DCMAKE_Fortran_COMPILER=gfortran-10 -DBUILD_SHARED_LIBS=ON .. + cmake -DTARGET=CORE2 -DDYNAMIC_ARCH=1 -DDYNAMIC_LIST='NEHALEM HASWELL SKYLAKEX' -DCMAKE_C_COMPILER=gcc-13 -DCMAKE_Fortran_COMPILER=gfortran-13 -DBUILD_SHARED_LIBS=ON .. cmake --build . ctest @@ -242,7 +242,7 @@ jobs: - job: OSX_NDK_ARMV7 pool: - vmImage: 'macOS-11' + vmImage: 'macOS-12' steps: - script: | brew update @@ -252,35 +252,35 @@ jobs: - job: OSX_IOS_ARMV8 pool: - vmImage: 'macOS-11' + vmImage: 'macOS-12' variables: - CC: /Applications/Xcode_12.4.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang - CFLAGS: -O2 -Wno-macro-redefined -isysroot /Applications/Xcode_12.4.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS14.4.sdk -arch arm64 -miphoneos-version-min=10.0 + CC: /Applications/Xcode_14.2.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang + CFLAGS: -O2 -Wno-macro-redefined -isysroot /Applications/Xcode_14.2.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS14.4.sdk -arch arm64 -miphoneos-version-min=10.0 steps: - script: | make TARGET=ARMV8 DYNAMIC_ARCH=1 NUM_THREADS=32 HOSTCC=clang NOFORTRAN=1 - job: OSX_IOS_ARMV7 pool: - vmImage: 'macOS-11' + vmImage: 'macOS-12' variables: - CC: /Applications/Xcode_12.4.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang - CFLAGS: -O2 -mno-thumb -Wno-macro-redefined -isysroot /Applications/Xcode_12.4.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS14.4.sdk -arch armv7 -miphoneos-version-min=5.1 + CC: /Applications/Xcode_14.2.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang + CFLAGS: -O2 -mno-thumb -Wno-macro-redefined -isysroot /Applications/Xcode_14.2.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS14.4.sdk -arch armv7 -miphoneos-version-min=5.1 steps: - script: | make TARGET=ARMV7 DYNAMIC_ARCH=1 NUM_THREADS=32 HOSTCC=clang NOFORTRAN=1 - job: OSX_xbuild_DYNAMIC_ARM64 pool: - vmImage: 'macOS-11' + vmImage: 'macOS-12' variables: - CC: /Applications/Xcode_12.5.1.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang - CFLAGS: -O2 -Wno-macro-redefined -isysroot /Applications/Xcode_12.5.1.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.3.sdk -arch arm64 + CC: /Applications/Xcode_14.2.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang + CFLAGS: -O2 -Wno-macro-redefined -isysroot /Applications/Xcode_14.2.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.3.sdk -arch arm64 steps: - script: | - ls /Applications/Xcode_12.5.1.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs - /Applications/Xcode_12.5.1.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang -arch arm64 --print-supported-cpus - /Applications/Xcode_11.7.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang --version + ls /Applications/Xcode_14.2.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs + /Applications/Xcode_12.2.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang -arch arm64 --print-supported-cpus + /Applications/Xcode_14.2.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang --version make TARGET=ARMV8 DYNAMIC_ARCH=1 NUM_THREADS=32 HOSTCC=clang NOFORTRAN=1 - job: ALPINE_MUSL From 2df400742597d4fdacc00c765550810526d4a74e Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 3 Jul 2024 16:48:43 +0200 Subject: [PATCH 177/267] Update compiler and sdk versions for osx --- azure-pipelines.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f539b05948..2c1fadd8ac 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -145,8 +145,8 @@ jobs: steps: - script: | brew update - make TARGET=CORE2 DYNAMIC_ARCH=1 USE_OPENMP=1 INTERFACE64=1 CC=gcc-10 FC=gfortran-10 - make TARGET=CORE2 DYNAMIC_ARCH=1 USE_OPENMP=1 INTERFACE64=1 CC=gcc-10 FC=gfortran-10 PREFIX=../blasinst install + make TARGET=CORE2 DYNAMIC_ARCH=1 USE_OPENMP=1 INTERFACE64=1 CC=gcc-13 FC=gfortran-13 + make TARGET=CORE2 DYNAMIC_ARCH=1 USE_OPENMP=1 INTERFACE64=1 CC=gcc-13 FC=gfortran-13 PREFIX=../blasinst install ls -lR ../blasinst - job: OSX_GCC_Nothreads @@ -155,7 +155,7 @@ jobs: steps: - script: | brew update - make USE_THREADS=0 CC=gcc-10 FC=gfortran-10 + make USE_THREADS=0 CC=gcc-13 FC=gfortran-13 - job: OSX_GCC12 pool: @@ -255,7 +255,7 @@ jobs: vmImage: 'macOS-12' variables: CC: /Applications/Xcode_14.2.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang - CFLAGS: -O2 -Wno-macro-redefined -isysroot /Applications/Xcode_14.2.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS14.4.sdk -arch arm64 -miphoneos-version-min=10.0 + CFLAGS: -O2 -Wno-macro-redefined -isysroot /Applications/Xcode_14.2.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS16.2.sdk -arch arm64 -miphoneos-version-min=10.0 steps: - script: | make TARGET=ARMV8 DYNAMIC_ARCH=1 NUM_THREADS=32 HOSTCC=clang NOFORTRAN=1 @@ -265,7 +265,7 @@ jobs: vmImage: 'macOS-12' variables: CC: /Applications/Xcode_14.2.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang - CFLAGS: -O2 -mno-thumb -Wno-macro-redefined -isysroot /Applications/Xcode_14.2.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS14.4.sdk -arch armv7 -miphoneos-version-min=5.1 + CFLAGS: -O2 -mno-thumb -Wno-macro-redefined -isysroot /Applications/Xcode_14.2.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS16.2.sdk -arch armv7 -miphoneos-version-min=5.1 steps: - script: | make TARGET=ARMV7 DYNAMIC_ARCH=1 NUM_THREADS=32 HOSTCC=clang NOFORTRAN=1 @@ -275,7 +275,7 @@ jobs: vmImage: 'macOS-12' variables: CC: /Applications/Xcode_14.2.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang - CFLAGS: -O2 -Wno-macro-redefined -isysroot /Applications/Xcode_14.2.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.3.sdk -arch arm64 + CFLAGS: -O2 -Wno-macro-redefined -isysroot /Applications/Xcode_14.2.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX13.1.sdk -arch arm64 steps: - script: | ls /Applications/Xcode_14.2.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs From 6ede8b14c6bb9746500f77d841955477804d0f81 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Wed, 3 Jul 2024 20:01:21 +0200 Subject: [PATCH 178/267] ci: fix CI job to deploy docs, and make it run on pull requests too --- .github/workflows/docs.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8f594cdd18..da40b853f0 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,8 +1,13 @@ name: Publish docs via GitHub Pages + on: push: branches: - develop + pull_request: + branches: + - develop + jobs: build: name: Deploy docs @@ -10,12 +15,22 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 with: python-version: "3.10" - - run: pip install mkdocs mkdocs-material - # mkdocs gh-deploy command only builds to the top-level, hence building then deploying ourselves - - run: mkdocs build + + - name: Install MkDocs and doc theme packages + run: pip install mkdocs mkdocs-material mkdocs-git-revision-date-localized-plugin + + - name: Build docs site + run: mkdocs build + + # mkdocs gh-deploy command only builds to the top-level, hence deploying + # with this action instead. + # Deploys to http://www.openmathlib.org/OpenBLAS/docs/ - name: Deploy docs uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 if: ${{ github.ref == 'refs/heads/develop' }} From cea4abcac0cc37cea6e89daec282721218196e30 Mon Sep 17 00:00:00 2001 From: Jaap Aarts Date: Thu, 4 Jul 2024 14:56:16 +0200 Subject: [PATCH 179/267] Fix compiling on mingw --- cmake/system.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/system.cmake b/cmake/system.cmake index 479f50538f..e4778249f3 100644 --- a/cmake/system.cmake +++ b/cmake/system.cmake @@ -615,7 +615,7 @@ if (${CMAKE_SYSTEM_NAME} STREQUAL "Windows") endif () if (CMAKE_Fortran_COMPILER) -if (${F_COMPILER} STREQUAL "NAG" OR ${F_COMPILER} STREQUAL "CRAY" OR CMAKE_Fortran_COMPILER_ID MATCHES "LLVMFlang.*") +if ("${F_COMPILER}" STREQUAL "NAG" OR "${F_COMPILER}" STREQUAL "CRAY" OR CMAKE_Fortran_COMPILER_ID MATCHES "LLVMFlang.*") set(FILTER_FLAGS "-msse3;-mssse3;-msse4.1;-mavx;-mavx2,-mskylake-avx512") if (CMAKE_Fortran_COMPILER_ID MATCHES "LLVMFlang.*") message(STATUS "removing fortran flags") From b4227428995ee2e8ad01550f5a00e6589493ec5b Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 4 Jul 2024 15:42:34 +0200 Subject: [PATCH 180/267] collect error output from ctest, if any --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7f82a461dc..88696a7d2f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -136,7 +136,7 @@ jobs: cmake -G "Ninja" -DCMAKE_C_COMPILER=cl -DCMAKE_Fortran_COMPILER=flang-new -DC_LAPACK=1 -DCMAKE_MT=mt -DCMAKE_BUILD_TYPE=Release -DMSVC_STATIC_CRT=ON .. cmake --build . --config Release ctest - + ctest --rerun-failed --output-on-failure - job: OSX_OpenMP From 3063d0302142c2aad46dbbc6f3d2e6ef3bb745cf Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 4 Jul 2024 16:05:05 +0200 Subject: [PATCH 181/267] Add another CPUID for Meteor Lake --- cpuid_x86.c | 1 + 1 file changed, 1 insertion(+) diff --git a/cpuid_x86.c b/cpuid_x86.c index e157232cf2..4fb51f8b3c 100644 --- a/cpuid_x86.c +++ b/cpuid_x86.c @@ -1529,6 +1529,7 @@ int get_cpuname(void){ switch (model) { case 5: // Comet Lake H and S case 6: // Comet Lake U + case 10: // Meteor Lake if(support_avx2()) return CPUTYPE_HASWELL; if(support_avx()) From 536200bc9e26069889d682143a374407c33b7adf Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 4 Jul 2024 17:47:19 +0200 Subject: [PATCH 182/267] fix handling of INF or NAN --- kernel/arm/scal.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/arm/scal.c b/kernel/arm/scal.c index 4ef49e2934..377a67f51b 100644 --- a/kernel/arm/scal.c +++ b/kernel/arm/scal.c @@ -48,7 +48,11 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS { if ( da == 0.0 ) + if (!isnan(x[i] && !isinf(x[i])) { x[i]=0.0; + } else { + x[i]=NAN; + } else x[i] = da * x[i] ; From 4547908901be81fcf5c5d453339b6bb0bb93142c Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Thu, 4 Jul 2024 09:56:42 +0200 Subject: [PATCH 183/267] docs: rewrite "Install OpenBLAS" page (part 1: binaries, basic from source) --- docs/install.md | 292 +++++++++++++++++++++++++++++------------------- 1 file changed, 179 insertions(+), 113 deletions(-) diff --git a/docs/install.md b/docs/install.md index dff18e1507..4cbb3143bf 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1,108 +1,203 @@ # Install OpenBLAS +OpenBLAS can be installed through package managers or from source. If you only +want to use OpenBLAS rather than make changes to it, we recommend installing a +pre-built binary package with your package manager of choice. + +This page contains an overview of installing with package managers as well as +from source. For the latter, see [further down on this page](#building-from-source). + + +## Installing with a package manager + !!! note - Lists of precompiled packages are not comprehensive, is not meant to validate nor endorse a particular third-party build over others, and may not always lead to the newest version + Almost every package manager provides OpenBLAS packages; the list on this + page is not comprehensive. If your package manager of choice isn't shown + here, please search its package database for `openblas` or `libopenblas`. -## Quick install +### Linux -Precompiled packages have recently become available for a number of platforms through their normal installation procedures, so for users of desktop devices at least, the instructions below are mostly relevant when you want to try the most recent development snapshot from git. See your platform's relevant "Precompiled packages" section. +On Linux, OpenBLAS can be installed with the system package manager, or with a +package manager like [Conda](https://docs.conda.io/en/latest/) +(or alternative package managers for the conda-forge ecosystem, like +[Mamba](https://mamba.readthedocs.io/en/latest/), +[Micromamba](https://mamba.readthedocs.io/en/latest/user_guide/micromamba.html), +or [Pixi](https://pixi.sh/latest/#windows-installer)), +[Spack](https://spack.io/), or [Nix](https://nixos.org/). For the latter set of +tools, the package name in all cases is `openblas`. Since package management in +quite a few of these tools is declarative (i.e., managed by adding `openblas` +to a metadata file describing the dependencies for your project or +environment), we won't attempt to give detailed instructions for these tools here. -The [Conda-Forge](https://github.com/conda-forge) project maintains packages for the conda package manager at . +Linux distributions typically split OpenBLAS up in two packages: one containing +the library itself (typically named `openblas` or `libopenblas`), and one containing headers, +pkg-config and CMake files (typically named the same as the package for the +library with `-dev` or `-devel` appended; e.g., `openblas-devel`). Please keep +in mind that if you want to install OpenBLAS in order to use it directly in +your own project, you will need to install both of those packages. -## Source -Download the latest [stable version](https://github.com/xianyi/OpenBLAS/releases) from release page. +Distro-specific installation commands: -## Platforms +=== "Debian/Ubuntu/Mint/Kali" -### Linux + ```bash + $ sudo apt update + $ sudo apt install libopenblas-dev + ``` + OpenBLAS can be configured as the default BLAS through the `update-alternatives` mechanism: -Just type `make` to compile the library. + ```bash + $ sudo update-alternatives --config libblas.so.3 + ``` -Notes: +=== "openSUSE/SLE" -* OpenBLAS doesn't support g77. Please use gfortran or other Fortran compilers. e.g. `make FC=gfortran`. -* When building in an emulator (KVM,QEMU etc.) make sure that the combination of CPU features exposed to - the virtual environment matches that of an existing CPU to allow detection of the cpu model to succeed. - (With qemu, this can be done by passing `-cpu host` or a supported model name at invocation) + ```bash + $ sudo zypper refresh + $ sudo zypper install openblas-devel + ``` + OpenBLAS can be configured as the default BLAS through the `update-alternatives` mechanism: + ```bash + $ sudo update-alternatives --config libblas.so.3 + ``` -#### Precompiled packages +=== "Fedora/CentOS/RHEL" -##### Debian/Ubuntu/Mint/Kali - OpenBLAS package is available in default repositories and can act as default BLAS in system + ```bash + $ dnf check-update + $ dnf install openblas-devel + ``` -Example installation commands: -```bash -$ sudo apt update -$ apt search openblas -$ sudo apt install libopenblas-dev -$ sudo update-alternatives --config libblas.so.3 -``` - Alternatively, if distributor's package proves unsatisfactory, you may try latest version of OpenBLAS, [Following guide in OpenBLAS FAQ](faq.md#debianlts) - -##### openSuSE/SLE - Recent OpenSUSE versions include OpenBLAS in default repositories and also permit OpenBLAS to act as replacement of system-wide BLAS. + !!! warning - Example installation commands: -```bash -$ sudo zypper ref -$ zypper se openblas -$ sudo zypper in openblas-devel -$ sudo update-alternatives --config libblas.so.3 -``` -Should you be using older OpenSUSE or SLE that provides no OpenBLAS, you can attach optional or experimental openSUSE repository as a new package source to acquire recent build of OpenBLAS following [instructions on openSUSE software site](https://software.opensuse.org/package/openblas) + Fedora does not ship the pkg-config files for OpenBLAS. Instead, it wants you to + link against [FlexiBLAS](https://www.mpi-magdeburg.mpg.de/projects/flexiblas) (which + uses OpenBLAS by default as its backend on Fedora), which you can install with: -##### Fedora/CentOS/RHEL -Fedora provides OpenBLAS in default installation repositories. + ```bash + $ dnf install flexiblas-devel + ``` -To install it try following: -```bash -$ dnf search openblas -$ dnf install openblas-devel -``` -For CentOS/RHEL/Scientific Linux packages are provided via [Fedora EPEL repository](https://fedoraproject.org/wiki/EPEL) + For CentOS and RHEL, OpenBLAS packages are provided via the [Fedora EPEL repository](https://fedoraproject.org/wiki/EPEL). + After adding that repository and its repository keys, you can install + `openblas-devel` with either `dnf` or `yum`. -After adding repository and repository keys installation is pretty straightforward: -```bash -$ yum search openblas -$ yum install openblas-devel -``` -No alternatives mechanism is provided for BLAS, and packages in system repositories are linked against NetLib BLAS or ATLAS BLAS libraries. You may wish to re-package RPMs to use OpenBLAS instead [as described here](https://fedoraproject.org/wiki/How_to_create_an_RPM_package) +=== "Arch/Manjaro/Antergos" -##### Mageia -Mageia offers ATLAS and NetLIB LAPACK in base repositories. -You can build your own OpenBLAS replacement, and once installed in /opt -TODO: populate /usr/lib64 /usr/include accurately to replicate netlib with update-alternatives + ```bash + $ sudo pacman -S openblas + ``` -##### Arch/Manjaro/Antergos -```bash -$ sudo pacman -S openblas -``` ### Windows -The precompiled binaries available with each release (in ) are -created with MinGW using an option list of -"NUM_THREADS=64 TARGET=GENERIC DYNAMIC_ARCH=1 DYNAMIC_OLDER=1 CONSISTENT_FPCSR=1" - they should work on -any x86 or x86_64 computer. The zip archive contains the include files, static and dll libraries as well -as configuration files for getting them found via CMAKE or pkgconfig - just create a suitable folder for -your OpenBLAS installation and unzip it there. (Note that you will need to edit the provided openblas.pc -and OpenBLASConfig.cmake to reflect the installation path on your computer, as distributed they have "win" -or "win64" reflecting the local paths on the system they were built on). Some programs will expect the DLL -name to be lapack.dll, blas.dll, or (in the case of the statistics package "R") even Rblas.dll to act as a -direct replacement for whatever other implementation of BLAS and LAPACK they use by default. Just copy the -openblas.dll to the desired name(s). -Note that the provided binaries are built with INTERFACE64=0, meaning they use standard 32bit integers for -array indexing and the like (as is the default for most if not all BLAS and LAPACK implementations). If the -documentation of whatever program you are using with OpenBLAS mentions 64bit integers (INTERFACE64=1) for -addressing huge matrix sizes, you will need to build OpenBLAS from source (or open an issue ticket to make -the demand for such a precompiled build known). - -#### Precompiled packages - -* -* +=== "Conda-forge" + + OpenBLAS can be installed with `conda` (or `mamba`, `micromamba`, or + `pixi`) from conda-forge: + ``` + conda install openblas + ``` + + Conda-forge provides a method for switching the default BLAS implementation + used by all packages. To use that for OpenBLAS, install `libblas=*=*openblas` + (see [the docs on this mechanism](https://conda-forge.org/docs/maintainer/knowledge_base/#switching-blas-implementation) + for more details). + +=== "vcpkg" + + OpenBLAS can be installed with vcpkg: + ```cmd + # In classic mode: + vcpkg install openblas + + # Or in manifest mode: + vcpkg add port openblas + ``` + +=== "OpenBLAS releases" + + Windows is the only platform for which binaries are made available by the + OpenBLAS project itself. They can be downloaded from the GitHub + Releases](https://github.com/OpenMathLib/OpenBLAS/releases) page. These + binaries are built with MinGW, using the following build options: + ``` + NUM_THREADS=64 TARGET=GENERIC DYNAMIC_ARCH=1 DYNAMIC_OLDER=1 CONSISTENT_FPCSR=1 INTERFACE=0 + ``` + There are separate packages for x86-64 and x86. The zip archive contains + the include files, static and shared libraries, as well as configuration + files for getting them found via CMake or pkg-config. To use these + binaries, create a suitable folder for your OpenBLAS installation and unzip + the `.zip` bundle there (note that you will need to edit the provided + `openblas.pc` and `OpenBLASConfig.cmake` to reflect the installation path + on your computer, as distributed they have "win" or "win64" reflecting the + local paths on the system they were built on). + + Note that the same binaries can be downloaded + [from SourceForge](http://sourceforge.net/projects/openblas/files); this is + mostly of historical interest. + + +### macOS + +To install OpenBLAS with a package manager on macOS, run: + +=== "Homebrew" + + ```zsh + % brew install openblas + ``` + +=== "MacPorts" + + ```zsh + % sudo port install OpenBLAS-devel + ``` + +=== "Conda-forge" + + ```zsh + % conda install openblas + ``` + + Conda-forge provides a method for switching the default BLAS implementation + used by all packages. To use that for OpenBLAS, install `libblas=*=*openblas` + (see [the docs on this mechanism](https://conda-forge.org/docs/maintainer/knowledge_base/#switching-blas-implementation) + for more details). + + +## Building from source + +We recommend download the latest [stable version](https://github.com/OpenMathLib/OpenBLAS/releases) +from the GitHub Releases page, or checking it out from a git tag, rather than a +dev version from the `develop` branch. + +!!! tip + + The User manual contains [a section with detailed information on compiling OpenBLAS](user_manual.md#compiling-openblas), + including how to customize builds and how to cross-compile. Please read + that documentation first. This page contains only platform-specific build + information, and assumes you already understand the general build system + invocations to build OpenBLAS, with the specific build options you want to + control multi-threading and other non-platform-specific behavior). + + +### Linux and macOS + +Ensure you have C and Fortran compilers installed, then simply type `make` to compile the library. +There are no other build dependencies, nor unusual platform-specific +environment variables to set or other system setup to do. + +!!! note + + When building in an emulator (KVM, QEMU, etc.), please make sure that the combination of CPU features exposed to + the virtual environment matches that of an existing CPU to allow detection of the CPU model to succeed. + (With `qemu`, this can be done by passing `-cpu host` or a supported model name at invocation). + + +### Windows #### Visual Studio @@ -181,7 +276,7 @@ In newer visual studio versions, Microsoft has changed [how it handles complex t #endif ``` -For reference, see https://github.com/xianyi/OpenBLAS/issues/3661, https://github.com/Reference-LAPACK/lapack/issues/683, and https://stackoverflow.com/questions/47520244/using-openblas-lapacke-in-visual-studio. +For reference, see https://github.com/OpenMathLib/OpenBLAS/issues/3661, https://github.com/Reference-LAPACK/lapack/issues/683, and https://stackoverflow.com/questions/47520244/using-openblas-lapacke-in-visual-studio. ###### CMake and Visual Studio @@ -208,7 +303,7 @@ Step 2 will build the OpenBLAS solution, open it in VS, and build the projects. ###### Build OpenBLAS for Universal Windows Platform -OpenBLAS can be built for use on the [Universal Windows Platform](https://en.wikipedia.org/wiki/Universal_Windows_Platform) using a two step process since commit [c66b842](https://github.com/xianyi/OpenBLAS/commit/c66b842d66c5516e52804bf5a0544d18b1da1b44). +OpenBLAS can be built for use on the [Universal Windows Platform](https://en.wikipedia.org/wiki/Universal_Windows_Platform) using a two step process since commit [c66b842](https://github.com/OpenMathLib/OpenBLAS/commit/c66b842d66c5516e52804bf5a0544d18b1da1b44). ####### 1. Follow steps 1 and 2 above to build the Visual Studio solution files for Windows. This builds the helper executables which are required when building the OpenBLAS Visual Studio solution files for UWP in step 2. @@ -246,7 +341,7 @@ Note also that older versions of the alternative builds of mingw-w64 available t ``` :0:4: error: expected identifier or '(' before numeric constant ``` -If you encounter this, please upgrade your msys2 setup or see https://github.com/xianyi/OpenBLAS/issues/1503 for a workaround. +If you encounter this, please upgrade your msys2 setup or see https://github.com/OpenMathLib/OpenBLAS/issues/1503 for a workaround. ###### Generate import library (before 0.2.10 version) @@ -266,7 +361,7 @@ Tool to do so is available at https://github.com/rainers/cv2pdb 1. Please follow the documentation about using third-party .dll libraries in MS Visual Studio 2008 or 2010. Make sure to link against a library for the correct architecture. For example, you may receive an error such as "The application was unable to start correctly (0xc000007b)" which typically indicates a mismatch between 32/64-bit libraries. !!! note - If you need CBLAS, you should include cblas.h in /your/installation/path/include in Visual Studio. Please read [this page](http://github.com/xianyi/OpenBLAS/issues/95). + If you need CBLAS, you should include cblas.h in /your/installation/path/include in Visual Studio. Please read [this page](http://github.com/OpenMathLib/OpenBLAS/issues/95). ###### Limitations * Both static and dynamic linking are supported with MinGW. With Visual Studio, however, only dynamic linking is supported and so you should use the import library. @@ -365,35 +460,6 @@ The import libraries of MSVC have the suffix `.lib`. They are generated from a ` * Always remember that MinGW is **not the same** as MSYS2 or Cygwin. MSYS2 and Cygwin are full POSIX environments with a lot of magic such as `fork()` and its own `malloc()`. MinGW, which builds on the normal Microsoft C Runtime, has none of that. Be clear about which one you are building for. -### Mac OSX - -If your CPU is Sandy Bridge, please use Clang version 3.1 and above. The Clang 3.0 will generate the wrong AVX binary code of OpenBLAS. - -#### Precompiled packages - - - -`brew install openblas` - -or using the conda package manager from - -(which also has packages for the new M1 cpu) - - `conda install openblas` - -#### Build on Apple M1 - -On newer versions of Xcode and on arm64, you might need to compile with a newer macOS target (11.0) than the default (10.8) with `MACOSX_DEPLOYMENT_TARGET=11.0`, or switch your command-line tools to use an older SDK (e.g., [13.1](https://developer.apple.com/download/all/?q=Xcode%2013)). - -* without Fortran compiler （cannot build LAPACK) - ```bash - $ make CC=cc NOFORTRAN=1 - ``` -* with Fortran compiler (you could `brew install gfortran`) https://github.com/xianyi/OpenBLAS/issues/3032 - ```bash - $ export MACOSX_DEPLOYMENT_TARGET=11.0 - $ make CC=cc FC=gfortran - ``` ### Android @@ -508,7 +574,7 @@ make \ -j4 sudo make install ``` -Also you can find full list of target architectures in [TargetsList.txt](https://github.com/xianyi/OpenBLAS/blob/develop/TargetList.txt) +Also you can find full list of target architectures in [TargetList.txt](https://github.com/OpenMathLib/OpenBLAS/blob/develop/TargetList.txt) *** anything below this line should be irrelevant nowadays unless you need to perform software archeology From 452014341e297eb658bf76afd739419c79280b6b Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Thu, 4 Jul 2024 11:05:01 +0200 Subject: [PATCH 184/267] docs: rework building from source on Windows section --- docs/install.md | 450 +++++++++++++++++++++++++++--------------------- 1 file changed, 254 insertions(+), 196 deletions(-) diff --git a/docs/install.md b/docs/install.md index 4cbb3143bf..dad4089aeb 100644 --- a/docs/install.md +++ b/docs/install.md @@ -199,266 +199,324 @@ environment variables to set or other system setup to do. ### Windows -#### Visual Studio - -As of OpenBLAS v0.2.15, we support MinGW and Visual Studio (using CMake to generate visual studio solution files – note that you will need at least version 3.11 of CMake for linking to work correctly) to build OpenBLAS on Windows. - -Note that you need a Fortran compiler if you plan to build and use the LAPACK functions included with OpenBLAS. The sections below describe using either `flang` as an add-on to clang/LLVM or `gfortran` as part of MinGW for this purpose. If you want to use the Intel Fortran compiler `ifort` for this, be sure to also use the Intel C compiler `icc` for building the C parts, as the ABI imposed by `ifort` is incompatible with `msvc`. - -##### 1. Native (MSVC) ABI - -A fully-optimized OpenBLAS that can be statically or dynamically linked to your application can currently be built for the 64-bit architecture with the LLVM compiler infrastructure. We're going to use Miniconda3 to grab all of the tools we need, since some of them are in an experimental status. Before you begin, you'll need to have Microsoft Visual Studio 2015 or newer installed. - -1. Install Miniconda3 for 64 bits using `winget install --id Anaconda.Miniconda3` or easily download from [conda.io](https://docs.conda.io/en/latest/miniconda.html). -2. Open the "Anaconda Command Prompt," now available in the Start Menu, or at `%USERPROFILE%\miniconda3\shell\condabin\conda-hook.ps1`. -3. In that command prompt window, use `cd` to change to the directory where you want to build OpenBLAS +We support building OpenBLAS with either MinGW or Visual Studio on Windows. +Using MSVC will yield an OpenBLAS build with the Windows platform-native ABI. +Using MinGW will yield a different ABI. We'll describe both methods in detail +in this section, since the process for each is quite different. + +#### Visual Studio & native Windows ABI + +For Visual Studio, you can use CMake to generate Visual Studio solution files; +note that you will need at least CMake 3.11 for linking to work correctly). + +Note that you need a Fortran compiler if you plan to build and use the LAPACK +functions included with OpenBLAS. The sections below describe using either +`flang` as an add-on to clang/LLVM or `gfortran` as part of MinGW for this +purpose. If you want to use the Intel Fortran compiler (`ifort` or `ifx`) for +this, be sure to also use the Intel C compiler (`icc` or `icx`) for building +the C parts, as the ABI imposed by `ifort` is incompatible with MSVC + +A fully-optimized OpenBLAS that can be statically or dynamically linked to your +application can currently be built for the 64-bit architecture with the LLVM +compiler infrastructure. We're going to use [Miniconda3](https://docs.anaconda.com/miniconda/) +to grab all of the tools we need, since some of them are in an experimental +status. Before you begin, you'll need to have Microsoft Visual Studio 2015 or +newer installed. + +1. Install Miniconda3 for 64-bit Windows using `winget install --id Anaconda.Miniconda3`, + or easily download from [conda.io](https://docs.conda.io/en/latest/miniconda.html). +2. Open the "Anaconda Command Prompt" now available in the Start Menu, or at `%USERPROFILE%\miniconda3\shell\condabin\conda-hook.ps1`. +3. In that command prompt window, use `cd` to change to the directory where you want to build OpenBLAS. 4. Now install all of the tools we need: - ``` conda update -n base conda conda config --add channels conda-forge conda install -y cmake flang clangdev perl libflang ninja ``` +5. Still in the Anaconda Command Prompt window, activate the 64-bit MSVC environment with `vcvarsall x64`. + On Windows 11 with Visual Studio 2022, this would be done by invoking: + + ```shell + "c:\Program Files\Microsoft Visual Studio\2022\Preview\vc\Auxiliary\Build\vcvars64.bat" + ``` + + With VS2019, the command should be the same (except for the year number of course). + For other versions of MSVC, please check the Visual Studio documentation for + exactly how to invoke the `vcvars64.bat` script. + + Confirm that the environment is active by typing `link`. This should return + a long list of possible options for the `link` command. If it just returns + _"command not found"_ or similar, review and retype the call to `vcvars64.bat`. + + !!! note + + if you are working from a Visual Studio command prompt window instead + (so that you do not have to do the `vcvars` call), you need to invoke + `conda activate` so that `CONDA_PREFIX` etc. get set up correctly before + proceeding to step 6. Failing to do so will lead to link errors like + `libflangmain.lib` not getting found later in the build. + +6. Now configure the project with CMake. Starting in the project directory, execute the following: + ``` + set "LIB=%CONDA_PREFIX%\Library\lib;%LIB%" + set "CPATH=%CONDA_PREFIX%\Library\include;%CPATH%" + mkdir build + cd build + cmake .. -G "Ninja" -DCMAKE_CXX_COMPILER=clang-cl -DCMAKE_C_COMPILER=clang-cl -DCMAKE_Fortran_COMPILER=flang -DCMAKE_MT=mt -DBUILD_WITHOUT_LAPACK=no -DNOFORTRAN=0 -DDYNAMIC_ARCH=ON -DCMAKE_BUILD_TYPE=Release + ``` -5. Still in the Anaconda Command Prompt window, activate the MSVC environment for 64 bits with `vcvarsall x64`. On Windows 11 with Visual Studio 2022, this would be done by invoking: - - ```shell - "c:\Program Files\Microsoft Visual Studio\2022\Preview\vc\Auxiliary\Build\vcvars64.bat" - ``` - - With VS2019, the command should be the same – except for the year number, obviously. For other/older versions of MSVC, - the VS documentation or a quick search on the web should turn up the exact wording you need. - - Confirm that the environment is active by typing `link` – this should return a long list of possible options for the `link` command. If it just - returns "command not found" or similar, review and retype the call to vcvars64.bat. - **NOTE:** if you are working from a Visual Studio Command prompt window instead (so that you do not have to do the vcvars call), you need to invoke - `conda activate` so that CONDA_PREFIX etc. get set up correctly before proceeding to step 6. Failing to do so will lead to link errors like - libflangmain.lib not getting found later in the build. - -6. Now configure the project with CMake. Starting in the project directory, execute the following: - - ``` - set "LIB=%CONDA_PREFIX%\Library\lib;%LIB%" - set "CPATH=%CONDA_PREFIX%\Library\include;%CPATH%" - mkdir build - cd build - cmake .. -G "Ninja" -DCMAKE_CXX_COMPILER=clang-cl -DCMAKE_C_COMPILER=clang-cl -DCMAKE_Fortran_COMPILER=flang -DCMAKE_MT=mt -DBUILD_WITHOUT_LAPACK=no -DNOFORTRAN=0 -DDYNAMIC_ARCH=ON -DCMAKE_BUILD_TYPE=Release - ``` - - You may want to add further options in the `cmake` command here – for instance, the default only produces a static .lib version of the library. If you would rather have a DLL, add -DBUILD_SHARED_LIBS=ON above. Note that this step only creates some command files and directories, the actual build happens next. - - -7. Build the project: - - ``` - cmake --build . --config Release - ``` - This step will create the OpenBLAS library in the "lib" directory, and various build-time tests in the `test`, `ctest` and `openblas_utest` directories. However it will not separate the header files you might need for building your own programs from those used internally. To put all relevant files in a more convenient arrangement, run the next step. - -8. Install all relevant files created by the build - - ``` - cmake --install . --prefix c:\opt -v - ``` - This will copy all files that are needed for building and running your own programs with OpenBLAS to the given location, creating appropriate subdirectories for the individual kinds of files. In the case of "C:\opt" as given above, this would be C:\opt\include\openblas for the header files, - C:\opt\bin for the libopenblas.dll and C:\opt\lib for the static library. C:\opt\share holds various support files that enable other cmake-based build scripts to find OpenBLAS automatically. + You may want to add further options in the `cmake` command here. For + instance, the default only produces a static `.lib` version of the library. + If you would rather have a DLL, add `-DBUILD_SHARED_LIBS=ON` above. Note that + this step only creates some command files and directories, the actual build + happens next. -###### Visual studio 2017+ (C++2017 standard) +7. Build the project: -In newer visual studio versions, Microsoft has changed [how it handles complex types](https://docs.microsoft.com/en-us/cpp/c-runtime-library/complex-math-support?view=msvc-170#types-used-in-complex-math). Even when using a precompiled version of OpenBLAS, you might need to define `LAPACK_COMPLEX_CUSTOM` in order to define complex types properly for MSVC. For example, some variant of the following might help: + ``` + cmake --build . --config Release + ``` + This step will create the OpenBLAS library in the `lib` directory, and + various build-time tests in the `test`, `ctest` and `openblas_utest` + directories. However it will not separate the header files you might need + for building your own programs from those used internally. To put all + relevant files in a more convenient arrangement, run the next step. -``` -#if defined(_MSC_VER) - #include - #define LAPACK_COMPLEX_CUSTOM - #define lapack_complex_float _Fcomplex - #define lapack_complex_double _Dcomplex -#endif -``` +8. Install all relevant files created by the build: -For reference, see https://github.com/OpenMathLib/OpenBLAS/issues/3661, https://github.com/Reference-LAPACK/lapack/issues/683, and https://stackoverflow.com/questions/47520244/using-openblas-lapacke-in-visual-studio. + ``` + cmake --install . --prefix c:\opt -v + ``` + This will copy all files that are needed for building and running your own + programs with OpenBLAS to the given location, creating appropriate + subdirectories for the individual kinds of files. In the case of `C:\opt` as + given above, this would be: + + - `C:\opt\include\openblas` for the header files, + - `C:\opt\bin` for the `libopenblas.dll` shared library, + - `C:\opt\lib` for the static library, and + - `C:\opt\share` holds various support files that enable other cmake-based + build scripts to find OpenBLAS automatically. + + +!!! tip "Change in complex types for Visual Studio 2017 and up" + + In newer Visual Studio versions, Microsoft has changed + [how it handles complex types](https://docs.microsoft.com/en-us/cpp/c-runtime-library/complex-math-support?view=msvc-170#types-used-in-complex-math). + Even when using a precompiled version of OpenBLAS, you might need to define + `LAPACK_COMPLEX_CUSTOM` in order to define complex types properly for MSVC. + For example, some variant of the following might help: + + ```c + #if defined(_MSC_VER) + #include + #define LAPACK_COMPLEX_CUSTOM + #define lapack_complex_float _Fcomplex + #define lapack_complex_double _Dcomplex + #endif + ``` -###### CMake and Visual Studio + For reference, see + [openblas#3661](https://github.com/OpenMathLib/OpenBLAS/issues/3661), + [lapack#683](https://github.com/Reference-LAPACK/lapack/issues/683), and + [this Stack Overflow question](https://stackoverflow.com/questions/47520244/using-openblas-lapacke-in-visual-studio). -To build OpenBLAS for the 32-bit architecture, you'll need to use the builtin Visual Studio compilers. -!!! note - This method may produce binaries which demonstrate significantly lower performance than those built with the other methods. (The Visual Studio compiler does not support the dialect of assembly used in the cpu-specific optimized files, so only the "generic" TARGET which is - written in pure C will get built. For the same reason it is not possible (and not necessary) to use -DDYNAMIC_ARCH=ON in a Visual Studio build) You may consider building for the 32-bit architecture using the GNU (MinGW) ABI. +!!! warning "Building 32-bit binaries with MSVC" -####### 1. Install CMake at Windows + This method may produce binaries which demonstrate significantly lower + performance than those built with the other methods. The Visual Studio + compiler does not support the dialect of assembly used in the cpu-specific + optimized files, so only the "generic" `TARGET` which is written in pure C + will get built. For the same reason it is not possible (and not necessary) + to use `-DDYNAMIC_ARCH=ON` in a Visual Studio build. You may consider + building for the 32-bit architecture using the GNU (MinGW) ABI instead. -####### 2. Use CMake to generate Visual Studio solution files +##### CMake & Visual Studio integration +To generate Visual Studio solution files, ensure CMake is installed and then run: ``` # Do this from Powershell so cmake can find visual studio cmake -G "Visual Studio 14 Win64" -DCMAKE_BUILD_TYPE=Release . ``` -###### Build the solution at Visual Studio - -Note that this step depends on perl, so you'll need to install perl for windows, and put perl on your path so VS can start perl (http://stackoverflow.com/questions/3051049/active-perl-installation-on-windows-operating-system). +To then build OpenBLAS using those solution files from within Visual Studio, we +also need Perl. Please install it and ensure it's on the `PATH` (see, e.g., +[this Stack Overflow question for how](http://stackoverflow.com/questions/3051049/active-perl-installation-on-windows-operating-system)). -Step 2 will build the OpenBLAS solution, open it in VS, and build the projects. Note that the dependencies do not seem to be automatically configured: if you try to build libopenblas directly, it will fail with a message saying that some .obj files aren't found, but if you build the projects libopenblas depends on before building libopenblas, the build will succeed. +If you build from within Visual Studio, the dependencies may not be +automatically configured: if you try to build `libopenblas` directly, it may +fail with a message saying that some `.obj` files aren't found. If this +happens, you can work around the problem by building the projects that +`libopenblas` depends on before building `libopenblas` itself. ###### Build OpenBLAS for Universal Windows Platform -OpenBLAS can be built for use on the [Universal Windows Platform](https://en.wikipedia.org/wiki/Universal_Windows_Platform) using a two step process since commit [c66b842](https://github.com/OpenMathLib/OpenBLAS/commit/c66b842d66c5516e52804bf5a0544d18b1da1b44). - -####### 1. Follow steps 1 and 2 above to build the Visual Studio solution files for Windows. This builds the helper executables which are required when building the OpenBLAS Visual Studio solution files for UWP in step 2. - -####### 2. Remove the generated CMakeCache.txt and CMakeFiles directory from the OpenBLAS source directory and re-run CMake with the following options: - -``` -# do this to build UWP compatible solution files -cmake -G "Visual Studio 14 Win64" -DCMAKE_SYSTEM_NAME=WindowsStore -DCMAKE_SYSTEM_VERSION="10.0" -DCMAKE_SYSTEM_PROCESSOR=AMD64 -DVS_WINRT_COMPONENT=TRUE -DCMAKE_BUILD_TYPE=Release . -``` - -####### Build the solution with Visual Studio - -This will build the OpenBLAS binaries with the required settings for use with UWP. - -##### 2. GNU (MinGW) ABI - -The resulting library can be used in Visual Studio, but it can only be linked dynamically. This configuration has not been thoroughly tested and should be considered experimental. +OpenBLAS can be built targeting [Universal Windows Platform](https://en.wikipedia.org/wiki/Universal_Windows_Platform) +(UWP) like this: -###### Incompatible x86 calling conventions +1. Follow the steps above to build the Visual Studio solution files for + Windows. This builds the helper executables which are required when building + the OpenBLAS Visual Studio solution files for UWP in step 2. +2. Remove the generated `CMakeCache.txt` and the `CMakeFiles` directory from + the OpenBLAS source directory, then re-run CMake with the following options: -Due to incompatibilities between the calling conventions of MinGW and Visual Studio you will need to make the following modifications ( **32-bit only** ): - -1. Use the newer GCC 4.7.0. The older GCC (<4.7.0) has an ABI incompatibility for returning aggregate structures larger than 8 bytes with MSVC. - - -###### Build OpenBLAS on Windows OS -1. Install the MinGW (GCC) compiler suite, either 32-bit (http://www.mingw.org/) or 64-bit (http://mingw-w64.sourceforge.net/). Be sure to install its gfortran package as well (unless you really want to build the BLAS part of OpenBLAS only) and check that gcc and gfortran are the same version – mixing compilers from different sources or release versions can lead to strange error messages in the linking stage. In addition, please install MSYS with MinGW. -1. Build OpenBLAS in the MSYS shell. Usually, you can just type "make". OpenBLAS will detect the compiler and CPU automatically. -1. After the build is complete, OpenBLAS will generate the static library "libopenblas.a" and the shared dll library "libopenblas.dll" in the folder. You can type "make PREFIX=/your/installation/path install" to install the library to a certain location. - -!!! note - We suggest using official MinGW or MinGW-w64 compilers. A user reported that s/he met `Unhandled exception` by other compiler suite. https://groups.google.com/forum/#!topic/openblas-users/me2S4LkE55w - -Note also that older versions of the alternative builds of mingw-w64 available through http://www.msys2.org may contain a defect that leads to a compilation failure accompanied by the error message -``` -:0:4: error: expected identifier or '(' before numeric constant -``` -If you encounter this, please upgrade your msys2 setup or see https://github.com/OpenMathLib/OpenBLAS/issues/1503 for a workaround. - -###### Generate import library (before 0.2.10 version) - -1. First, you will need to have the `lib.exe` tool in the Visual Studio command prompt. -1. Open the command prompt and type `cd OPENBLAS_TOP_DIR/exports`, where OPENBLAS_TOP_DIR is the main folder of your OpenBLAS installation. -1. For a 32-bit library, type `lib /machine:i386 /def:libopenblas.def`. For 64-bit, type `lib /machine:X64 /def:libopenblas.def`. -1. This will generate the import library "libopenblas.lib" and the export library "libopenblas.exp" in OPENBLAS_TOP_DIR/exports. Although these two files have the same name, they are totally different. - -###### Generate import library (0.2.10 and after version) -1. OpenBLAS already generated the import library "libopenblas.dll.a" for "libopenblas.dll". + ``` + # do this to build UWP compatible solution files + cmake -G "Visual Studio 14 Win64" -DCMAKE_SYSTEM_NAME=WindowsStore -DCMAKE_SYSTEM_VERSION="10.0" -DCMAKE_SYSTEM_PROCESSOR=AMD64 -DVS_WINRT_COMPONENT=TRUE -DCMAKE_BUILD_TYPE=Release . + ``` +3. Now build the solution with Visual Studio. -###### generate windows native PDB files from gcc/gfortran build -Tool to do so is available at https://github.com/rainers/cv2pdb -###### Use OpenBLAS .dll library in Visual Studio -1. Copy the import library (before 0.2.10: "OPENBLAS_TOP_DIR/exports/libopenblas.lib", 0.2.10 and after: "OPENBLAS_TOP_DIR/libopenblas.dll.a") and .dll library "libopenblas.dll" into the same folder(The folder of your project that is going to use the BLAS library. You may need to add the libopenblas.dll.a to the linker input list: properties->Linker->Input). -1. Please follow the documentation about using third-party .dll libraries in MS Visual Studio 2008 or 2010. Make sure to link against a library for the correct architecture. For example, you may receive an error such as "The application was unable to start correctly (0xc000007b)" which typically indicates a mismatch between 32/64-bit libraries. +#### MinGW & GNU ABI !!! note - If you need CBLAS, you should include cblas.h in /your/installation/path/include in Visual Studio. Please read [this page](http://github.com/OpenMathLib/OpenBLAS/issues/95). -###### Limitations -* Both static and dynamic linking are supported with MinGW. With Visual Studio, however, only dynamic linking is supported and so you should use the import library. -* Debugging from Visual Studio does not work because MinGW and Visual Studio have incompatible formats for debug information (PDB vs. DWARF/STABS). You should either debug with GDB on the command-line or with a visual frontend, for instance [Eclipse](http://www.eclipse.org/cdt/) or [Qt Creator](http://qt.nokia.com/products/developer-tools/). + The resulting library from building with MinGW as described below can be + used in Visual Studio, but it can only be linked dynamically. This + configuration has not been thoroughly tested and should be considered + experimental. + + +To build OpenBLAS on Windows with MinGW: + +1. Install the MinGW (GCC) compiler suite, either the 32-bit + [MinGW]((http://www.mingw.org/) or the 64-bit + [MinGW-w64](http://mingw-w64.sourceforge.net/) toolchain. Be sure to install + its `gfortran` package as well (unless you really want to build the BLAS part + of OpenBLAS only) and check that `gcc` and `gfortran` are the same version. + In addition, please install MSYS2 with MinGW. +2. Build OpenBLAS in the MSYS2 shell. Usually, you can just type `make`. + OpenBLAS will detect the compiler and CPU automatically. +3. After the build is complete, OpenBLAS will generate the static library + `libopenblas.a` and the shared library `libopenblas.dll` in the folder. You + can type `make PREFIX=/your/installation/path install` to install the + library to a certain location. + +Note that OpenBLAS will generate the import library `libopenblas.dll.a` for +`libopenblas.dll` by default. + +If you want to generate Windows-native PDB files from a MinGW build, you can +use the [cv2pdb](https://github.com/rainers/cv2pdb) tool to do so. + +To then use the built OpenBLAS shared library in Visual Studio: + +1. Copy the import library (`OPENBLAS_TOP_DIR/libopenblas.dll.a`) and the + shared library (`libopenblas.dll`) into the same folder (this must be the + folder of your project that is going to use the BLAS library. You may need + to add `libopenblas.dll.a` to the linker input list: `properties->Linker->Input`). +2. Please follow the Visual Studio documentation about using third-party .dll + libraries, and make sure to link against a library for the correct + architecture.[^1] +3. If you need CBLAS, you should include `cblas.h` in + `/your/installation/path/include` in Visual Studio. Please see + [openblas#95](http://github.com/OpenMathLib/OpenBLAS/issues/95) for more details. + +[^1]: + If the OpenBLAS DLLs are not linked correctly, you may see an error like + _"The application was unable to start correctly (0xc000007b)"_, which typically + indicates a mismatch between 32-bit and 64-bit libraries. + +!!! info "Limitations of using the MinGW build within Visual Studio" + + - Both static and dynamic linking are supported with MinGW. With Visual + Studio, however, only dynamic linking is supported and so you should use + the import library. + - Debugging from Visual Studio does not work because MinGW and Visual + Studio have incompatible formats for debug information (PDB vs. + DWARF/STABS). You should either debug with GDB on the command line or + with a visual frontend, for instance [Eclipse](http://www.eclipse.org/cdt/) or + [Qt Creator](http://qt.nokia.com/products/developer-tools/). #### Windows on Arm -##### Prerequisites - -Following tools needs to be installed - -###### 1. Download and install clang for windows on arm - -Find the latest LLVM build for WoA from [LLVM release page](https://releases.llvm.org/) - -E.g: LLVM 12 build for WoA64 can be found [here](https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.0/LLVM-12.0.0-woa64.exe) - -Run the LLVM installer and ensure that LLVM is added to environment PATH. - -###### 2. Download and install classic flang for windows on arm - -Classic flang is the only available FORTRAN compiler for windows on arm for now and a pre-release build can be found [here](https://github.com/kaadam/flang/releases/tag/v0.1) - -There is no installer for classic flang and the zip package can be extracted and the path needs to be added to environment PATH. - -E.g: on PowerShell - -``` -$env:Path += ";C:\flang_woa\bin" -``` +The following tools needs to be installed to build for Windows on Arm (WoA): + +- Clang for Windows on Arm. + Find the latest LLVM build for WoA from [LLVM release page](https://releases.llvm.org/). + E.g: LLVM 12 build for WoA64 can be found [here](https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.0/LLVM-12.0.0-woa64.exe) + Run the LLVM installer and ensure that LLVM is added to environment PATH. +- Download and install classic Flang for Windows on Arm. + Classic Flang is the only available Fortran compiler for Windows on Arm for now. + A pre-release build can be found [here](https://github.com/kaadam/flang/releases/tag/v0.1) + There is no installer for classic flang and the zip package can be + extracted and the path needs to be added to environment `PATH`. + E.g., in PowerShell: + ``` + $env:Path += ";C:\flang_woa\bin" + ``` -##### Build +The following steps describe how to build the static library for OpenBLAS with and without LAPACK: -The following steps describe how to build the static library for OpenBLAS with and without LAPACK +1. Build OpenBLAS static library with BLAS and LAPACK routines with Make: -###### 1. Build OpenBLAS static library with BLAS and LAPACK routines with Make + ```bash + $ make CC="clang-cl" HOSTCC="clang-cl" AR="llvm-ar" BUILD_WITHOUT_LAPACK=0 NOFORTRAN=0 DYNAMIC_ARCH=0 TARGET=ARMV8 ARCH=arm64 BINARY=64 USE_OPENMP=0 PARALLEL=1 RANLIB="llvm-ranlib" MAKE=make F_COMPILER=FLANG FC=FLANG FFLAGS_NOOPT="-march=armv8-a -cpp" FFLAGS="-march=armv8-a -cpp" NEED_PIC=0 HOSTARCH=arm64 libs netlib + ``` -Following command can be used to build OpenBLAS static library with BLAS and LAPACK routines +2. Build static library with BLAS routines using CMake: -```bash -$ make CC="clang-cl" HOSTCC="clang-cl" AR="llvm-ar" BUILD_WITHOUT_LAPACK=0 NOFORTRAN=0 DYNAMIC_ARCH=0 TARGET=ARMV8 ARCH=arm64 BINARY=64 USE_OPENMP=0 PARALLEL=1 RANLIB="llvm-ranlib" MAKE=make F_COMPILER=FLANG FC=FLANG FFLAGS_NOOPT="-march=armv8-a -cpp" FFLAGS="-march=armv8-a -cpp" NEED_PIC=0 HOSTARCH=arm64 libs netlib -``` + Classic Flang has compatibility issues with CMake, hence only BLAS routines can be compiled with CMake: -###### 2. Build static library with BLAS routines using CMake + ```bash + $ mkdir build + $ cd build + $ cmake .. -G Ninja -DCMAKE_C_COMPILER=clang -DBUILD_WITHOUT_LAPACK=1 -DNOFORTRAN=1 -DDYNAMIC_ARCH=0 -DTARGET=ARMV8 -DARCH=arm64 -DBINARY=64 -DUSE_OPENMP=0 -DCMAKE_SYSTEM_PROCESSOR=ARM64 -DCMAKE_CROSSCOMPILING=1 -DCMAKE_SYSTEM_NAME=Windows + $ cmake --build . --config Release + ``` -Classic flang has compatibility issues with cmake hence only BLAS routines can be compiled with CMake +!!! tip "`getarch.exe` execution error" -```bash -$ mkdir build -$ cd build -$ cmake .. -G Ninja -DCMAKE_C_COMPILER=clang -DBUILD_WITHOUT_LAPACK=1 -DNOFORTRAN=1 -DDYNAMIC_ARCH=0 -DTARGET=ARMV8 -DARCH=arm64 -DBINARY=64 -DUSE_OPENMP=0 -DCMAKE_SYSTEM_PROCESSOR=ARM64 -DCMAKE_CROSSCOMPILING=1 -DCMAKE_SYSTEM_NAME=Windows -$ cmake --build . --config Release -``` + If you notice that platform-specific headers by `getarch.exe` are not + generated correctly, this could be due to a known debug runtime DLL issue for + arm64 platforms. Please check out [this page](https://linaro.atlassian.net/wiki/spaces/WOAR/pages/28677636097/Debug+run-time+DLL+issue#Workaround) + for a workaround. -###### `getarch.exe` execution error -If you notice that platform-specific headers by `getarch.exe` are not generated correctly, It could be due to a known debug runtime DLL issue for arm64 platforms. Please check out [link](https://linaro.atlassian.net/wiki/spaces/WOAR/pages/28677636097/Debug+run-time+DLL+issue#Workaround) for the workaround. +#### Generating an import library -#### MinGW import library +Microsoft Windows has this thing called "import libraries". You need it for +MSVC; you don't need it for MinGW because the `ld` linker is smart enough - +however, you may still want it for some reason, so we'll describe the process +for both MSVC and MinGW. -Microsoft Windows has this thing called "import libraries". You don't need it in MinGW because the `ld` linker from GNU Binutils is smart, but you may still want it for whatever reason. +Import libraries are compiled from a list of what symbols to use, which are +contained in a `.def` file. A `.def` file should be already be present in the +`exports` directory under the top-level OpenBLAS directory after you've run a build. +In your shell, move to this directory: `cd exports`. -##### Make the `.def` +=== "MSVC" -Import libraries are compiled from a list of what symbols to use, `.def`. This should be already in your `exports` directory: `cd OPENBLAS_TOP_DIR/exports`. + Unlike MinGW, MSVC absolutely requires an import library. Now the C ABI of + MSVC and MinGW are actually identical, so linking is actually okay (any + incompatibility in the C ABI would be a bug). -##### Making a MinGW import library + The import libraries of MSVC have the suffix `.lib`. They are generated + from a `.def` file using MSVC's `lib.exe`. See [the MSVC instructions](use_visual_studio.md#generate-import-library-before-0210-version). -MinGW import libraries have the suffix `.a`, same as static libraries. (It's actually more common to do `.dll.a`...) +=== "MinGW" -You need to first prepend `libopenblas.def` with a line `LIBRARY libopenblas.dll`: + MinGW import libraries have the suffix `.a`, just like static libraries. + Our goal is to produce the file `libopenblas.dll.a`. + You need to first insert a line `LIBRARY libopenblas.dll` in `libopenblas.def`: + ``` cat <(echo "LIBRARY libopenblas.dll") libopenblas.def > libopenblas.def.1 mv libopenblas.def.1 libopenblas.def + ``` -Now it probably looks like: - + Now the `.def` file probably looks like: + ``` LIBRARY libopenblas.dll EXPORTS caxpy=caxpy_ @1 caxpy_=caxpy_ @2 ... + ``` + Then, generate the import library: `dlltool -d libopenblas.def -l libopenblas.dll.a` -Then, generate the import library: `dlltool -d libopenblas.def -l libopenblas.a` - -Again, there is basically **no point** in making an import library for use in MinGW. It actually slows down linking. - -##### Making a MSVC import library - -Unlike MinGW, MSVC absolutely requires an import library. Now the C ABI of MSVC and MinGW are actually identical, so linking is actually okay. (Any incompatibility in the C ABI would be a bug.) - -The import libraries of MSVC have the suffix `.lib`. They are generated from a `.def` file using MSVC's `lib.exe`. See [the MSVC instructions](use_visual_studio.md#generate-import-library-before-0210-version). - -##### Notes - -* Always remember that MinGW is **not the same** as MSYS2 or Cygwin. MSYS2 and Cygwin are full POSIX environments with a lot of magic such as `fork()` and its own `malloc()`. MinGW, which builds on the normal Microsoft C Runtime, has none of that. Be clear about which one you are building for. + _Again, there is basically **no point** in making an import library for use in MinGW. It actually slows down linking._ ### Android From 268dcd8f451b5f8f04a63147a7b2719e50ffc2c9 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Thu, 4 Jul 2024 19:15:07 +0200 Subject: [PATCH 185/267] docs: convert remaining install sections (Android, iOS, FreeBSD, Cortex-M) --- docs/install.md | 322 +++++++++++++++++++++--------------------------- 1 file changed, 138 insertions(+), 184 deletions(-) diff --git a/docs/install.md b/docs/install.md index dad4089aeb..1b04165c73 100644 --- a/docs/install.md +++ b/docs/install.md @@ -168,6 +168,14 @@ To install OpenBLAS with a package manager on macOS, run: for more details). +### FreeBSD + +You can install OpenBLAS from the FreeBSD [Ports collection](https://www.freebsd.org/ports/index.html): +``` +pkg install openblas +``` + + ## Building from source We recommend download the latest [stable version](https://github.com/OpenMathLib/OpenBLAS/releases) @@ -521,281 +529,221 @@ In your shell, move to this directory: `cd exports`. ### Android -#### Prerequisites +To build OpenBLAS for Android, you will need the following tools installed on your machine: -In addition to the Android NDK, you will need both Perl and a C compiler on the build host as these are currently -required by the OpenBLAS build environment. +- [The Android NDK](https://developer.android.com/ndk/) +- Perl +- Clang compiler on the build machine +The next two sections below describe how to build with Clang for ARMV7 and +ARMV8 targets, respectively. The same basic principles as described below for +ARMV8 should also apply to building an x86 or x86-64 version (substitute +something like `NEHALEM` for the target instead of `ARMV8`, and replace all the +`aarch64` in the toolchain paths with `x86` or `x96_64` as appropriate). -#### Building with android NDK using clang compiler -Around version 11 Android NDKs stopped supporting gcc, so you would need to use clang to compile OpenBLAS. clang is supported from OpenBLAS 0.2.20 version onwards. See below sections on how to build with clang for ARMV7 and ARMV8 targets. The same basic principles as described below for ARMV8 should also apply to building an x86 or x86_64 version (substitute something like NEHALEM for the target instead of ARMV8 and replace all the aarch64 in the toolchain paths obviously) -"Historic" notes: -Since version 19 the default toolchain is provided as a standalone toolchain, so building one yourself following [building a standalone toolchain](http://developer.android.com/ndk/guides/standalone_toolchain.html) should no longer be necessary. -If you want to use static linking with an old NDK version older than about r17, you need to choose an API level below 23 currently due to NDK bug 272 (https://github.com/android-ndk/ndk/issues/272 , the libc.a lacks a definition of stderr) that will probably be fixed in r17 of the NDK. +!!! info "Historic note" -#### Build ARMV7 with clang -``` -## Set path to ndk-bundle + Since NDK version 19, the default toolchain is provided as a standalone + toolchain, so building one yourself following + [building a standalone toolchain](http://developer.android.com/ndk/guides/standalone_toolchain.html) + should no longer be necessary. + + +#### Building for ARMV7 + +```bash +# Set path to ndk-bundle export NDK_BUNDLE_DIR=/path/to/ndk-bundle -## Set the PATH to contain paths to clang and arm-linux-androideabi-* utilities +# Set the PATH to contain paths to clang and arm-linux-androideabi-* utilities export PATH=${NDK_BUNDLE_DIR}/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/bin:${NDK_BUNDLE_DIR}/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH -## Set LDFLAGS so that the linker finds the appropriate libgcc +# Set LDFLAGS so that the linker finds the appropriate libgcc export LDFLAGS="-L${NDK_BUNDLE_DIR}/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/lib/gcc/arm-linux-androideabi/4.9.x" -## Set the clang cross compile flags +# Set the clang cross compile flags export CLANG_FLAGS="-target arm-linux-androideabi -marm -mfpu=vfp -mfloat-abi=softfp --sysroot ${NDK_BUNDLE_DIR}/platforms/android-23/arch-arm -gcc-toolchain ${NDK_BUNDLE_DIR}/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/" #OpenBLAS Compile make TARGET=ARMV7 ONLY_CBLAS=1 AR=ar CC="clang ${CLANG_FLAGS}" HOSTCC=gcc ARM_SOFTFP_ABI=1 -j4 ``` -On a Mac, it may also be necessary to give the complete path to the `ar` utility in the make command above, like so: -``` + +On macOS, it may also be necessary to give the complete path to the `ar` +utility in the make command above, like so: +```bash AR=${NDK_BUNDLE_DIR}/toolchains/arm-linux-androideabi-4.9/prebuilt/darwin-x86_64/bin/arm-linux-androideabi-gcc-ar ``` -otherwise you may get a linker error complaining about a "malformed archive header name at 8" when the native OSX ar command was invoked instead. +otherwise you may get a linker error complaining like `malformed archive header +name at 8` when the native macOS `ar` command was invoked instead. + -#### Build ARMV8 with clang -``` -## Set path to ndk-bundle +#### Building for ARMV8 + +```bash +# Set path to ndk-bundle export NDK_BUNDLE_DIR=/path/to/ndk-bundle/ -## Export PATH to contain directories of clang and aarch64-linux-android-* utilities +# Export PATH to contain directories of clang and aarch64-linux-android-* utilities export PATH=${NDK_BUNDLE_DIR}/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/bin/:${NDK_BUNDLE_DIR}/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH -## Setup LDFLAGS so that loader can find libgcc and pass -lm for sqrt +# Setup LDFLAGS so that loader can find libgcc and pass -lm for sqrt export LDFLAGS="-L${NDK_BUNDLE_DIR}/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/lib/gcc/aarch64-linux-android/4.9.x -lm" -## Setup the clang cross compile options +# Setup the clang cross compile options export CLANG_FLAGS="-target aarch64-linux-android --sysroot ${NDK_BUNDLE_DIR}/platforms/android-23/arch-arm64 -gcc-toolchain ${NDK_BUNDLE_DIR}/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/" -## Compile +# Compile make TARGET=ARMV8 ONLY_CBLAS=1 AR=ar CC="clang ${CLANG_FLAGS}" HOSTCC=gcc -j4 ``` -Note: Using TARGET=CORTEXA57 in place of ARMV8 will pick up better optimized routines. Implementations for CORTEXA57 target is compatible with all other armv8 targets. +Note: using `TARGET=CORTEXA57` in place of `ARMV8` will pick up better +optimized routines. Implementations for the `CORTEXA57` target are compatible +with all other `ARMV8` targets. -Note: For NDK 23b, something as simple as -``` +Note: for NDK 23b, something as simple as: +```bash export PATH=/opt/android-ndk-r23b/toolchains/llvm/prebuilt/linux-x86_64/bin/:$PATH make HOSTCC=gcc CC=/opt/android-ndk-r23b/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android31-clang ONLY_CBLAS=1 TARGET=ARMV8 ``` appears to be sufficient on Linux. -#### Alternative script which was tested on OSX with NDK(21.3.6528147) -This script will build openblas for 3 architecture (ARMV7,ARMV8,X86) and put them with `sudo make install` to `/opt/OpenBLAS/lib` -``` -export NDK=YOUR_PATH_TO_SDK/Android/sdk/ndk/21.3.6528147 -export TOOLCHAIN=$NDK/toolchains/llvm/prebuilt/darwin-x86_64 -make clean -make \ - TARGET=ARMV7 \ - ONLY_CBLAS=1 \ - CC="$TOOLCHAIN"/bin/armv7a-linux-androideabi21-clang \ - AR="$TOOLCHAIN"/bin/arm-linux-androideabi-ar \ - HOSTCC=gcc \ - ARM_SOFTFP_ABI=1 \ - -j4 -sudo make install +??? note "Alternative build script for 3 architectures" -make clean -make \ - TARGET=CORTEXA57 \ - ONLY_CBLAS=1 \ - CC=$TOOLCHAIN/bin/aarch64-linux-android21-clang \ - AR=$TOOLCHAIN/bin/aarch64-linux-android-ar \ - HOSTCC=gcc \ + This script will build OpenBLAS for 3 architecture (`ARMV7`, `ARMV8`, `X86`) and install them to `/opt/OpenBLAS/lib`. + It was tested on macOS with NDK version 21.3.6528147. + + ```bash + export NDK=YOUR_PATH_TO_SDK/Android/sdk/ndk/21.3.6528147 + export TOOLCHAIN=$NDK/toolchains/llvm/prebuilt/darwin-x86_64 + + make clean + make \ + TARGET=ARMV7 \ + ONLY_CBLAS=1 \ + CC="$TOOLCHAIN"/bin/armv7a-linux-androideabi21-clang \ + AR="$TOOLCHAIN"/bin/arm-linux-androideabi-ar \ + HOSTCC=gcc \ + ARM_SOFTFP_ABI=1 \ + -j4 + sudo make install + + make clean + make \ + TARGET=CORTEXA57 \ + ONLY_CBLAS=1 \ + CC=$TOOLCHAIN/bin/aarch64-linux-android21-clang \ + AR=$TOOLCHAIN/bin/aarch64-linux-android-ar \ + HOSTCC=gcc \ -j4 -sudo make install + sudo make install -make clean -make \ - TARGET=ATOM \ - ONLY_CBLAS=1 \ + make clean + make \ + TARGET=ATOM \ + ONLY_CBLAS=1 \ CC="$TOOLCHAIN"/bin/i686-linux-android21-clang \ AR="$TOOLCHAIN"/bin/i686-linux-android-ar \ HOSTCC=gcc \ ARM_SOFTFP_ABI=1 \ -j4 -sudo make install + sudo make install -## This will build for x86_64 -make clean -make \ - TARGET=ATOM BINARY=64\ + ## This will build for x86_64 + make clean + make \ + TARGET=ATOM BINARY=64\ ONLY_CBLAS=1 \ CC="$TOOLCHAIN"/bin/x86_64-linux-android21-clang \ AR="$TOOLCHAIN"/bin/x86_64-linux-android-ar \ HOSTCC=gcc \ ARM_SOFTFP_ABI=1 \ -j4 -sudo make install -``` -Also you can find full list of target architectures in [TargetList.txt](https://github.com/OpenMathLib/OpenBLAS/blob/develop/TargetList.txt) - -*** -anything below this line should be irrelevant nowadays unless you need to perform software archeology -*** -#### Building OpenBLAS with very old gcc-based versions of the NDK, without Fortran - -The prebuilt Android NDK toolchains do not include Fortran, hence parts like LAPACK cannot be built. You can still build OpenBLAS without it. For instructions on how to build OpenBLAS with Fortran, see the [next section](#building-openblas-with-fortran). - -To use easily the prebuilt toolchains, follow [building a standalone toolchain](http://developer.android.com/ndk/guides/standalone_toolchain.html) for your desired architecture. -This would be `arm-linux-androideabi-gcc-4.9` for ARMV7 and `aarch64-linux-android-gcc-4.9` for ARMV8. - -You can build OpenBLAS (0.2.19 and earlier) with: -``` -## Add the toolchain to your path -export PATH=/path/to/standalone-toolchain/bin:$PATH - -## Build without Fortran for ARMV7 -make TARGET=ARMV7 HOSTCC=gcc CC=arm-linux-androideabi-gcc NOFORTRAN=1 libs -## Build without Fortran for ARMV8 -make TARGET=ARMV8 BINARY=64 HOSTCC=gcc CC=aarch64-linux-android-gcc NOFORTRAN=1 libs -``` - -Since we are cross-compiling, we make the `libs` recipe, not `all`. Otherwise you will get errors when trying to link/run tests as versions up to and including 0.2.19 cannot build a shared library for Android. - -From 0.2.20 on, you should leave off the "libs" to get a full build, and you may want to use the softfp ABI instead of the deprecated hardfp one on ARMV7 so you would use -``` -## Add the toolchain to your path -export PATH=/path/to/standalone-toolchain/bin:$PATH - -## Build without Fortran for ARMV7 -make TARGET=ARMV7 ARM_SOFTFP_ABI=1 HOSTCC=gcc CC=arm-linux-androideabi-gcc NOFORTRAN=1 -## Build without Fortran for ARMV8 -make TARGET=ARMV8 BINARY=64 HOSTCC=gcc CC=aarch64-linux-android-gcc NOFORTRAN=1 -``` - -If you get an error about stdio.h not being found, you need to specify your sysroot in the CFLAGS argument to `make` like -```CFLAGS=--sysroot=$NDK/platforms/android-16/arch-arm``` -When you are done, install OpenBLAS into the desired directory. Be sure to also use all command line options -here that you specified for building, otherwise errors may occur as it tries to install things you did not build: -``` -make PREFIX=/path/to/install-dir TARGET=... install -``` - -#### Building OpenBLAS with Fortran - -Instructions on how to build the GNU toolchains with Fortran can be found [here](https://github.com/buffer51/android-gfortran). The [Releases section](https://github.com/buffer51/android-gfortran/releases) provides prebuilt versions, use the standalone one. - -You can build OpenBLAS with: -``` -## Add the toolchain to your path -export PATH=/path/to/standalone-toolchain-with-fortran/bin:$PATH - -## Build with Fortran for ARMV7 -make TARGET=ARMV7 HOSTCC=gcc CC=arm-linux-androideabi-gcc FC=arm-linux-androideabi-gfortran libs -## Build with LAPACK for ARMV8 -make TARGET=ARMV8 BINARY=64 HOSTCC=gcc CC=aarch64-linux-android-gcc FC=aarch64-linux-android-gfortran libs -``` - -As mentioned above you can leave off the `libs` argument here when building 0.2.20 and later, and you may want to add ARM_SOFTFP_ABI=1 when building for ARMV7. - -#### Linking OpenBLAS (0.2.19 and earlier) for ARMV7 - -If you are using `ndk-build`, you need to set the ABI to hard floating points in your Application.mk: -``` -APP_ABI := armeabi-v7a-hard -``` - -This will set the appropriate flags for you. If you are not using `ndk-build`, you will want to add the following flags: -``` -TARGET_CFLAGS += -mhard-float -D_NDK_MATH_NO_SOFTFP=1 -TARGET_LDFLAGS += -Wl,--no-warn-mismatch -lm_hard -``` + sudo make install + ``` + You can find full list of target architectures in [TargetList.txt](https://github.com/OpenMathLib/OpenBLAS/blob/develop/TargetList.txt) -From 0.2.20 on, it is also possible to build for the softfp ABI by specifying ARM_SOFTFP_ABI=1 during the build. -In that case, also make sure that all your dependencies are compiled with -mfloat-abi=softfp as well, as mixing -"hard" and "soft" floating point ABIs in a program will make it crash. ### iPhone/iOS -As none of the current developers uses iOS, the following instructions are what was found to work in our Azure CI setup, but as far as we know this builds a fully working OpenBLAS for this platform. +As none of the current developers uses iOS, the following instructions are what +was found to work in our Azure CI setup, but as far as we know this builds a +fully working OpenBLAS for this platform. Go to the directory where you unpacked OpenBLAS,and enter the following commands: -``` - CC=/Applications/Xcode_12.4.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang +```bash +CC=/Applications/Xcode_12.4.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang CFLAGS= -O2 -Wno-macro-redefined -isysroot /Applications/Xcode_12.4.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS14.4.sdk -arch arm64 -miphoneos-version-min=10.0 make TARGET=ARMV8 DYNAMIC_ARCH=1 NUM_THREADS=32 HOSTCC=clang NOFORTRAN=1 ``` -Adjust MIN_IOS_VERSION as necessary for your installation, e.g. change the version number +Adjust `MIN_IOS_VERSION` as necessary for your installation. E.g., change the version number to the minimum iOS version you want to target and execute this file to build the library. + ### MIPS -For mips targets you will need latest toolchains -P5600 - MTI GNU/Linux Toolchain -I6400, P6600 - IMG GNU/Linux Toolchain +For MIPS targets you will need latest toolchains: -The download link is below -(http://codescape-mips-sdk.imgtec.com/components/toolchain/2016.05-03/downloads.html) +- P5600 - MTI GNU/Linux Toolchain +- I6400, P6600 - IMG GNU/Linux Toolchain -You can use following commandlines for builds +You can use following commandlines for builds: +```bash +IMG_TOOLCHAIN_DIR={full IMG GNU/Linux Toolchain path including "bin" directory -- for example, /opt/linux_toolchain/bin} +IMG_GCC_PREFIX=mips-img-linux-gnu +IMG_TOOLCHAIN=${IMG_TOOLCHAIN_DIR}/${IMG_GCC_PREFIX} - IMG_TOOLCHAIN_DIR={full IMG GNU/Linux Toolchain path including "bin" directory -- for example, /opt/linux_toolchain/bin} - IMG_GCC_PREFIX=mips-img-linux-gnu - IMG_TOOLCHAIN=${IMG_TOOLCHAIN_DIR}/${IMG_GCC_PREFIX} +# I6400 Build (n32): +make BINARY=32 BINARY32=1 CC=$IMG_TOOLCHAIN-gcc AR=$IMG_TOOLCHAIN-ar FC="$IMG_TOOLCHAIN-gfortran -EL -mabi=n32" RANLIB=$IMG_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS=$CFLAGS LDFLAGS=$CFLAGS TARGET=I6400 - I6400 Build (n32): - make BINARY=32 BINARY32=1 CC=$IMG_TOOLCHAIN-gcc AR=$IMG_TOOLCHAIN-ar FC="$IMG_TOOLCHAIN-gfortran -EL -mabi=n32" RANLIB=$IMG_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS=$CFLAGS LDFLAGS=$CFLAGS TARGET=I6400 +# I6400 Build (n64): +make BINARY=64 BINARY64=1 CC=$IMG_TOOLCHAIN-gcc AR=$IMG_TOOLCHAIN-ar FC="$IMG_TOOLCHAIN-gfortran -EL" RANLIB=$IMG_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS=$CFLAGS LDFLAGS=$CFLAGS TARGET=I6400 - I6400 Build (n64): - make BINARY=64 BINARY64=1 CC=$IMG_TOOLCHAIN-gcc AR=$IMG_TOOLCHAIN-ar FC="$IMG_TOOLCHAIN-gfortran -EL" RANLIB=$IMG_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS=$CFLAGS LDFLAGS=$CFLAGS TARGET=I6400 +# P6600 Build (n32): +make BINARY=32 BINARY32=1 CC=$IMG_TOOLCHAIN-gcc AR=$IMG_TOOLCHAIN-ar FC="$IMG_TOOLCHAIN-gfortran -EL -mabi=n32" RANLIB=$IMG_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS=$CFLAGS LDFLAGS=$CFLAGS TARGET=P6600 - P6600 Build (n32): - make BINARY=32 BINARY32=1 CC=$IMG_TOOLCHAIN-gcc AR=$IMG_TOOLCHAIN-ar FC="$IMG_TOOLCHAIN-gfortran -EL -mabi=n32" RANLIB=$IMG_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS=$CFLAGS LDFLAGS=$CFLAGS TARGET=P6600 +# P6600 Build (n64): +make BINARY=64 BINARY64=1 CC=$IMG_TOOLCHAIN-gcc AR=$IMG_TOOLCHAIN-ar FC="$IMG_TOOLCHAIN-gfortran -EL" RANLIB=$IMG_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS="$CFLAGS" LDFLAGS="$CFLAGS" TARGET=P6600 - P6600 Build (n64): - make BINARY=64 BINARY64=1 CC=$IMG_TOOLCHAIN-gcc AR=$IMG_TOOLCHAIN-ar FC="$IMG_TOOLCHAIN-gfortran -EL" RANLIB=$IMG_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS="$CFLAGS" LDFLAGS="$CFLAGS" TARGET=P6600 +MTI_TOOLCHAIN_DIR={full MTI GNU/Linux Toolchain path including "bin" directory -- for example, /opt/linux_toolchain/bin} +MTI_GCC_PREFIX=mips-mti-linux-gnu +MTI_TOOLCHAIN=${IMG_TOOLCHAIN_DIR}/${IMG_GCC_PREFIX} - MTI_TOOLCHAIN_DIR={full MTI GNU/Linux Toolchain path including "bin" directory -- for example, /opt/linux_toolchain/bin} - MTI_GCC_PREFIX=mips-mti-linux-gnu - MTI_TOOLCHAIN=${IMG_TOOLCHAIN_DIR}/${IMG_GCC_PREFIX} +# P5600 Build: - P5600 Build: +make BINARY=32 BINARY32=1 CC=$MTI_TOOLCHAIN-gcc AR=$MTI_TOOLCHAIN-ar FC="$MTI_TOOLCHAIN-gfortran -EL" RANLIB=$MTI_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS=$CFLAGS LDFLAGS=$CFLAGS TARGET=P5600 +``` - make BINARY=32 BINARY32=1 CC=$MTI_TOOLCHAIN-gcc AR=$MTI_TOOLCHAIN-ar FC="$MTI_TOOLCHAIN-gfortran -EL" RANLIB=$MTI_TOOLCHAIN-ranlib HOSTCC=gcc CFLAGS="-EL" FFLAGS=$CFLAGS LDFLAGS=$CFLAGS TARGET=P5600 ### FreeBSD You will need to install the following tools from the FreeBSD ports tree: -* lang/gcc [1] + +* lang/gcc * lang/perl5.12 * ftp/curl * devel/gmake * devel/patch To compile run the command: - - $ gmake CC=gcc46 FC=gfortran46 - -Note that you need to build with GNU make and manually specify the compiler, otherwhise gcc 4.2 from the base system would be used. - -[1]: [Removal of Fortran from the FreeBSD base system](http://www.bsdunix.ch/serendipity/index.php?/archives/345-Removal-of-Fortran-from-the-FreeBSD-base-system.html) - - -``` -pkg install openblas +```bash +$ gmake CC=gcc FC=gfortran ``` -see ### Cortex-M -Cortex-M is a widely used microcontroller that is present in a variety of industrial and consumer electronics. -A common variant of the Cortex-M is the STM32F4xx series. Here, we will give instructions for building for -the STM32F4xx. +Cortex-M is a widely used microcontroller that is present in a variety of +industrial and consumer electronics. A common variant of the Cortex-M is the +`STM32F4xx` series. Here, we will give instructions for building for that +series. -First, install the embedded arm gcc compiler from the arm website. Then, create the following toolchain file and build as follows. +First, install the embedded Arm GCC compiler from the Arm website. Then, create +the following `toolchain.cmake` file: ```cmake -# cmake .. -G Ninja -DCMAKE_C_COMPILER=arm-none-eabi-gcc -DCMAKE_TOOLCHAIN_FILE:PATH="toolchain.cmake" -DNOFORTRAN=1 -DTARGET=ARMV5 -DEMBEDDED=1 - set(CMAKE_SYSTEM_NAME Generic) set(CMAKE_SYSTEM_PROCESSOR arm) @@ -810,14 +758,20 @@ set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) ``` -In your embedded application, the following functions need to be provided for OpenBLAS to work correctly: +Then build OpenBLAS with: +```bash +$ cmake .. -G Ninja -DCMAKE_C_COMPILER=arm-none-eabi-gcc -DCMAKE_TOOLCHAIN_FILE:PATH="toolchain.cmake" -DNOFORTRAN=1 -DTARGET=ARMV5 -DEMBEDDED=1 +``` +In your embedded application, the following functions need to be provided for OpenBLAS to work correctly: ```C void free(void* ptr); void* malloc(size_t size); ``` !!! note - If you are developing for an embedded platform, it is your responsibility to make sure that the device has sufficient memory for malloc calls. [Libmemory][2] provides one implementation of malloc for embedded platforms. -[2]: https://github.com/embeddedartistry/libmemory + If you are developing for an embedded platform, it is your responsibility + to make sure that the device has sufficient memory for `malloc` calls. + [Libmemory](https://github.com/embeddedartistry/libmemory) + provides one implementation of `malloc` for embedded platforms. From a5c04e326a604faf4e52bedac1c661199f993153 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 4 Jul 2024 22:28:01 +0200 Subject: [PATCH 186/267] Update scal.c --- kernel/arm/scal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/arm/scal.c b/kernel/arm/scal.c index 377a67f51b..4455d7643b 100644 --- a/kernel/arm/scal.c +++ b/kernel/arm/scal.c @@ -48,7 +48,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS { if ( da == 0.0 ) - if (!isnan(x[i] && !isinf(x[i])) { + if (!isnan(x[i]) && !isinf(x[i])) { x[i]=0.0; } else { x[i]=NAN; From cb154832f87d76d6e8b66961e91679948c12b9d7 Mon Sep 17 00:00:00 2001 From: Chip Kerchner Date: Tue, 9 Jul 2024 13:10:03 -0500 Subject: [PATCH 187/267] Vectorize SBGEMM incopy - 4x faster. --- kernel/power/sbgemm_ncopy_16_power10.c | 82 +++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/kernel/power/sbgemm_ncopy_16_power10.c b/kernel/power/sbgemm_ncopy_16_power10.c index c6b633011c..595edfdad4 100644 --- a/kernel/power/sbgemm_ncopy_16_power10.c +++ b/kernel/power/sbgemm_ncopy_16_power10.c @@ -37,8 +37,11 @@ /*********************************************************************/ #include +#include #include "common.h" +typedef uint32_t vec_bf16x2 __attribute__ ((vector_size (16))); + int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){ BLASLONG i, j; @@ -82,7 +85,84 @@ int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){ aoffset16 = aoffset15 + lda; aoffset += 16 * lda; - i = (m >> 1); + i = (m >> 3); + if (i > 0) { + do { + vec_bf16x2 vtemp01 = *(vec_bf16x2 *)(aoffset1); + vec_bf16x2 vtemp02 = *(vec_bf16x2 *)(aoffset2); + vec_bf16x2 vtemp03 = *(vec_bf16x2 *)(aoffset3); + vec_bf16x2 vtemp04 = *(vec_bf16x2 *)(aoffset4); + vec_bf16x2 vtemp05 = *(vec_bf16x2 *)(aoffset5); + vec_bf16x2 vtemp06 = *(vec_bf16x2 *)(aoffset6); + vec_bf16x2 vtemp07 = *(vec_bf16x2 *)(aoffset7); + vec_bf16x2 vtemp08 = *(vec_bf16x2 *)(aoffset8); + vec_bf16x2 vtemp09 = *(vec_bf16x2 *)(aoffset9); + vec_bf16x2 vtemp10 = *(vec_bf16x2 *)(aoffset10); + vec_bf16x2 vtemp11 = *(vec_bf16x2 *)(aoffset11); + vec_bf16x2 vtemp12 = *(vec_bf16x2 *)(aoffset12); + vec_bf16x2 vtemp13 = *(vec_bf16x2 *)(aoffset13); + vec_bf16x2 vtemp14 = *(vec_bf16x2 *)(aoffset14); + vec_bf16x2 vtemp15 = *(vec_bf16x2 *)(aoffset15); + vec_bf16x2 vtemp16 = *(vec_bf16x2 *)(aoffset16); + + vec_bf16x2 vtemp17 = vec_mergeh(vtemp01, vtemp03); + vec_bf16x2 vtemp18 = vec_mergel(vtemp01, vtemp03); + vec_bf16x2 vtemp19 = vec_mergeh(vtemp02, vtemp04); + vec_bf16x2 vtemp20 = vec_mergel(vtemp02, vtemp04); + vec_bf16x2 vtemp21 = vec_mergeh(vtemp05, vtemp07); + vec_bf16x2 vtemp22 = vec_mergel(vtemp05, vtemp07); + vec_bf16x2 vtemp23 = vec_mergeh(vtemp06, vtemp08); + vec_bf16x2 vtemp24 = vec_mergel(vtemp06, vtemp08); + vec_bf16x2 vtemp25 = vec_mergeh(vtemp09, vtemp11); + vec_bf16x2 vtemp26 = vec_mergel(vtemp09, vtemp11); + vec_bf16x2 vtemp27 = vec_mergeh(vtemp10, vtemp12); + vec_bf16x2 vtemp28 = vec_mergel(vtemp10, vtemp12); + vec_bf16x2 vtemp29 = vec_mergeh(vtemp13, vtemp15); + vec_bf16x2 vtemp30 = vec_mergel(vtemp13, vtemp15); + vec_bf16x2 vtemp31 = vec_mergeh(vtemp14, vtemp16); + vec_bf16x2 vtemp32 = vec_mergel(vtemp14, vtemp16); + + *(vec_bf16x2 *)(boffset + 0) = vec_mergeh(vtemp17, vtemp19); + *(vec_bf16x2 *)(boffset + 8) = vec_mergeh(vtemp21, vtemp23); + *(vec_bf16x2 *)(boffset + 16) = vec_mergeh(vtemp25, vtemp27); + *(vec_bf16x2 *)(boffset + 24) = vec_mergeh(vtemp29, vtemp31); + *(vec_bf16x2 *)(boffset + 32) = vec_mergel(vtemp17, vtemp19); + *(vec_bf16x2 *)(boffset + 40) = vec_mergel(vtemp21, vtemp23); + *(vec_bf16x2 *)(boffset + 48) = vec_mergel(vtemp25, vtemp27); + *(vec_bf16x2 *)(boffset + 56) = vec_mergel(vtemp29, vtemp31); + *(vec_bf16x2 *)(boffset + 64) = vec_mergeh(vtemp18, vtemp20); + *(vec_bf16x2 *)(boffset + 72) = vec_mergeh(vtemp22, vtemp24); + *(vec_bf16x2 *)(boffset + 80) = vec_mergeh(vtemp26, vtemp28); + *(vec_bf16x2 *)(boffset + 88) = vec_mergeh(vtemp30, vtemp32); + *(vec_bf16x2 *)(boffset + 96) = vec_mergel(vtemp18, vtemp20); + *(vec_bf16x2 *)(boffset + 104) = vec_mergel(vtemp22, vtemp24); + *(vec_bf16x2 *)(boffset + 112) = vec_mergel(vtemp26, vtemp28); + *(vec_bf16x2 *)(boffset + 120) = vec_mergel(vtemp30, vtemp32); + + aoffset1 += 8; + aoffset2 += 8; + aoffset3 += 8; + aoffset4 += 8; + aoffset5 += 8; + aoffset6 += 8; + aoffset7 += 8; + aoffset8 += 8; + aoffset9 += 8; + aoffset10 += 8; + aoffset11 += 8; + aoffset12 += 8; + aoffset13 += 8; + aoffset14 += 8; + aoffset15 += 8; + aoffset16 += 8; + + boffset += 128; + + i--; + } while (i > 0); + } + + i = (m & 7) >> 1; if (i > 0){ do{ ctemp01 = *(aoffset1 + 0); From e706bc1ec01fcf4c69c916ce6eeaa405186f8a8c Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 9 Jul 2024 20:22:56 +0200 Subject: [PATCH 188/267] Fix core assignment for Intel family 15 --- cpuid_x86.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpuid_x86.c b/cpuid_x86.c index 4fb51f8b3c..f77cca1d87 100644 --- a/cpuid_x86.c +++ b/cpuid_x86.c @@ -2392,10 +2392,10 @@ int get_coretype(void){ else return CORE_NEHALEM; } - case 15: - if (model <= 0x2) return CORE_NORTHWOOD; - else return CORE_PRESCOTT; } + case 15: + if (model <= 0x2) return CORE_NORTHWOOD; + else return CORE_PRESCOTT; } } From f708944fea4b10fc6ba3a9b607cba4eca30939be Mon Sep 17 00:00:00 2001 From: Chip Kerchner Date: Wed, 10 Jul 2024 13:07:48 -0500 Subject: [PATCH 189/267] Add all 4 variations of the SBGEMM to compare_sgemm_sbgemm --- test/compare_sgemm_sbgemm.c | 92 +++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 34 deletions(-) diff --git a/test/compare_sgemm_sbgemm.c b/test/compare_sgemm_sbgemm.c index bc74233abc..de589458bc 100644 --- a/test/compare_sgemm_sbgemm.c +++ b/test/compare_sgemm_sbgemm.c @@ -86,14 +86,26 @@ main (int argc, char *argv[]) { blasint m, n, k; int i, j, l; - blasint x; + blasint x, y; int ret = 0; int loop = 100; char transA = 'N', transB = 'N'; float alpha = 1.0, beta = 0.0; for (x = 0; x <= loop; x++) + { + for (y = 0; y < 4; y++) { + if ((y == 0) || (y == 2)) { + transA = 'N'; + } else { + transA = 'T'; + } + if ((y == 0) || (y == 1)) { + transB = 'N'; + } else { + transB = 'T'; + } m = k = n = x; float A[m * k]; float B[k * n]; @@ -104,43 +116,55 @@ main (int argc, char *argv[]) blasint one=1; for (j = 0; j < m; j++) - { - for (i = 0; i < m; i++) - { - A[j * k + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; - B[j * k + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; - C[j * k + i] = 0; - sbstobf16_(&one, &A[j*k+i], &one, &atmp, &one); - sbstobf16_(&one, &B[j*k+i], &one, &btmp, &one); - AA[j * k + i].v = atmp; - BB[j * k + i].v = btmp; - CC[j * k + i] = 0; - DD[j * k + i] = 0; - } - } + { + for (i = 0; i < m; i++) + { + A[j * k + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; + B[j * k + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; + C[j * k + i] = 0; + sbstobf16_(&one, &A[j*k+i], &one, &atmp, &one); + sbstobf16_(&one, &B[j*k+i], &one, &btmp, &one); + AA[j * k + i].v = atmp; + BB[j * k + i].v = btmp; + CC[j * k + i] = 0; + DD[j * k + i] = 0; + } + } SGEMM (&transA, &transB, &m, &n, &k, &alpha, A, - &m, B, &k, &beta, C, &m); + &m, B, &k, &beta, C, &m); SBGEMM (&transA, &transB, &m, &n, &k, &alpha, (bfloat16*) AA, - &m, (bfloat16*)BB, &k, &beta, CC, &m); + &m, (bfloat16*)BB, &k, &beta, CC, &m); + for (i = 0; i < n; i++) + for (j = 0; j < m; j++) + if (fabs (CC[i * m + j] - C[i * m + j]) > 1.0) + ret++; for (i = 0; i < n; i++) - for (j = 0; j < m; j++) - if (fabs (CC[i * m + j] - C[i * m + j]) > 1.0) - ret++; - if (transA == 'N' && transB == 'N') - { - for (i = 0; i < n; i++) - for (j = 0; j < m; j++) - for (l = 0; l < k; l++) - { - DD[i * m + j] += - float16to32 (AA[l * m + j]) * float16to32 (BB[l + k * i]); - } - for (i = 0; i < n; i++) - for (j = 0; j < m; j++) - if (CC[i * m + j] != DD[i * m + j]) - ret++; - } + for (j = 0; j < m; j++) + for (l = 0; l < k; l++) + if (transA == 'N' && transB == 'N') + { + DD[i * m + j] += + float16to32 (AA[l * m + j]) * float16to32 (BB[l + k * i]); + } else if (transA == 'T' && transB == 'N') + { + DD[i * m + j] += + float16to32 (AA[k * j + l]) * float16to32 (BB[l + k * i]); + } else if (transA == 'N' && transB == 'T') + { + DD[i * m + j] += + float16to32 (AA[l * m + j]) * float16to32 (BB[i + l * n]); + } else if (transA == 'T' && transB == 'T') + { + DD[i * m + j] += + float16to32 (AA[k * j + l]) * float16to32 (BB[i + l * n]); + } + for (i = 0; i < n; i++) + for (j = 0; j < m; j++) + if (CC[i * m + j] != DD[i * m + j]) + ret++; } + } + if (ret != 0) fprintf (stderr, "FATAL ERROR SBGEMM - Return code: %d\n", ret); return ret; From 4c1209077645e2869357d873bb434552958f355f Mon Sep 17 00:00:00 2001 From: Piotr Kubaj Date: Wed, 10 Jul 2024 22:21:48 +0000 Subject: [PATCH 190/267] Fix build on FreeBSD/powerpc64* --- param.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/param.h b/param.h index 2ebe824db5..2618e1f609 100644 --- a/param.h +++ b/param.h @@ -2553,7 +2553,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -#if defined(POWER9) && defined(OS_LINUX) +#if defined(POWER9) && (defined(OS_LINUX) || defined(OS_FREEBSD)) #define SNUMOPT 16 #define DNUMOPT 8 From 475bd2452b9fb8ab5bc827823e0e7aee0b50fbb6 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 11 Jul 2024 20:13:57 +0200 Subject: [PATCH 191/267] Suffix BUFFERSIZEs as UL to prevent int overflow in computations --- common_power.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/common_power.h b/common_power.h index 3fe776f235..6b13f06b10 100644 --- a/common_power.h +++ b/common_power.h @@ -841,17 +841,17 @@ Lmcount$lazy_ptr: #endif #if defined(PPC440) -#define BUFFER_SIZE ( 2 << 20) +#define BUFFER_SIZE ( 2UL << 20) #elif defined(PPC440FP2) -#define BUFFER_SIZE ( 16 << 20) +#define BUFFER_SIZE ( 16UL << 20) #elif defined(POWER6) || defined(POWER8) || defined(POWER9) || defined(POWER10) -#define BUFFER_SIZE ( 64 << 22) +#define BUFFER_SIZE ( 64UL << 22) #else -#define BUFFER_SIZE ( 16 << 20) +#define BUFFER_SIZE ( 16UL << 20) #endif #ifdef DYNAMIC_ARCH #undef BUFFER_SIZE -#define BUFFER_SIZE (64 << 22) +#define BUFFER_SIZE (64UL << 22) #endif #ifndef PAGESIZE From f3cebb3ca3c0b3c75c287862462bc996f88095a2 Mon Sep 17 00:00:00 2001 From: gxw Date: Wed, 10 Jul 2024 15:11:12 +0800 Subject: [PATCH 192/267] x86: Fixed numpy CI failure when the target is ZEN. --- interface/scal.c | 4 +- kernel/x86_64/dscal.c | 306 ++++++++++++++++++++---------------------- kernel/x86_64/sscal.c | 273 ++++++++++++++++++------------------- 3 files changed, 283 insertions(+), 300 deletions(-) diff --git a/interface/scal.c b/interface/scal.c index 0a7fee640c..c6638a62d5 100644 --- a/interface/scal.c +++ b/interface/scal.c @@ -85,7 +85,7 @@ void CNAME(blasint n, FLOAT alpha, FLOAT *x, blasint incx){ if (nthreads == 1) { #endif - SCAL_K(n, 0, 0, alpha, x, incx, NULL, 0, NULL, 0); + SCAL_K(n, 0, 0, alpha, x, incx, NULL, 0, NULL, 1); #ifdef SMP } else { @@ -102,7 +102,7 @@ void CNAME(blasint n, FLOAT alpha, FLOAT *x, blasint incx){ #else &alpha, #endif - x, incx, NULL, 0, NULL, 0, (int (*)(void))SCAL_K, nthreads); + x, incx, NULL, 0, NULL, 1, (int (*)(void))SCAL_K, nthreads); } #endif diff --git a/kernel/x86_64/dscal.c b/kernel/x86_64/dscal.c index e7182c5ce8..641f86f903 100644 --- a/kernel/x86_64/dscal.c +++ b/kernel/x86_64/dscal.c @@ -43,21 +43,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. static void dscal_kernel_8( BLASLONG n, FLOAT *da , FLOAT *x ) { - BLASLONG i; - FLOAT alpha = *da; - - for( i=0; i 0 ) - { - dscal_kernel_inc_8(n1, &da, x, inc_x); - i = n1 * inc_x; - j = n1; - } - - while(j < n) - { - - x[i] *= da; - i += inc_x ; - j++; - - } - - } - - return(0); - } - - BLASLONG n1 = n & -8; - if ( n1 > 0 ) - { -// if ( da == 0.0 ) -// dscal_kernel_8_zero(n1 , &da , x); -// else - dscal_kernel_8(n1 , &da , x); - } - - if ( da == 0.0 ) - { - for ( i=n1 ; i 0 ) + { + dscal_kernel_inc_8(n1, &da, x, inc_x); + i = n1 * inc_x; + j = n1; + } + while(j < n) + { + x[i] *= da; + i += inc_x ; + j++; + } + } + else + { + BLASLONG n1 = n & -8; + if ( n1 > 0) + dscal_kernel_8(n1 , &da , x); + for ( i = n1 ; i < n; i++ ) + x[i] *= da; + } + } + else + { + if ( inc_x != 1 ) + { + if( da == 0.0) + { + BLASLONG n1 = n & -2; + while(j < n1) + { + x[i] = 0.0; + x[i+inc_x] = 0.0; + i += 2 * inc_x ; + j += 2; + } + while(j < n) + { + x[i] = 0.0; + i += inc_x ; + j++; + } + } + else + { + BLASLONG n1 = n & -8; + if ( n1 > 0 ) + { + dscal_kernel_inc_8(n1, &da, x, inc_x); + i = n1 * inc_x; + j = n1; + } + while(j < n) + { + x[i] *= da; + i += inc_x ; + j++; + } + } + } + else + { + if ( da == 0.0 ) + { + BLASLONG n1 = n & -8; + if ( n1 > 0) + dscal_kernel_8_zero(n1, &da, x); + for ( i = n1 ; i < n; i++ ) + x[i] = 0.0; + } + else + { + BLASLONG n1 = n & -8; + if ( n1 > 0) + dscal_kernel_8(n1 , &da , x); + for ( i = n1 ; i < n; i++ ) + x[i] *= da; + } + } + } } - - diff --git a/kernel/x86_64/sscal.c b/kernel/x86_64/sscal.c index a85d20564f..6e54f8893c 100644 --- a/kernel/x86_64/sscal.c +++ b/kernel/x86_64/sscal.c @@ -39,21 +39,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. static void sscal_kernel_16( BLASLONG n, FLOAT *da , FLOAT *x ) { - BLASLONG i; - FLOAT alpha = *da; - - for( i=0; i 0 ) - { - sscal_kernel_inc_8(n1, &da, x, inc_x); - i = n1 * inc_x; - j = n1; - } -#endif - while(j < n) - { - x[i] *= da; - i += inc_x ; - j++; - - } - - } - return(0); - } - - BLASLONG n1 = n & -16; - if ( n1 > 0 ) - { - //if ( da == 0.0 ) - // sscal_kernel_16_zero(n1 , &da , x); - //else - sscal_kernel_16(n1 , &da , x); - } - - if ( da == 0.0 ) - { - for ( i=n1 ; i 0 ) + { + sscal_kernel_inc_8(n1, &da, x, inc_x); + i = n1 * inc_x; + j = n1; + } + while(j < n) + { + x[i] *= da; + i += inc_x ; + j++; + } + } + else + { + BLASLONG n1 = n & -16; + if ( n1 > 0) + sscal_kernel_16(n1 , &da , x); + for ( i = n1 ; i < n; i++ ) + x[i] *= da; + } + } + else + { + if ( inc_x != 1 ) + { + if( da == 0.0) + { + BLASLONG n1 = n & -2; + while(j < n1) + { + x[i] = 0.0; + x[i+inc_x] = 0.0; + i += 2 * inc_x ; + j += 2; + } + while(j < n) + { + x[i] = 0.0; + i += inc_x ; + j++; + } + } + else + { + BLASLONG n1 = n & -8; + if ( n1 > 0 ) + { + sscal_kernel_inc_8(n1, &da, x, inc_x); + i = n1 * inc_x; + j = n1; + } + while(j < n) + { + x[i] *= da; + i += inc_x ; + j++; + } + } + } + else + { + if ( da == 0.0 ) + { + BLASLONG n1 = n & -16; + if ( n1 > 0) + sscal_kernel_16_zero(n1, &da, x); + for ( i = n1 ; i < n; i++ ) + x[i] = 0.0; + } + else + { + BLASLONG n1 = n & -16; + if ( n1 > 0) + sscal_kernel_16(n1 , &da , x); + for ( i = n1 ; i < n; i++ ) + x[i] *= da; + } + } + } } - - From 3f39c8f94f5d61f69a7b1c578e7fc90c4c95d6e2 Mon Sep 17 00:00:00 2001 From: gxw Date: Fri, 12 Jul 2024 16:56:35 +0800 Subject: [PATCH 193/267] LoongArch: Fixed numpy CI failure --- kernel/loongarch64/scal.S | 77 +++++++++++++++++++++++++++++++ kernel/loongarch64/scal_lasx.S | 79 ++++++++++++++++++++++++++++++-- kernel/loongarch64/scal_lsx.S | 82 +++++++++++++++++++++++++++++++++- 3 files changed, 233 insertions(+), 5 deletions(-) diff --git a/kernel/loongarch64/scal.S b/kernel/loongarch64/scal.S index 8de710f413..431a44c1cd 100644 --- a/kernel/loongarch64/scal.S +++ b/kernel/loongarch64/scal.S @@ -53,9 +53,86 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. PROLOGUE li.d TEMP, SIZE + ld.d XX, $sp, 0 // Load dummy2 + slli.d XX, XX, BASE_SHIFT MTC a1, $r0 slli.d INCX, INCX, BASE_SHIFT bge $r0, N, .L999 + CMPEQ $fcc0, ALPHA, a1 + bceqz $fcc0, .L50 + beq XX, TEMP, .L50 // if dummp2 == 1, do not directly copy 0 + srai.d I, N, 3 + bne INCX, TEMP, .L20 + bge $r0, I, .L15 + .align 3 + +.L12: + ST a1, X, 0 * SIZE + ST a1, X, 1 * SIZE + ST a1, X, 2 * SIZE + ST a1, X, 3 * SIZE + ST a1, X, 4 * SIZE + ST a1, X, 5 * SIZE + ST a1, X, 6 * SIZE + ST a1, X, 7 * SIZE + addi.w I, I, -1 + addi.d X, X, 8 * SIZE + blt $r0, I, .L12 + .align 3 + +.L15: + andi I, N, 7 + bge $r0, I, .L999 + .align 3 +.L16: + ST a1, X, 0 * SIZE + addi.d I, I, -1 + addi.d X, X, SIZE + blt $r0, I, .L16 + move $r4, $r17 + fmov.d $f0, $f22 + jirl $r0, $r1, 0x0 + .align 3 + +.L20: + srai.d I, N, 3 + bge $r0, I, .L25 + .align 3 + +.L22: + ST a1, X, 0 * SIZE + add.d X, X, INCX + ST a1, X, 0 * SIZE + add.d X, X, INCX + ST a1, X, 0 * SIZE + add.d X, X, INCX + ST a1, X, 0 * SIZE + add.d X, X, INCX + ST a1, X, 0 * SIZE + add.d X, X, INCX + ST a1, X, 0 * SIZE + add.d X, X, INCX + ST a1, X, 0 * SIZE + add.d X, X, INCX + ST a1, X, 0 * SIZE + addi.d I, I, -1 + add.d X, X, INCX + blt $r0, I, .L22 + .align 3 + +.L25: + andi I, N, 7 + bge $r0, I, .L999 + .align 3 +.L26: + addi.d I, I, -1 + ST a1, X, 0 * SIZE + add.d X, X, INCX + blt $r0, I, .L26 + move $r4, $r17 + fmov.d $f0, $f22 + jirl $r0, $r1, 0x0 + .align 3 .L50: srai.d I, N, 3 diff --git a/kernel/loongarch64/scal_lasx.S b/kernel/loongarch64/scal_lasx.S index b4585c1b96..dd69636e6f 100644 --- a/kernel/loongarch64/scal_lasx.S +++ b/kernel/loongarch64/scal_lasx.S @@ -52,17 +52,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. bge $r0, N, .L999 bge $r0, INCX, .L999 li.d TEMP, 1 + ld.d t1, $sp, 0 // Load dummp2 movgr2fr.d a1, $r0 FFINT a1, a1 movgr2fr.d a2, TEMP FFINT a2, a2 slli.d TEMP, TEMP, BASE_SHIFT slli.d INCX, INCX, BASE_SHIFT + slli.d t1, t1, BASE_SHIFT + CMPEQ $fcc0, ALPHA, a1 + bcnez $fcc0, .L20 //ALPHA==0 CMPEQ $fcc0, ALPHA, a2 bcnez $fcc0, .L999 //ALPHA==1 return - +.L1: srai.d I, N, 3 - beq INCX, TEMP, .L30 //ALPHA!=1 and INCX==1 + beq INCX, TEMP, .L30 //ALPHA !=0|1 and INCX==1 MTG TEMP, ALPHA #ifdef DOUBLE xvreplgr2vr.d VALPHA, TEMP @@ -72,7 +76,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. move XX, X .align 3 -.L10: //ALPHA!=1 and INCX!=1 +.L10: //ALPHA !=0|1 and INCX!=1 bge $r0, I, .L32 .align 3 .L11: @@ -165,6 +169,75 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. blt $r0, I, .L11 b .L32 .align 3 + +.L20: + beq t1, TEMP, .L1 // if dummp2 == 1, do not directly copy 0 + srai.d I, N, 3 + beq INCX, TEMP, .L24 + bge $r0, I, .L22 + .align 3 + +.L21: + ST a1, X, 0 + add.d X, X, INCX + ST a1, X, 0 + add.d X, X, INCX + ST a1, X, 0 + add.d X, X, INCX + ST a1, X, 0 + add.d X, X, INCX + ST a1, X, 0 + add.d X, X, INCX + ST a1, X, 0 + add.d X, X, INCX + ST a1, X, 0 + add.d X, X, INCX + ST a1, X, 0 + add.d X, X, INCX + addi.d I, I, -1 + blt $r0, I, .L21 + .align 3 + +.L22: + andi I, N, 7 + bge $r0, I, .L999 + .align 3 + +.L23: + ST a1, X, 0 * SIZE + addi.d I, I, -1 + add.d X, X, INCX + blt $r0, I, .L23 + jirl $r0, $r1, 0 + .align 3 + +.L24: + bge $r0, I, .L26 /*N<8 INCX==1*/ + .align 3 +.L25: + xvxor.v VX0, VX0, VX0 + xvst VX0, X, 0 * SIZE +#ifdef DOUBLE + xvst VX0, X, 4 * SIZE +#endif + addi.d I, I, -1 + addi.d X, X, 8 * SIZE + blt $r0, I, .L25 + .align 3 + +.L26: + andi I, N, 7 + bge $r0, I, .L999 + .align 3 + +.L27: + ST a1, X, 0 * SIZE + addi.d I, I, -1 + addi.d X, X, SIZE + blt $r0, I, .L27 + jirl $r0, $r1, 0 + .align 3 + .L30: bge $r0, I, .L32/*N<8 INCX==1*/ MTG TEMP, ALPHA diff --git a/kernel/loongarch64/scal_lsx.S b/kernel/loongarch64/scal_lsx.S index a27e050ed8..57dc5d0d31 100644 --- a/kernel/loongarch64/scal_lsx.S +++ b/kernel/loongarch64/scal_lsx.S @@ -51,6 +51,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. bge $r0, N, .L999 bge $r0, INCX, .L999 + ld.d t1, $sp, 0 // Load dummy2 li.d TEMP, 1 movgr2fr.d a1, $r0 FFINT a1, a1 @@ -58,10 +59,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. FFINT a2, a2 slli.d TEMP, TEMP, BASE_SHIFT slli.d INCX, INCX, BASE_SHIFT + slli.d t1, t1, BASE_SHIFT + CMPEQ $fcc0, ALPHA, a1 + bcnez $fcc0, .L20 //ALPHA==0 CMPEQ $fcc0, ALPHA, a2 bcnez $fcc0, .L999 //ALPHA==1 return +.L1: srai.d I, N, 3 - beq INCX, TEMP, .L30 //ALPHA!=1 and INCX==1 + beq INCX, TEMP, .L30 //ALPHA !=0|1 and INCX==1 MTG TEMP, ALPHA #ifdef DOUBLE vreplgr2vr.d VALPHA, TEMP @@ -71,7 +76,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. move XX, X .align 3 -.L10: //ALPHA!=1 and INCX!=1 +.L10: //ALPHA !=0|1 and INCX!=1 bge $r0, I, .L32 .align 3 @@ -169,6 +174,79 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. b .L32 .align 3 +.L20: + beq t1, TEMP, .L1 // if dummp2 == 1, do not directly copy 0 + srai.d I, N, 3 + beq INCX, TEMP, .L24 + bge $r0, I, .L22 + .align 3 + +.L21: + ST a1, X, 0 + add.d X, X, INCX + ST a1, X, 0 + add.d X, X, INCX + ST a1, X, 0 + add.d X, X, INCX + ST a1, X, 0 + add.d X, X, INCX + ST a1, X, 0 + add.d X, X, INCX + ST a1, X, 0 + add.d X, X, INCX + ST a1, X, 0 + add.d X, X, INCX + ST a1, X, 0 + add.d X, X, INCX + addi.d I, I, -1 + blt $r0, I, .L21 + .align 3 + +.L22: + andi I, N, 7 + bge $r0, I, .L999 + .align 3 + +.L23: + ST a1, X, 0 * SIZE + addi.d I, I, -1 + add.d X, X, INCX + blt $r0, I, .L23 + jirl $r0, $r1, 0 + .align 3 + +.L24: + bge $r0, I, .L26 /*N<8 INCX==1*/ + .align 3 + +.L25: + vxor.v VX0, VX0, VX0 + vst VX0, X, 0 * SIZE +#ifdef DOUBLE + vst VX0, X, 2 * SIZE + vst VX0, X, 4 * SIZE + vst VX0, X, 6 * SIZE +#else + vst VX0, X, 4 * SIZE +#endif + addi.d I, I, -1 + addi.d X, X, 8 * SIZE + blt $r0, I, .L25 + .align 3 + +.L26: + andi I, N, 7 + bge $r0, I, .L999 + .align 3 + +.L27: + ST a1, X, 0 * SIZE + addi.d I, I, -1 + addi.d X, X, SIZE + blt $r0, I, .L27 + jirl $r0, $r1, 0 + .align 3 + .L30: bge $r0, I, .L32/*N<8 INCX==1*/ MTG TEMP, ALPHA From 9b3e80efe24046701a681ba53c4179c0b60ca970 Mon Sep 17 00:00:00 2001 From: gxw Date: Mon, 15 Jul 2024 16:33:09 +0800 Subject: [PATCH 194/267] utest: Add test_gemv --- utest/CMakeLists.txt | 1 + utest/Makefile | 2 +- utest/test_gemv.c | 126 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 utest/test_gemv.c diff --git a/utest/CMakeLists.txt b/utest/CMakeLists.txt index 4771d8a27b..6a61899da5 100644 --- a/utest/CMakeLists.txt +++ b/utest/CMakeLists.txt @@ -18,6 +18,7 @@ else () test_zscal.c test_amin.c test_axpby.c + test_gemv.c ) endif () diff --git a/utest/Makefile b/utest/Makefile index 36acf96cdb..b82937093f 100644 --- a/utest/Makefile +++ b/utest/Makefile @@ -14,7 +14,7 @@ UTESTEXTBIN=openblas_utest_ext include $(TOPDIR)/Makefile.system OBJS=utest_main.o test_min.o test_amax.o test_ismin.o test_rotmg.o test_axpy.o test_dotu.o test_dsdot.o test_swap.o test_rot.o test_dnrm2.o test_zscal.o \ - test_amin.o test_axpby.o + test_amin.o test_axpby.o test_gemv.o #test_rot.o test_swap.o test_axpy.o test_dotu.o test_dsdot.o test_fork.o OBJS_EXT=utest_main.o $(DIR_EXT)/xerbla.o $(DIR_EXT)/common.o OBJS_EXT+=$(DIR_EXT)/test_isamin.o $(DIR_EXT)/test_idamin.o $(DIR_EXT)/test_icamin.o $(DIR_EXT)/test_izamin.o diff --git a/utest/test_gemv.c b/utest/test_gemv.c new file mode 100644 index 0000000000..c85ef3f389 --- /dev/null +++ b/utest/test_gemv.c @@ -0,0 +1,126 @@ +#include "openblas_utest.h" +#include + +#ifndef NAN +#define NAN 0.0/0.0 +#endif +#ifndef INFINITY +#define INFINITY 1.0/0.0 +#endif + +#ifdef BUILD_SINGLE + +CTEST(sgemv, 0_nan_inf) +{ + blasint N = 17; + blasint incX = 1; + blasint incY = 1; + float alpha = 0.0; + float beta = 0.0; + char trans = 'N'; + float A[N * N]; + float X[N]; + float Y[N]; + + memset(A, 0, sizeof(A)); + memset(X, 0, sizeof(X)); + for (int i = 0; i < (N - 1); i += 2) + { + Y[i] = NAN; + Y[i + 1] = INFINITY; + } + Y[N - 1] = NAN; + BLASFUNC(sgemv)(&trans, &N, &N, &alpha, A, &N, X, &incX, &beta, Y, &incY); + for (int i = 0; i < N; i ++) + ASSERT_TRUE(Y[i] == 0.0); +} + +CTEST(sgemv, 0_nan_inf_incy_2) +{ + blasint N = 17; + blasint Ny = 33; + blasint incX = 1; + blasint incY = 2; + float alpha = 0.0; + float beta = 0.0; + char trans = 'N'; + float A[N * N]; + float X[N]; + float Y[Ny]; + float *ay = Y; + + memset(A, 0, sizeof(A)); + memset(X, 0, sizeof(X)); + memset(Y, 0, sizeof(Y)); + for (int i = 0; i < (N - 1); i += 2) + { + ay[0] = NAN; + ay += 2; + ay[0] = INFINITY; + ay += 2; + } + Y[Ny - 1] = NAN; + BLASFUNC(sgemv)(&trans, &N, &N, &alpha, A, &N, X, &incX, &beta, Y, &incY); + for (int i = 0; i < Ny; i ++) + ASSERT_TRUE(Y[i] == 0.0); +} + +#endif + +#ifdef BUILD_DOUBLE +CTEST(dgemv, 0_nan_inf) +{ + blasint N = 17; + blasint incX = 1; + blasint incY = 1; + double alpha = 0.0; + double beta = 0.0; + char trans = 'N'; + double A[N * N]; + double X[N]; + double Y[N]; + + memset(A, 0, sizeof(A)); + memset(X, 0, sizeof(X)); + for (int i = 0; i < (N - 1); i += 2) + { + Y[i] = NAN; + Y[i + 1] = INFINITY; + } + Y[N - 1] = NAN; + BLASFUNC(dgemv)(&trans, &N, &N, &alpha, A, &N, X, &incX, &beta, Y, &incY); + for (int i = 0; i < N; i ++) + ASSERT_TRUE(Y[i] == 0.0); +} + +CTEST(dgemv, 0_nan_inf_incy_2) +{ + blasint N = 17; + blasint Ny = 33; + blasint incX = 1; + blasint incY = 2; + double alpha = 0.0; + double beta = 0.0; + char trans = 'N'; + double A[N * N]; + double X[N]; + double Y[Ny]; + double *ay = Y; + + memset(A, 0, sizeof(A)); + memset(X, 0, sizeof(X)); + memset(Y, 0, sizeof(Y)); + for (int i = 0; i < (N - 1); i += 2) + { + ay[0] = NAN; + ay += 2; + ay[0] = INFINITY; + ay += 2; + } + Y[Ny - 1] = NAN; + BLASFUNC(dgemv)(&trans, &N, &N, &alpha, A, &N, X, &incX, &beta, Y, &incY); + for (int i = 0; i < Ny; i ++) + ASSERT_TRUE(Y[i] == 0.0); +} + +#endif From 3b715e6162686edd2ec1e62eb4d273c037b26730 Mon Sep 17 00:00:00 2001 From: Mark Ryan Date: Fri, 5 Jul 2024 10:39:07 +0000 Subject: [PATCH 195/267] Add autodetection for riscv64 Implement DYNAMIC_ARCH support for riscv64. Three cpu types are supported, riscv64_generic, riscv64_zvl256b, riscv64_zvl128b. The two non-generic kernels require CPU support for RVV 1.0 to function correctly. Detecting that a riscv64 device supports RVV 1.0 is a little complicated as there are some boards on the market that advertise support for V via hwcap but only support RVV 0.7.1, which is not binary compatible with RVV 1.0. The approach taken is to first try hwprobe. If hwprobe is not available, we fall back to hwcap + an additional check to distinguish between RVV 1.0 and RVV 0.7.1. Tested on a VM with VLEN=256, a CanMV K230 with VLEN=128 (with only the big core enabled), a Lichee Pi with RVV 0.7.1 and a VF2 with no vector. A compiler with RVV 1.0 support must be used to build OpenBLAS for riscv64 when DYNAMIC_ARCH=1. Signed-off-by: Mark Ryan --- .github/workflows/riscv64_vector.yml | 3 + Makefile.system | 11 ++ README.md | 2 + driver/others/Makefile | 11 ++ driver/others/detect_riscv64.c | 75 +++++++ driver/others/dynamic_riscv64.c | 269 ++++++++++++++++++++++++++ kernel/riscv64/KERNEL.RISCV64_GENERIC | 16 +- kernel/setparam-ref.c | 31 +++ lapack/laswp/riscv64/Makefile | 5 + 9 files changed, 415 insertions(+), 8 deletions(-) create mode 100644 driver/others/detect_riscv64.c create mode 100644 driver/others/dynamic_riscv64.c diff --git a/.github/workflows/riscv64_vector.yml b/.github/workflows/riscv64_vector.yml index dd6fe9ca80..9209ebb7d1 100644 --- a/.github/workflows/riscv64_vector.yml +++ b/.github/workflows/riscv64_vector.yml @@ -28,6 +28,9 @@ jobs: - target: RISCV64_ZVL256B opts: TARGET=RISCV64_ZVL256B BINARY=64 ARCH=riscv64 qemu_cpu: rv64,g=true,c=true,v=true,vext_spec=v1.0,vlen=256,elen=64 + - target: DYNAMIC_ARCH=1 + opts: TARGET=RISCV64_GENERIC BINARY=64 ARCH=riscv64 DYNAMIC_ARCH=1 + qemu_cpu: rv64,g=true,c=true,v=true,vext_spec=v1.0,vlen=256,elen=64 steps: - name: Checkout repository diff --git a/Makefile.system b/Makefile.system index 4cd4e4a1c7..847bab1798 100644 --- a/Makefile.system +++ b/Makefile.system @@ -715,6 +715,17 @@ ifeq ($(ARCH), loongarch64) DYNAMIC_CORE = LOONGSON3R5 LOONGSON2K1000 LOONGSONGENERIC endif +ifeq ($(ARCH), riscv64) +DYNAMIC_CORE = RISCV64_GENERIC +DYNAMIC_CORE += RISCV64_ZVL128B +DYNAMIC_CORE += RISCV64_ZVL256B +ifdef DYNAMIC_LIST +override DYNAMIC_CORE = RISCV64_GENERIC $(DYNAMIC_LIST) +XCCOMMON_OPT = -DDYNAMIC_LIST -DDYN_RISCV64_GENERIC +XCCOMMON_OPT += $(foreach dcore,$(DYNAMIC_LIST),-DDYN_$(dcore)) +endif +endif + ifeq ($(ARCH), zarch) DYNAMIC_CORE = ZARCH_GENERIC diff --git a/README.md b/README.md index b52e485647..169087cec1 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,8 @@ For **POWER**, the list encompasses POWER6, POWER8 and POWER9. POWER10 is additi on **ZARCH** it comprises Z13 and Z14 as well as generic zarch support. +On **riscv64**, DYNAMIC_ARCH enables support for riscv64_zvl128b and riscv64_zvl256b in addition to generic riscv64 support. A compiler that supports RVV 1.0 is required to build OpenBLAS for riscv64 when DYNAMIC_ARCH is enabled. + The `TARGET` option can be used in conjunction with `DYNAMIC_ARCH=1` to specify which cpu model should be assumed for all the common code in the library, usually you will want to set this to the oldest model you expect to encounter. Please note that it is not possible to combine support for different architectures, so no combined 32 and 64 bit or x86_64 and arm64 in the same library. diff --git a/driver/others/Makefile b/driver/others/Makefile index ff7e3e96db..719d617c45 100644 --- a/driver/others/Makefile +++ b/driver/others/Makefile @@ -30,12 +30,16 @@ else ifeq ($(ARCH),loongarch64) COMMONOBJS += dynamic_loongarch64.$(SUFFIX) else +ifeq ($(ARCH),riscv64) +COMMONOBJS += dynamic_riscv64.$(SUFFIX) detect_riscv64.$(SUFFIX) +else COMMONOBJS += dynamic.$(SUFFIX) endif endif endif endif endif +endif else COMMONOBJS += parameter.$(SUFFIX) endif @@ -106,12 +110,16 @@ else ifeq ($(ARCH),loongarch64) HPLOBJS = memory.$(SUFFIX) xerbla.$(SUFFIX) dynamic_loongarch64.$(SUFFIX) else +ifeq ($(ARCH),riscv64) +HPLOBJS = memory.$(SUFFIX) xerbla.$(SUFFIX) dynamic_riscv64.$(SUFFIX) detect_riscv64.$(SUFFIX) +else HPLOBJS = memory.$(SUFFIX) xerbla.$(SUFFIX) dynamic.$(SUFFIX) endif endif endif endif endif +endif else HPLOBJS = memory.$(SUFFIX) xerbla.$(SUFFIX) parameter.$(SUFFIX) endif @@ -209,6 +217,9 @@ addx.$(SUFFIX) : $(ARCH)/addx.c mulx.$(SUFFIX) : $(ARCH)/mulx.c $(CC) $(CFLAGS) -c -DXDOUBLE -UCOMPLEX $< -o $(@F) +detect_riscv64.$(SUFFIX): detect_riscv64.c + $(CC) $(CFLAGS) -c -march=rv64imafdcv $< -o $(@F) + xerbla.$(PSUFFIX) : xerbla.c $(CC) $(PFLAGS) -c $< -o $(@F) diff --git a/driver/others/detect_riscv64.c b/driver/others/detect_riscv64.c new file mode 100644 index 0000000000..5a5cc03916 --- /dev/null +++ b/driver/others/detect_riscv64.c @@ -0,0 +1,75 @@ +/***************************************************************************** +Copyright (c) 2024, The OpenBLAS Project +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. Neither the name of the OpenBLAS project nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************************/ + +#include + +#ifdef __riscv_v_intrinsic +#include +#endif + +unsigned detect_riscv64_get_vlenb(void) { +#ifdef __riscv_v_intrinsic + return __riscv_vlenb(); +#else + return 0; +#endif +} + +/* + * Based on the approach taken here: + * https://code.videolan.org/videolan/dav1d/-/merge_requests/1629 + * + * Only to be called after we've determined we have some sort of + * RVV support. + */ + +uint64_t detect_riscv64_rvv100(void) +{ + uint64_t rvv10_supported; + + /* + * After the vsetvli statement vtype will either be a value > 0 if the + * vsetvli succeeded or less than 0 if it failed. If 0 < vtype + * we're good and the function will return 1, otherwise there's no + * RVV 1.0 and we return 0. + */ + + asm volatile("vsetvli x0, x0, e8, m1, ta, ma\n\t" + "csrr %0, vtype\n\t" + "slt %0, x0, %0\n" + : "=r" (rvv10_supported) + : + :); + + return rvv10_supported; +} + diff --git a/driver/others/dynamic_riscv64.c b/driver/others/dynamic_riscv64.c new file mode 100644 index 0000000000..78e3bb67a4 --- /dev/null +++ b/driver/others/dynamic_riscv64.c @@ -0,0 +1,269 @@ +/***************************************************************************** +Copyright (c) 2024, The OpenBLAS Project +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. Neither the name of the OpenBLAS project nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************************/ + +#include + +#include "common.h" + +/* + * OpenBLAS contains some kernels that are optimised for RVV 1.0. Before we + * can use these kernels we need to determine whether the device supports + * RVV 1.0 and what the device's VLEN is. Our strategy will be as follows. + * + * First we'll invoke the hwprobe syscall to detect RVV 1.0. In an ideal world, + * this is all we should need to do. If the syscall is not implemented we + * should be able to deduce that RVV 1.0 is not supported (as it was added to + * Linux after hwprobe) and if the syscall is implemented we can use it to + * determine whether RVV 1.0 is supported. However, there are some riscv64 + * boards out there that implement RVV 1.0 but ship with a Linux kernel that + * predates RVV vector support and hwprobe support. These kernels contain + * the backported RVV patches but not the hwprobe patches and so they + * advertise support for RVV via hwcap. To cater for these boards we need + * to fall back to hwcap if hwprobe is not supported. Unfortunately, some + * boards indicate support for RVV via hwcap even though they only support + * RVV 0.7.1, which is incompatible with RVV 1.0. So an additional check is + * required to test if the devices advertising support for RVV via hwcap really + * support RVV 1.0. This test works by executing a vsetvli instruction that + * sets the tail agnostic and mask agnostic bits in the vtype register. + * These bits are not supported prior to RVV 0.9 so will cause the VIL bit to + * be set on the VTYPE register in CPUs supporting 0.7.1. If this bit is set + * we can determine that RVV 1.0 is not supported. + * + * This approach is borrowed from + * VideoLan dav1d: + * (https://code.videolan.org/videolan/dav1d/-/merge_requests/1629). + * + * We assume that if a kernel reports the presence of RVV via hwcap that + * the device supports the vsetvli instruction. + * + * For now we're just going to invoke the hwprobe syscall directly, rather than + * invoking it through glibc. Support for hwprobe has been added to glibc but + * at the time of writing this support has not yet been included in a glibc + * release. Once it has, it will be better to invoke hwprobe via glibc as doing + * so should take advantage of the vdso entry and be more efficient. + */ + +/* + * This should work on Android as well but I have no way of testing. + */ + +#if defined(OS_LINUX) +#include +#include +#include +#include + +#define DETECT_RISCV64_HWCAP_ISA_V (1 << ('V' - 'A')) + +struct riscv_hwprobe { + int64_t key; + uint64_t value; +}; + +/* The constants below are copied from + * /usr/include/riscv64-linux-gnu/asm/hwprobe.h. We duplicate the + * constants as the header file from which they are copied will only + * be present if we're building on a device with Linux 6.5 or greater. + */ + +#define RISCV_HWPROBE_KEY_IMA_EXT_0 4 +#define RISCV_HWPROBE_IMA_V (1 << 2) + +#ifndef NR_riscv_hwprobe +#ifndef NR_arch_specific_syscall +#define NR_arch_specific_syscall 244 +#endif +#define NR_riscv_hwprobe (NR_arch_specific_syscall + 14) +#endif +#endif // defined(OS_LINUX) + +unsigned detect_riscv64_get_vlenb(void); +uint64_t detect_riscv64_rvv100(void); + +extern gotoblas_t gotoblas_RISCV64_GENERIC; +#if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL256B) +extern gotoblas_t gotoblas_RISCV64_ZVL256B; +#endif +#if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL128B) +extern gotoblas_t gotoblas_RISCV64_ZVL128B; +#endif + +#define CPU_GENERIC 0 +#define CPU_RISCV64_ZVL256B 1 +#define CPU_RISCV64_ZVL128B 2 + +static char *cpuname[] = { + "riscv64_generic", + "riscv64_zvl256b", + "riscv64_zvl128b" +}; +#define NUM_CORETYPES (sizeof(cpuname)/sizeof(char*)) + +extern int openblas_verbose(void); +extern void openblas_warning(int verbose, const char* msg); + +char* gotoblas_corename(void) { +#if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL256B) + if (gotoblas == &gotoblas_RISCV64_ZVL256B) + return cpuname[CPU_RISCV64_ZVL256B]; +#endif +#if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL128B) + if (gotoblas == &gotoblas_RISCV64_ZVL128B) + return cpuname[CPU_RISCV64_ZVL128B]; +#endif + if (gotoblas == &gotoblas_RISCV64_GENERIC) + return cpuname[CPU_GENERIC]; + + return "unknown"; +} + +static gotoblas_t* get_coretype(void) { + unsigned vlenb = 0; + +#if !defined(OS_LINUX) + return NULL; +#else + + /* + * See the hwprobe documentation + * + * ( https://docs.kernel.org/arch/riscv/hwprobe.html ) + * for more details. + */ + + struct riscv_hwprobe pairs[] = { + { .key = RISCV_HWPROBE_KEY_IMA_EXT_0, }, + }; + int ret = syscall(NR_riscv_hwprobe, pairs, 1, 0, NULL, 0); + if (ret == 0) { + if (!(pairs[0].value & RISCV_HWPROBE_IMA_V)) + return NULL; + } else { + if (!(getauxval(AT_HWCAP) & DETECT_RISCV64_HWCAP_ISA_V)) + return NULL; + + if (!detect_riscv64_rvv100()) + return NULL; + } + + /* + * RVV 1.0 is supported. We now just need to determine the coretype + * based on the VLEN. + */ + + vlenb = detect_riscv64_get_vlenb(); + + if (vlenb < 16) + return NULL; +#if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL256B) + if (vlenb >= 32) + return &gotoblas_RISCV64_ZVL256B; +#endif + +#if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL128B) + return &gotoblas_RISCV64_ZVL128B; +#else + return NULL; +#endif + +#endif // !defined(OS_LINUX) +} + +static gotoblas_t* force_coretype(char* coretype) { + size_t i; + char message[128]; + + for (i = 0; i < NUM_CORETYPES && strcasecmp(coretype, cpuname[i]); i++); + + if (i == CPU_GENERIC) + return &gotoblas_RISCV64_GENERIC; + + if (i == CPU_RISCV64_ZVL256B) { +#if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL256B) + return &gotoblas_RISCV64_ZVL256B; +#else + openblas_warning(1, + "riscv64_zvl256b support not compiled in\n"); + return NULL; +#endif + } + + if (i == CPU_RISCV64_ZVL128B) { +#if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL128B) + return &gotoblas_RISCV64_ZVL128B; +#else + openblas_warning(1, + "riscv64_zvl128b support not compiled in\n"); + return NULL; +#endif + } + + snprintf(message, sizeof(message), "Core not found: %s\n", coretype); + openblas_warning(1, message); + + return NULL; +} + +void gotoblas_dynamic_init(void) { + + char coremsg[128]; + char* p; + + if (gotoblas) return; + + p = getenv("OPENBLAS_CORETYPE"); + if (p) + gotoblas = force_coretype(p); + else + gotoblas = get_coretype(); + + if (!gotoblas) { + snprintf(coremsg, sizeof(coremsg), "Falling back to generic riscv64 core\n"); + openblas_warning(1, coremsg); + gotoblas = &gotoblas_RISCV64_GENERIC; + } + + if (gotoblas->init) { + snprintf(coremsg, sizeof(coremsg), "Core: %s\n", + gotoblas_corename()); + openblas_warning(2, coremsg); + gotoblas->init(); + return; + } + + openblas_warning(0, "OpenBLAS : Architecture Initialization failed. No initialization function found.\n"); + exit(1); +} + +void gotoblas_dynamic_quit(void) { + gotoblas = NULL; +} diff --git a/kernel/riscv64/KERNEL.RISCV64_GENERIC b/kernel/riscv64/KERNEL.RISCV64_GENERIC index 15bcd2289e..67f81cacda 100644 --- a/kernel/riscv64/KERNEL.RISCV64_GENERIC +++ b/kernel/riscv64/KERNEL.RISCV64_GENERIC @@ -99,26 +99,26 @@ ZTRMMKERNEL = ../generic/ztrmmkernel_2x2.c SGEMMKERNEL = ../generic/gemmkernel_2x2.c SGEMMONCOPY = ../generic/gemm_ncopy_2.c SGEMMOTCOPY = ../generic/gemm_tcopy_2.c -SGEMMONCOPYOBJ = sgemm_oncopy.o -SGEMMOTCOPYOBJ = sgemm_otcopy.o +SGEMMONCOPYOBJ = sgemm_oncopy$(TSUFFIX).$(SUFFIX) +SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX) DGEMMKERNEL = ../generic/gemmkernel_2x2.c DGEMMONCOPY = ../generic/gemm_ncopy_2.c DGEMMOTCOPY = ../generic/gemm_tcopy_2.c -DGEMMONCOPYOBJ = dgemm_oncopy.o -DGEMMOTCOPYOBJ = dgemm_otcopy.o +DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX) +DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX) CGEMMKERNEL = ../generic/zgemmkernel_2x2.c CGEMMONCOPY = ../generic/zgemm_ncopy_2.c CGEMMOTCOPY = ../generic/zgemm_tcopy_2.c -CGEMMONCOPYOBJ = cgemm_oncopy.o -CGEMMOTCOPYOBJ = cgemm_otcopy.o +CGEMMONCOPYOBJ = cgemm_oncopy$(TSUFFIX).$(SUFFIX) +CGEMMOTCOPYOBJ = cgemm_otcopy$(TSUFFIX).$(SUFFIX) ZGEMMKERNEL = ../generic/zgemmkernel_2x2.c ZGEMMONCOPY = ../generic/zgemm_ncopy_2.c ZGEMMOTCOPY = ../generic/zgemm_tcopy_2.c -ZGEMMONCOPYOBJ = zgemm_oncopy.o -ZGEMMOTCOPYOBJ = zgemm_otcopy.o +ZGEMMONCOPYOBJ = zgemm_oncopy$(TSUFFIX).$(SUFFIX) +ZGEMMOTCOPYOBJ = zgemm_otcopy$(TSUFFIX).$(SUFFIX) STRSMKERNEL_LN = ../generic/trsm_kernel_LN.c STRSMKERNEL_LT = ../generic/trsm_kernel_LT.c diff --git a/kernel/setparam-ref.c b/kernel/setparam-ref.c index c790b0a583..9d494bfc62 100644 --- a/kernel/setparam-ref.c +++ b/kernel/setparam-ref.c @@ -1244,6 +1244,36 @@ static void init_parameter(void) { } #else //ZARCH +#if (ARCH_RISCV64) +static void init_parameter(void) { + +#ifdef BUILD_BFLOAT16 + TABLE_NAME.sbgemm_p = SBGEMM_DEFAULT_P; +#endif + TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P; + TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P; + TABLE_NAME.cgemm_p = CGEMM_DEFAULT_P; + TABLE_NAME.zgemm_p = ZGEMM_DEFAULT_P; + +#ifdef BUILD_BFLOAT16 + TABLE_NAME.sbgemm_r = SBGEMM_DEFAULT_R; +#endif + TABLE_NAME.sgemm_r = SGEMM_DEFAULT_R; + TABLE_NAME.dgemm_r = DGEMM_DEFAULT_R; + TABLE_NAME.cgemm_r = CGEMM_DEFAULT_R; + TABLE_NAME.zgemm_r = ZGEMM_DEFAULT_R; + + +#ifdef BUILD_BFLOAT16 + TABLE_NAME.sbgemm_q = SBGEMM_DEFAULT_Q; +#endif + TABLE_NAME.sgemm_q = SGEMM_DEFAULT_Q; + TABLE_NAME.dgemm_q = DGEMM_DEFAULT_Q; + TABLE_NAME.cgemm_q = CGEMM_DEFAULT_Q; + TABLE_NAME.zgemm_q = ZGEMM_DEFAULT_Q; +} +#else //RISCV64 + #ifdef ARCH_X86 static int get_l2_size_old(void){ int i, eax, ebx, ecx, edx, cpuid_level; @@ -2046,6 +2076,7 @@ static void init_parameter(void) { } +#endif //RISCV64 #endif //POWER #endif //ZARCH #endif //(ARCH_LOONGARCH64) diff --git a/lapack/laswp/riscv64/Makefile b/lapack/laswp/riscv64/Makefile index 75411deb54..bc39a30f66 100644 --- a/lapack/laswp/riscv64/Makefile +++ b/lapack/laswp/riscv64/Makefile @@ -1,6 +1,11 @@ TOPDIR = ../../.. include ../../../Makefile.system +ifeq ($(DYNAMIC_ARCH), 1) +LASWP = ../generic/laswp_k_4.c +ZLASWP = ../generic/zlaswp_k_4.c +endif + ifndef LASWP LASWP = ../generic/laswp_k.c endif From 67bf4b6998d9ca2ec51311655004dc9bb32ea21c Mon Sep 17 00:00:00 2001 From: Mark Ryan Date: Fri, 12 Jul 2024 11:16:48 +0000 Subject: [PATCH 196/267] Fix axpby_rvv kernels for cases where inc_y = 0 The following openblas_utest tests fail when the RISCV64_ZVL128B is enabled. TEST 89/103 axpby:zaxpby_inc_0 [FAIL] TEST 92/103 axpby:caxpby_inc_0 [FAIL] TEST 95/103 axpby:daxpby_inc_0 [FAIL] TEST 98/103 axpby:saxpby_inc_0 [FAIL] The issue is that the vectorized kernels do not work when inc_y == 0. This patch updates the kernels to fall back to the scalar algorithms when inc_y == 0, fixing the failing tests. Signed-off-by: Mark Ryan --- kernel/riscv64/axpby_rvv.c | 12 ++++ kernel/riscv64/zaxpby_rvv.c | 109 ++++++++++++++++++++++-------------- 2 files changed, 78 insertions(+), 43 deletions(-) diff --git a/kernel/riscv64/axpby_rvv.c b/kernel/riscv64/axpby_rvv.c index d7fb86eab6..27abc0ff4b 100644 --- a/kernel/riscv64/axpby_rvv.c +++ b/kernel/riscv64/axpby_rvv.c @@ -114,6 +114,11 @@ int CNAME(BLASLONG n, FLOAT alpha, FLOAT *x, BLASLONG inc_x, FLOAT beta, FLOAT * vy = VFMULVF_FLOAT(vy, beta, vl); VSEV_FLOAT (y, vy, vl); } + } else if (inc_y == 0) { + FLOAT vf = y[0]; + for (; n > 0; n--) + vf *= beta; + y[0] = vf; } else { BLASLONG stride_y = inc_y * sizeof(FLOAT); for (size_t vl; n > 0; n -= vl, y += vl*inc_y) { @@ -134,6 +139,13 @@ int CNAME(BLASLONG n, FLOAT alpha, FLOAT *x, BLASLONG inc_x, FLOAT beta, FLOAT * vy = VFMACCVF_FLOAT(vy, alpha, vx, vl); VSEV_FLOAT (y, vy, vl); } + } else if (inc_y == 0) { + FLOAT vf = y[0]; + for (; n > 0; n--) { + vf = (vf * beta) + (x[0] * alpha); + x += inc_x; + } + y[0] = vf; } else if (1 == inc_x) { BLASLONG stride_y = inc_y * sizeof(FLOAT); for (size_t vl; n > 0; n -= vl, x += vl, y += vl*inc_y) { diff --git a/kernel/riscv64/zaxpby_rvv.c b/kernel/riscv64/zaxpby_rvv.c index 66e38c1e47..9bf5bdd5b6 100644 --- a/kernel/riscv64/zaxpby_rvv.c +++ b/kernel/riscv64/zaxpby_rvv.c @@ -79,8 +79,10 @@ int CNAME(BLASLONG n, FLOAT alpha_r, FLOAT alpha_i, FLOAT *x, BLASLONG inc_x, FL BLASLONG stride_x = inc_x2 * sizeof(FLOAT); BLASLONG stride_y = inc_y2 * sizeof(FLOAT); + BLASLONG ix; FLOAT_V_T vx0, vx1, vy0, vy1; FLOAT_VX2_T vxx2, vyx2; + FLOAT temp; if ( beta_r == 0.0 && beta_i == 0.0) { @@ -125,53 +127,74 @@ int CNAME(BLASLONG n, FLOAT alpha_r, FLOAT alpha_i, FLOAT *x, BLASLONG inc_x, FL if ( alpha_r == 0.0 && alpha_i == 0.0 ) { - for (size_t vl; n > 0; n -= vl, y += vl*inc_y2) - { - vl = VSETVL(n); - - vyx2 = VLSSEG_FLOAT(y, stride_y, vl); - vy0 = VGET_VX2(vyx2, 0); - vy1 = VGET_VX2(vyx2, 1); - - v0 = VFMULVF_FLOAT(vy1, beta_i, vl); - v0 = VFMSACVF_FLOAT(v0, beta_r, vy0, vl); - - v1 = VFMULVF_FLOAT(vy1, beta_r, vl); - v1 = VFMACCVF_FLOAT(v1, beta_i, vy0, vl); - - v_x2 = VSET_VX2(v_x2, 0, v0); - v_x2 = VSET_VX2(v_x2, 1, v1); - VSSSEG_FLOAT(y, stride_y, v_x2, vl); + if ( inc_y == 0 ) { + for (; n > 0; n--) + { + temp = (beta_r * y[0] - beta_i * y[1]); + y[1] = (beta_r * y[1] + beta_i * y[0]); + y[0] = temp; + } + } else { + for (size_t vl; n > 0; n -= vl, y += vl*inc_y2) + { + vl = VSETVL(n); + + vyx2 = VLSSEG_FLOAT(y, stride_y, vl); + vy0 = VGET_VX2(vyx2, 0); + vy1 = VGET_VX2(vyx2, 1); + + v0 = VFMULVF_FLOAT(vy1, beta_i, vl); + v0 = VFMSACVF_FLOAT(v0, beta_r, vy0, vl); + + v1 = VFMULVF_FLOAT(vy1, beta_r, vl); + v1 = VFMACCVF_FLOAT(v1, beta_i, vy0, vl); + + v_x2 = VSET_VX2(v_x2, 0, v0); + v_x2 = VSET_VX2(v_x2, 1, v1); + VSSSEG_FLOAT(y, stride_y, v_x2, vl); + } } } else { - for (size_t vl; n > 0; n -= vl, x += vl*inc_x2, y += vl*inc_y2) - { - vl = VSETVL(n); - - vxx2 = VLSSEG_FLOAT(x, stride_x, vl); - vyx2 = VLSSEG_FLOAT(y, stride_y, vl); - - vx0 = VGET_VX2(vxx2, 0); - vx1 = VGET_VX2(vxx2, 1); - vy0 = VGET_VX2(vyx2, 0); - vy1 = VGET_VX2(vyx2, 1); - - v0 = VFMULVF_FLOAT(vx0, alpha_r, vl); - v0 = VFNMSACVF_FLOAT(v0, alpha_i, vx1, vl); - v0 = VFMACCVF_FLOAT(v0, beta_r, vy0, vl); - v0 = VFNMSACVF_FLOAT(v0, beta_i, vy1, vl); - - v1 = VFMULVF_FLOAT(vx1, alpha_r, vl); - v1 = VFMACCVF_FLOAT(v1, alpha_i, vx0, vl); - v1 = VFMACCVF_FLOAT(v1, beta_r, vy1, vl); - v1 = VFMACCVF_FLOAT(v1, beta_i, vy0, vl); - - v_x2 = VSET_VX2(v_x2, 0, v0); - v_x2 = VSET_VX2(v_x2, 1, v1); - - VSSSEG_FLOAT(y, stride_y, v_x2, vl); + if ( inc_y == 0 ) { + ix = 0; + for (; n > 0; n--) { + temp = (alpha_r * x[ix] - alpha_i * x[ix+1] ) + + (beta_r * y[0] - beta_i * y[1]); + y[1] = (alpha_r * x[ix+1] + alpha_i * x[ix]) + + (beta_r * y[1] + beta_i * y[0]); + y[0] = temp; + ix += inc_x2; + } + } else { + for (size_t vl; n > 0; n -= vl, x += vl*inc_x2, y += vl*inc_y2) + { + vl = VSETVL(n); + + vxx2 = VLSSEG_FLOAT(x, stride_x, vl); + vyx2 = VLSSEG_FLOAT(y, stride_y, vl); + + vx0 = VGET_VX2(vxx2, 0); + vx1 = VGET_VX2(vxx2, 1); + vy0 = VGET_VX2(vyx2, 0); + vy1 = VGET_VX2(vyx2, 1); + + v0 = VFMULVF_FLOAT(vx0, alpha_r, vl); + v0 = VFNMSACVF_FLOAT(v0, alpha_i, vx1, vl); + v0 = VFMACCVF_FLOAT(v0, beta_r, vy0, vl); + v0 = VFNMSACVF_FLOAT(v0, beta_i, vy1, vl); + + v1 = VFMULVF_FLOAT(vx1, alpha_r, vl); + v1 = VFMACCVF_FLOAT(v1, alpha_i, vx0, vl); + v1 = VFMACCVF_FLOAT(v1, beta_r, vy1, vl); + v1 = VFMACCVF_FLOAT(v1, beta_i, vy0, vl); + + v_x2 = VSET_VX2(v_x2, 0, v0); + v_x2 = VSET_VX2(v_x2, 1, v1); + + VSSSEG_FLOAT(y, stride_y, v_x2, vl); + } } } } From a373d0f107dd85b0bc711b36ef88ec27ec187296 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Mon, 15 Jul 2024 18:32:21 +0200 Subject: [PATCH 197/267] Improve the error message for thread creation failure --- driver/others/blas_server.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/driver/others/blas_server.c b/driver/others/blas_server.c index 64130b0dd0..765511d8c7 100644 --- a/driver/others/blas_server.c +++ b/driver/others/blas_server.c @@ -595,6 +595,8 @@ int blas_thread_init(void){ struct rlimit rlim; const char *msg = strerror(ret); fprintf(STDERR, "OpenBLAS blas_thread_init: pthread_create failed for thread %ld of %d: %s\n", i+1,blas_num_threads,msg); + fprintf(STDERR, "OpenBLAS blas_thread_init: ensure that your address space and process count limits are big enough (ulimit -a)\n"); + fprintf(STDERR, "OpenBLAS blas_thread_init: or set a smaller OPENBLAS_NUM_THREADS to fit into what you have available\n"); #ifdef RLIMIT_NPROC if(0 == getrlimit(RLIMIT_NPROC, &rlim)) { fprintf(STDERR, "OpenBLAS blas_thread_init: RLIMIT_NPROC " From 127ea5d0d95bd7665897994ea204c85096c8f3e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladimir=20Nikoli=C4=87?= Date: Mon, 15 Jul 2024 15:12:21 -0700 Subject: [PATCH 198/267] Add missing parenthesis --- ctest/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctest/Makefile b/ctest/Makefile index 46885f3bb7..c02e04e1a3 100644 --- a/ctest/Makefile +++ b/ctest/Makefile @@ -26,7 +26,7 @@ endif override CFLAGS += -DADD$(BU) -DCBLAS ifeq ($(F_COMPILER),GFORTRAN) ifneq (, $(filter $(CORE),LOONGSON3R3 LOONGSON3R4)) - override FFLAGS = $(filter_out(-O2 -O3,$(FFLAGS)) -O0 + override FFLAGS = $(filter_out(-O2 -O3,$(FFLAGS))) -O0 endif override FFLAGS += -fno-tree-vectorize endif From 56e1782ffba67271d6884e8d95e012a2980cd1b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladimir=20Nikoli=C4=87?= Date: Mon, 15 Jul 2024 15:15:23 -0700 Subject: [PATCH 199/267] Add another missing parenthesis --- test/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Makefile b/test/Makefile index 85bdca1030..27afce8d25 100644 --- a/test/Makefile +++ b/test/Makefile @@ -2,7 +2,7 @@ TOPDIR = .. include ../Makefile.system ifeq ($(F_COMPILER),GFORTRAN) ifneq (, $(filter $(CORE),LOONGSON3R3 LOONGSON3R4)) - override FFLAGS = $(filter_out(-O2 -O3,$(FFLAGS)) -O0 + override FFLAGS = $(filter_out(-O2 -O3,$(FFLAGS))) -O0 endif override FFLAGS += -fno-tree-vectorize endif From 0985fdc82b727ec3283f78e30f34910ea48937f9 Mon Sep 17 00:00:00 2001 From: iha fujitsu Date: Tue, 16 Jul 2024 17:31:33 +0900 Subject: [PATCH 200/267] A64FX: Add support for SVE to SGEMV/DGEMV kernels. --- kernel/arm64/KERNEL.A64FX | 5 +++ kernel/arm64/gemv_n_sve.c | 92 ++++++++++++++++++++++++++++++++++++++ kernel/arm64/gemv_t_sve.c | 94 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 191 insertions(+) create mode 100644 kernel/arm64/gemv_n_sve.c create mode 100644 kernel/arm64/gemv_t_sve.c diff --git a/kernel/arm64/KERNEL.A64FX b/kernel/arm64/KERNEL.A64FX index bc59990979..4abc840405 100644 --- a/kernel/arm64/KERNEL.A64FX +++ b/kernel/arm64/KERNEL.A64FX @@ -1 +1,6 @@ include $(KERNELDIR)/KERNEL.ARMV8SVE + +SGEMVNKERNEL = gemv_n_sve.c +DGEMVNKERNEL = gemv_n_sve.c +SGEMVTKERNEL = gemv_t_sve.c +DGEMVTKERNEL = gemv_t_sve.c diff --git a/kernel/arm64/gemv_n_sve.c b/kernel/arm64/gemv_n_sve.c new file mode 100644 index 0000000000..d3aa57ae32 --- /dev/null +++ b/kernel/arm64/gemv_n_sve.c @@ -0,0 +1,92 @@ +/*************************************************************************** +Copyright (c) 2024, The OpenBLAS Project +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. Neither the name of the OpenBLAS project nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include +#include "common.h" + +#ifdef DOUBLE +#define SV_COUNT svcntd +#define SV_TYPE svfloat64_t +#define SV_TRUE svptrue_b64 +#define SV_WHILE svwhilelt_b64 +#define SV_DUP svdup_f64 +#else +#define SV_COUNT svcntw +#define SV_TYPE svfloat32_t +#define SV_TRUE svptrue_b32 +#define SV_WHILE svwhilelt_b32 +#define SV_DUP svdup_f32 +#endif + +int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer) +{ + BLASLONG i; + BLASLONG ix,iy; + BLASLONG j; + FLOAT *a_ptr; + FLOAT temp; + + ix = 0; + a_ptr = a; + + if (inc_y == 1) { + uint64_t sve_size = SV_COUNT(); + for (j = 0; j < n; j++) { + SV_TYPE temp_vec = SV_DUP(alpha * x[ix]); + i = 0; + svbool_t pg = SV_WHILE(i, m); + while (svptest_any(SV_TRUE(), pg)) { + SV_TYPE a_vec = svld1(pg, a_ptr + i); + SV_TYPE y_vec = svld1(pg, y + i); + y_vec = svmla_x(pg, y_vec, temp_vec, a_vec); + svst1(pg, y + i, y_vec); + i += sve_size; + pg = SV_WHILE(i, m); + } + a_ptr += lda; + ix += inc_x; + } + return(0); + } + + for (j = 0; j < n; j++) { + temp = alpha * x[ix]; + iy = 0; + for (i = 0; i < m; i++) { + y[iy] += temp * a_ptr[i]; + iy += inc_y; + } + a_ptr += lda; + ix += inc_x; + } + return (0); +} diff --git a/kernel/arm64/gemv_t_sve.c b/kernel/arm64/gemv_t_sve.c new file mode 100644 index 0000000000..bff08b2579 --- /dev/null +++ b/kernel/arm64/gemv_t_sve.c @@ -0,0 +1,94 @@ +/*************************************************************************** +Copyright (c) 2024, The OpenBLAS Project +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. Neither the name of the OpenBLAS project nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include +#include "common.h" + +#ifdef DOUBLE +#define SV_COUNT svcntd +#define SV_TYPE svfloat64_t +#define SV_TRUE svptrue_b64 +#define SV_WHILE svwhilelt_b64 +#define SV_DUP svdup_f64 +#else +#define SV_COUNT svcntw +#define SV_TYPE svfloat32_t +#define SV_TRUE svptrue_b32 +#define SV_WHILE svwhilelt_b32 +#define SV_DUP svdup_f32 +#endif + +int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer) +{ + BLASLONG i; + BLASLONG ix,iy; + BLASLONG j; + FLOAT *a_ptr; + FLOAT temp; + + iy = 0; + a_ptr = a; + + if (inc_x == 1) { + uint64_t sve_size = SV_COUNT(); + for (j = 0; j < n; j++) { + SV_TYPE temp_vec = SV_DUP(0.0); + i = 0; + svbool_t pg = SV_WHILE(i, m); + while (svptest_any(SV_TRUE(), pg)) { + SV_TYPE a_vec = svld1(pg, a_ptr + i); + SV_TYPE x_vec = svld1(pg, x + i); + temp_vec = svmla_m(pg, temp_vec, a_vec, x_vec); + i += sve_size; + pg = SV_WHILE(i, m); + } + temp = svaddv(SV_TRUE(), temp_vec); + y[iy] += alpha * temp; + iy += inc_y; + a_ptr += lda; + } + return(0); + } + + for (j = 0; j < n; j++) { + temp = 0.0; + ix = 0; + for (i = 0; i < m; i++) { + temp += a_ptr[i] * x[ix]; + ix += inc_x; + } + y[iy] += alpha * temp; + iy += inc_y; + a_ptr += lda; + } + return (0); +} From f6d6c14a96602fb2cfb53584541017434f7492e2 Mon Sep 17 00:00:00 2001 From: gxw Date: Wed, 17 Jul 2024 09:23:49 +0800 Subject: [PATCH 201/267] mips: Fixed numpy CI failure --- kernel/mips/dscal_msa.c | 12 +++++----- kernel/mips/scal.c | 49 +++++++++++++++++++++++------------------ kernel/mips/sscal_msa.c | 9 +++----- 3 files changed, 36 insertions(+), 34 deletions(-) diff --git a/kernel/mips/dscal_msa.c b/kernel/mips/dscal_msa.c index e95f0a6552..cc8d834415 100644 --- a/kernel/mips/dscal_msa.c +++ b/kernel/mips/dscal_msa.c @@ -42,7 +42,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, if (1 == inc_x) { - if (0) //if (0.0 == da ) + if (0.0 == da && !dummy2) { v2f64 zero_v = {0.0, 0.0}; @@ -240,14 +240,12 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, } else { - if (da == 0.0) + if (da == 0.0 && !dummy2) { for (i = n; i--;) - { - if (isfinite(*x)) - *x = 0.0; - else - *x = NAN; + { + *x = 0.0; + x += inc_x; } } diff --git a/kernel/mips/scal.c b/kernel/mips/scal.c index d51fd9ccdd..5f12d42711 100644 --- a/kernel/mips/scal.c +++ b/kernel/mips/scal.c @@ -29,27 +29,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *dummy, BLASLONG dummy2) { - BLASLONG i=0,j=0; - - while(j < n) - { - - if ( da == 0.0 ) - if (isnan(x[i])||isinf(x[i])) - x[i]=NAN; - else - x[i]=0.0; - else if (isnan(da)) - x[i]=NAN; - else - x[i] = da * x[i] ; - - i += inc_x ; - j++; - - } - return 0; - + BLASLONG i = 0, j = 0; + + // Resolved issue 4728 when the caller is {s/d}scal + if (da == 0.0 && dummy2 == 1) + { + while(j < n) + { + x[i] = da * x[i] ; + + i += inc_x ; + j++; + } + } + else + { + while(j < n) + { + + if ( da == 0.0 ) + x[i] = 0.0; + else + x[i] = da * x[i] ; + + i += inc_x ; + j++; + } + } + return 0; } diff --git a/kernel/mips/sscal_msa.c b/kernel/mips/sscal_msa.c index bfd477b6a5..953cf0fbcf 100644 --- a/kernel/mips/sscal_msa.c +++ b/kernel/mips/sscal_msa.c @@ -42,7 +42,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, if (1 == inc_x) { - if (0) // if (0.0 == da) + if (0.0 == da && !dummy2) { v4f32 zero_v = {0.0, 0.0, 0.0, 0.0}; @@ -255,14 +255,11 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, } else { - if (0.0 == da) + if (0.0 == da && !dummy2) { for (i = n; i--;) { - if (isfinite(*x)) - *x = 0; - else - *x = NAN; + *x = 0; x += inc_x; } } From 34b80ce03f922ebdde8e081a7c5380f460cf215c Mon Sep 17 00:00:00 2001 From: gxw Date: Wed, 17 Jul 2024 09:52:14 +0800 Subject: [PATCH 202/267] mips64: Fixed numpy CI failure --- kernel/mips64/scal.S | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kernel/mips64/scal.S b/kernel/mips64/scal.S index e392a9c6a5..49716d3edd 100644 --- a/kernel/mips64/scal.S +++ b/kernel/mips64/scal.S @@ -48,6 +48,7 @@ #define TEMP $3 #define XX $5 +#define DUMMY2 $6 #define ALPHA $f15 @@ -73,13 +74,13 @@ blez N, .L999 dsll INCX, INCX, BASE_SHIFT - CMPEQ $fcc0, ALPHA, a1 - NOP + CMPEQ $fcc0, ALPHA, a1 + LDARG DUMMY2, 8($sp) bc1f $fcc0, .L50 - NOP + dsll DUMMY2, DUMMY2, BASE_SHIFT - bc1t $fcc0, .L50 + beq DUMMY2, TEMP, .L50 // If dummy2 == 1, do not directly copy 0 NOP bne INCX, TEMP, .L20 From eb4879e04cb93cb112c2ec5ec79cb4a837b5d517 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 17 Jul 2024 23:24:19 +0200 Subject: [PATCH 203/267] make NAN handling depend on the dummy2 parameter --- kernel/arm64/scal.S | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/kernel/arm64/scal.S b/kernel/arm64/scal.S index 5029890f67..33400b6309 100644 --- a/kernel/arm64/scal.S +++ b/kernel/arm64/scal.S @@ -33,7 +33,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define X_COPY x5 /* X vector address */ #define INC_X x4 /* X stride */ #define I x1 /* loop variable */ - +#define FLAG x9 /******************************************************************************* * Macro definitions *******************************************************************************/ @@ -168,9 +168,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cmp N, xzr ble .Lscal_kernel_L999 - //fcmp DA, #0.0 - //beq .Lscal_kernel_zero + ldr FLAG, [sp] + cmp FLAG, #1 + beq .Lscal_kernel_nansafe + + fcmp DA, #0.0 + beq .Lscal_kernel_zero +.Lscal_kernel_nansafe: cmp INC_X, #1 bne .Lscal_kernel_S_BEGIN From b9bfc8ce095754231912686c3a9e41c7a8a2df94 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 17 Jul 2024 23:29:50 +0200 Subject: [PATCH 204/267] make NAN handling depend on dummy2 parameter --- kernel/power/scal.S | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/kernel/power/scal.S b/kernel/power/scal.S index 7d3e232450..5e92a88aa1 100644 --- a/kernel/power/scal.S +++ b/kernel/power/scal.S @@ -47,9 +47,11 @@ #ifndef __64BIT__ #define X r6 #define INCX r7 +#define FLAG r11 #else #define X r7 #define INCX r8 +#define FLAG r12 #endif #endif @@ -57,9 +59,11 @@ #if !defined(__64BIT__) && defined(DOUBLE) #define X r8 #define INCX r9 +#define FLAG r13 #else #define X r7 #define INCX r8 +#define FLAG r12 #endif #endif @@ -84,9 +88,12 @@ cmpwi cr0, N, 0 blelr- cr0 -// fcmpu cr0, FZERO, ALPHA -// bne- cr0, LL(A1I1) - b LL(A1I1) + fcmpu cr0, FZERO, ALPHA + bne- cr0, LL(A1I1) + + ld FLAG, 48+64+8(SP) + cmpwi cr0, FLAG, 1 + beq- cr0, LL(A1I1) cmpwi cr0, INCX, SIZE bne- cr0, LL(A0IN) From 73751218a42d23017cc0f7899d9e52862867e5a3 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 17 Jul 2024 23:41:26 +0200 Subject: [PATCH 205/267] make NAN handling depend on dummy2 parameter --- kernel/arm/scal.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/kernel/arm/scal.c b/kernel/arm/scal.c index 4455d7643b..1f96f9b958 100644 --- a/kernel/arm/scal.c +++ b/kernel/arm/scal.c @@ -43,9 +43,22 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS if ( (n <= 0) || (inc_x <= 0)) return(0); + if (dummy2 == 0) + while(j < n) + { - while(j < n) - { + if ( da == 0.0 ) + x[i]=0.0; + else + x[i] = da * x[i] ; + + i += inc_x ; + j++; + } + } else { + + while(j < n) + { if ( da == 0.0 ) if (!isnan(x[i]) && !isinf(x[i])) { @@ -59,6 +72,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS i += inc_x ; j++; + } } return 0; From 7284c533b56748975c1794058f61e3d044c24ec2 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 17 Jul 2024 23:50:40 +0200 Subject: [PATCH 206/267] make NAN handling depend on dummy2 parameter --- kernel/riscv64/scal.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/kernel/riscv64/scal.c b/kernel/riscv64/scal.c index 6c713aa18c..bebbed67eb 100644 --- a/kernel/riscv64/scal.c +++ b/kernel/riscv64/scal.c @@ -43,9 +43,9 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS if ( (n <= 0) || (inc_x <= 0)) return(0); - - while(j < n) - { + if (dummy2 == 0) { + while(j < n) + { if ( da == 0.0 ) if (isfinite(x[i])) @@ -57,7 +57,19 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS i += inc_x ; j++; + } + } else { + while(j < n) + { + if ( da == 0.0 ) + x[i]=0.0; + else + x[i] = da * x[i] ; + + i += inc_x ; + j++; + } } return 0; From 3870995f01d731dc80861e7760a8216025c639ab Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 17 Jul 2024 23:54:24 +0200 Subject: [PATCH 207/267] make NAN handling depend on dummy2 parameter --- kernel/riscv64/scal_vector.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/riscv64/scal_vector.c b/kernel/riscv64/scal_vector.c index a1ba41c4f6..4792b514cd 100644 --- a/kernel/riscv64/scal_vector.c +++ b/kernel/riscv64/scal_vector.c @@ -71,7 +71,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS FLOAT_V_T v0, v1; unsigned int gvl = 0; if(inc_x == 1){ - if (0){ //if(da == 0.0){ + if(dummy2 == 0 && da == 0.0){ memset(&x[0], 0, n * sizeof(FLOAT)); }else{ gvl = VSETVL(n); @@ -96,7 +96,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS } } }else{ - if (0) { //if(da == 0.0){ + if(dummy2 == 0 && da == 0.0){ BLASLONG stride_x = inc_x * sizeof(FLOAT); BLASLONG ix = 0; gvl = VSETVL(n); From 2020569705b4fcf40e5f4e6aa7e600b038f295bc Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 17 Jul 2024 23:55:54 +0200 Subject: [PATCH 208/267] fix NAN handling and make it depend on dummy2 parameter --- kernel/riscv64/scal_rvv.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/riscv64/scal_rvv.c b/kernel/riscv64/scal_rvv.c index 2c273fb634..827ab120a6 100644 --- a/kernel/riscv64/scal_rvv.c +++ b/kernel/riscv64/scal_rvv.c @@ -56,7 +56,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS FLOAT_V_T v0; if(inc_x == 1) { - if(da == 0.0) { + if(dummy2 == 0 && da == 0.0) { int gvl = VSETVL_MAX; v0 = VFMVVF_FLOAT(0.0, gvl); for (size_t vl; n > 0; n -= vl, x += vl) { @@ -75,7 +75,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS } else { BLASLONG stride_x = inc_x * sizeof(FLOAT); - if(da == 0.0) { + if(dummy2 == 0 && da == 0.0) { int gvl = VSETVL_MAX; v0 = VFMVVF_FLOAT(0.0, gvl); for (size_t vl; n > 0; n -= vl, x += vl*inc_x) { From b1c9fafabb028c73f57d90be991356ea0760a56a Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Thu, 18 Jul 2024 17:37:18 +0100 Subject: [PATCH 209/267] Remove k2 loop from DGEMM TN and use a more conservative heuristic for SGEMM --- kernel/arm64/dgemm_small_kernel_tn_sve.c | 209 +------------------- kernel/arm64/gemm_small_kernel_permit_sve.c | 4 +- 2 files changed, 2 insertions(+), 211 deletions(-) diff --git a/kernel/arm64/dgemm_small_kernel_tn_sve.c b/kernel/arm64/dgemm_small_kernel_tn_sve.c index b8783c1d53..6d3f4dd289 100644 --- a/kernel/arm64/dgemm_small_kernel_tn_sve.c +++ b/kernel/arm64/dgemm_small_kernel_tn_sve.c @@ -265,43 +265,7 @@ CNAME(BLASLONG M, if (LIKELY(packed_a != NULL)) { if (j == 0) { - for (; k < k2; k += 2) { - - VECTOR_LOAD_B_K2(0, 0); - VECTOR_LOAD_B_K2(1, 0); - TRANSPOSE_B2_K2(0, 1, 0, 1); - SCALE_B2_K2(0, 0, 1); - GATHER_LOAD_A(pg_true, 0, 0); - VECTOR_PACK_A(0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - GATHER_LOAD_A(pg_true, 0, 1); - VECTOR_PACK_A(0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); - VECTOR_LOAD_B_K2(2, 0); - VECTOR_LOAD_B_K2(3, 0); - TRANSPOSE_B2_K2(2, 3, 0, 1); - SCALE_B2_K2(2, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 1); - GATHER_LOAD_A(pg_true, 1, 0); - VECTOR_PACK_A(1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0); - GATHER_LOAD_A(pg_true, 1, 1); - VECTOR_PACK_A(1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 1); - } for (; k < K; k++) { - BROADCAST_LOAD_B(0, 0); GATHER_LOAD_A(pg_true, 0, 0); VECTOR_PACK_A(0, 0); @@ -320,39 +284,7 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); } } else { - for (; k < k2; k += 2) { - - VECTOR_LOAD_B_K2(0, 0); - VECTOR_LOAD_B_K2(1, 0); - TRANSPOSE_B2_K2(0, 1, 0, 1); - SCALE_B2_K2(0, 0, 1); - UNPACK_VECTOR_A(0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UNPACK_VECTOR_A(0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); - VECTOR_LOAD_B_K2(2, 0); - VECTOR_LOAD_B_K2(3, 0); - TRANSPOSE_B2_K2(2, 3, 0, 1); - SCALE_B2_K2(2, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 1); - UNPACK_VECTOR_A(1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0); - UNPACK_VECTOR_A(1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 1); - } for (; k < K; k++) { - BROADCAST_LOAD_B(0, 0); UNPACK_VECTOR_A(0, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); @@ -370,37 +302,6 @@ CNAME(BLASLONG M, } } } else { - for (; k < k2; k += 2) { - - VECTOR_LOAD_B_K2(0, 0); - VECTOR_LOAD_B_K2(1, 0); - TRANSPOSE_B2_K2(0, 1, 0, 1); - SCALE_B2_K2(0, 0, 1); - GATHER_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - GATHER_LOAD_A(pg_true, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); - VECTOR_LOAD_B_K2(2, 0); - VECTOR_LOAD_B_K2(3, 0); - TRANSPOSE_B2_K2(2, 3, 0, 1); - SCALE_B2_K2(2, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 1); - GATHER_LOAD_A(pg_true, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0); - GATHER_LOAD_A(pg_true, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 1); - } for (; k < K; k++) { BROADCAST_LOAD_B(0, 0); @@ -443,27 +344,7 @@ CNAME(BLASLONG M, DECLARE_RESULT_VECTOR(1, 1); if (LIKELY(packed_a != NULL)) { - for (; k < k2; k += 2) { - - VECTOR_LOAD_B_K2(0, 0); - VECTOR_LOAD_B_K2(1, 0); - TRANSPOSE_B2_K2(0, 1, 0, 1); - SCALE_B2_K2(0, 0, 1); - UNPACK_VECTOR_A(0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - UNPACK_VECTOR_A(0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); - UNPACK_VECTOR_A(1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - UNPACK_VECTOR_A(1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); - } for (; k < K; k++) { - BROADCAST_LOAD_B(0, 0); UNPACK_VECTOR_A(0, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); @@ -474,27 +355,7 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); } } else { - for (; k < k2; k += 2) { - - VECTOR_LOAD_B_K2(0, 0); - VECTOR_LOAD_B_K2(1, 0); - TRANSPOSE_B2_K2(0, 1, 0, 1); - SCALE_B2_K2(0, 0, 1); - GATHER_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - GATHER_LOAD_A(pg_true, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); - GATHER_LOAD_A(pg_true, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); - GATHER_LOAD_A(pg_true, 1, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); - } for (; k < K; k++) { - BROADCAST_LOAD_B(0, 0); GATHER_LOAD_A(pg_true, 0, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); @@ -570,27 +431,6 @@ CNAME(BLASLONG M, DECLARE_RESULT_VECTOR(0, 2); DECLARE_RESULT_VECTOR(0, 3); - for (; k < k2; k += 2) { - - VECTOR_LOAD_B_K2(0, 0); - VECTOR_LOAD_B_K2(1, 0); - TRANSPOSE_B2_K2(0, 1, 0, 1); - SCALE_B2_K2(0, 0, 1); - GATHER_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - GATHER_LOAD_A(pg_true, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); - VECTOR_LOAD_B_K2(2, 0); - VECTOR_LOAD_B_K2(3, 0); - TRANSPOSE_B2_K2(2, 3, 0, 1); - SCALE_B2_K2(2, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 1); - } for (; k < K; k++) { BROADCAST_LOAD_B(0, 0); @@ -619,19 +459,6 @@ CNAME(BLASLONG M, DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); - for (; k < k2; k += 2) { - - VECTOR_LOAD_B_K2(0, 0); - VECTOR_LOAD_B_K2(1, 0); - TRANSPOSE_B2_K2(0, 1, 0, 1); - SCALE_B2_K2(0, 0, 1); - GATHER_LOAD_A(pg_true, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - GATHER_LOAD_A(pg_true, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); - } for (; k < K; k++) { BROADCAST_LOAD_B(0, 0); @@ -686,27 +513,6 @@ CNAME(BLASLONG M, DECLARE_RESULT_VECTOR(0, 2); DECLARE_RESULT_VECTOR(0, 3); - for (; k < k2; k += 2) { - - VECTOR_LOAD_B_K2(0, 0); - VECTOR_LOAD_B_K2(1, 0); - TRANSPOSE_B2_K2(0, 1, 0, 1); - SCALE_B2_K2(0, 0, 1); - GATHER_LOAD_A(pg_tail, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - GATHER_LOAD_A(pg_tail, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); - VECTOR_LOAD_B_K2(2, 0); - VECTOR_LOAD_B_K2(3, 0); - TRANSPOSE_B2_K2(2, 3, 0, 1); - SCALE_B2_K2(2, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 2, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 2, 1, 1); - } for (; k < K; k++) { BROADCAST_LOAD_B(0, 0); @@ -735,19 +541,6 @@ CNAME(BLASLONG M, DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); - for (; k < k2; k += 2) { - - VECTOR_LOAD_B_K2(0, 0); - VECTOR_LOAD_B_K2(1, 0); - TRANSPOSE_B2_K2(0, 1, 0, 1); - SCALE_B2_K2(0, 0, 1); - GATHER_LOAD_A(pg_tail, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); - GATHER_LOAD_A(pg_tail, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); - UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); - } for (; k < K; k++) { BROADCAST_LOAD_B(0, 0); @@ -787,4 +580,4 @@ CNAME(BLASLONG M, free(packed_a); return 0; -} \ No newline at end of file +} diff --git a/kernel/arm64/gemm_small_kernel_permit_sve.c b/kernel/arm64/gemm_small_kernel_permit_sve.c index c1275129d6..3d425624a2 100644 --- a/kernel/arm64/gemm_small_kernel_permit_sve.c +++ b/kernel/arm64/gemm_small_kernel_permit_sve.c @@ -35,11 +35,9 @@ int CNAME(int transa, int transb, BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alph if (MNK <= 64*64*64) return 1; #else // sgemm - if (MNK <= 256*256*256) + if (MNK <= 64*64*64) return 1; #endif - - return 0; } From 9984c5ce9dae74305b5bbfed0b1fa5602849ac35 Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Thu, 18 Jul 2024 17:34:43 +0000 Subject: [PATCH 210/267] Clean up k2 removal more and unroll SGEMM more --- kernel/arm64/dgemm_small_kernel_tn_sve.c | 23 +--- kernel/arm64/sgemm_small_kernel_nn_sve.c | 167 ++++++++++++++++++++++- kernel/arm64/sgemm_small_kernel_nt_sve.c | 95 ++++++++++++- kernel/arm64/sgemm_small_kernel_tn_sve.c | 104 +++++++++++++- kernel/arm64/sgemm_small_kernel_tt_sve.c | 101 +++++++++++++- 5 files changed, 458 insertions(+), 32 deletions(-) diff --git a/kernel/arm64/dgemm_small_kernel_tn_sve.c b/kernel/arm64/dgemm_small_kernel_tn_sve.c index 6d3f4dd289..1b0fada2ae 100644 --- a/kernel/arm64/dgemm_small_kernel_tn_sve.c +++ b/kernel/arm64/dgemm_small_kernel_tn_sve.c @@ -80,25 +80,12 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. float64x2_t a##m##_k##offset_k = vld1q_dup_f64(&A_ELEMENT_K(m, offset_k)); #define LOAD_A1(m, offset_k) \ float64_t a##m##_k##offset_k = A_ELEMENT_K(m, offset_k); -#define VECTOR_LOAD_B_K2(n, offset_k) \ - float64x2_t b##k##n##_k##offset_k = vld1q_f64(&B_ELEMENT_K(n, offset_k)); -#define TRANSPOSE_B2_K2(n0, n1, offset_k0, offset_k1) \ - float64x2_t b##n0##_k##offset_k0 = \ - vzip1q_f64(b##k##n0##_k##offset_k0, b##k##n1##_k##offset_k0); \ - float64x2_t b##n0##_k##offset_k1 = \ - vzip2q_f64(b##k##n0##_k##offset_k0, b##k##n1##_k##offset_k0); - -#define SCALE_B2_K2(n0, offset_k0, offset_k1) \ - svfloat64_t b##s##n0##_k##offset_k0 = svdup_neonq_f64(b##n0##_k##offset_k0); \ - svfloat64_t b##s##n0##_k##offset_k1 = svdup_neonq_f64(b##n0##_k##offset_k1); #define GATHER_LOAD_B2(n, offset_k) \ float64x2_t b##n##_k##offset_k = vdupq_n_f64(B_ELEMENT_K(n, offset_k)); \ b##n##_k##offset_k = \ vsetq_lane_f64(B_ELEMENT_K(n + 1, offset_k), b##n##_k##offset_k, 1); #define VECTOR_UNPACK_B2(n, offset_k) \ float64x2_t b##n##_k##offset_k = vld1q_f64(&PACK_ELEMENT_K(n, offset_k)); -#define VECTOR_PACK_B2(n, offset_k) \ - vst1q_f64(&PACK_ELEMENT_K(n, offset_k), b##n##_k##offset_k); #define PACK_B0(n, offset_k) \ PACK_ELEMENT_K(n, offset_k) = vget_lane_f64(b##n##_k##offset_k, 0); #define UPDATE_RESULT_VECTOR2(m, n, offset_k) \ @@ -128,9 +115,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. svfloat64_t b##s##n##_k##offset_k = svdup_f64(B_ELEMENT_K(n, offset_k)); #define VECTOR_LOAD_A(pg, m, offset_k) \ svfloat64_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k)); -#define QUADWORD_LOAD_B(n, offset_k) \ - svfloat64_t b##s##n##_k##offset_k = \ - svld1rq(pg_true, &B_ELEMENT_K(n, offset_k)); #define GATHER_LOAD_A(pg, m, offset_k) \ svfloat64_t a##s##m##_k##offset_k = \ svld1_gather_index(pg, &A_ELEMENT_K(m, offset_k), lda_vec); @@ -226,7 +210,6 @@ CNAME(BLASLONG M, const BLASLONG v_m1 = M & -v_size; const BLASLONG n4 = N & -4; const BLASLONG n2 = N & -2; - const BLASLONG k2 = K & -2; const int pack_a = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0; FLOAT* packed_a = @@ -266,6 +249,7 @@ CNAME(BLASLONG M, if (LIKELY(packed_a != NULL)) { if (j == 0) { for (; k < K; k++) { + BROADCAST_LOAD_B(0, 0); GATHER_LOAD_A(pg_true, 0, 0); VECTOR_PACK_A(0, 0); @@ -285,6 +269,7 @@ CNAME(BLASLONG M, } } else { for (; k < K; k++) { + BROADCAST_LOAD_B(0, 0); UNPACK_VECTOR_A(0, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); @@ -345,6 +330,7 @@ CNAME(BLASLONG M, if (LIKELY(packed_a != NULL)) { for (; k < K; k++) { + BROADCAST_LOAD_B(0, 0); UNPACK_VECTOR_A(0, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); @@ -356,6 +342,7 @@ CNAME(BLASLONG M, } } else { for (; k < K; k++) { + BROADCAST_LOAD_B(0, 0); GATHER_LOAD_A(pg_true, 0, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); @@ -580,4 +567,4 @@ CNAME(BLASLONG M, free(packed_a); return 0; -} +} \ No newline at end of file diff --git a/kernel/arm64/sgemm_small_kernel_nn_sve.c b/kernel/arm64/sgemm_small_kernel_nn_sve.c index 2e65e61ff3..0af073a149 100644 --- a/kernel/arm64/sgemm_small_kernel_nn_sve.c +++ b/kernel/arm64/sgemm_small_kernel_nn_sve.c @@ -237,6 +237,7 @@ CNAME(BLASLONG M, #endif { const uint64_t v_size = svcntw(); + const uint64_t v_size2 = v_size * 2; const svbool_t pg_true = svptrue_b32(); const svbool_t pg_quad = svwhilelt_b32(0, 4); const svbool_t pg_first = svwhilelt_b32(0, 1); @@ -245,10 +246,11 @@ CNAME(BLASLONG M, const svfloat32_t beta_vec = svdup_f32(beta); #endif const BLASLONG n4 = N & -4; + const BLASLONG v_m2 = M & -v_size2; const BLASLONG v_m1 = M & -v_size; const BLASLONG k4 = K & -4; - const int pack_b = M >= v_size && N >= 8 && K >= 8 ? 1 : 0; + const int pack_b = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0; FLOAT* packed_b = (pack_b) ? packed_b = (FLOAT*)malloc(K * 4 * sizeof(FLOAT)) : NULL; @@ -269,16 +271,21 @@ CNAME(BLASLONG M, CREATE_B_POINTER(3, 3); BLASLONG i = 0; - for (; i < v_m1; i += v_size) { + for (; i < v_m2; i += v_size2) { CREATE_A_POINTER(0, 0); - UPDATE_A_POINTER(v_size); + CREATE_A_POINTER(1, v_size); + UPDATE_A_POINTER(v_size2); BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); DECLARE_RESULT_VECTOR(0, 2); DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(1, 1); + DECLARE_RESULT_VECTOR(1, 2); + DECLARE_RESULT_VECTOR(1, 3); if (LIKELY(packed_b != NULL)) { if (i == 0) { @@ -314,6 +321,26 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + VECTOR_LOAD_A(pg_true, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1); + VECTOR_LOAD_A(pg_true, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2); + VECTOR_LOAD_A(pg_true, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3); } for (; k < K; k++) { @@ -324,12 +351,17 @@ CNAME(BLASLONG M, BROADCAST_LOAD_B(1, 0); PACK_B(1, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); BROADCAST_LOAD_B(2, 0); PACK_B(2, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); BROADCAST_LOAD_B(3, 0); PACK_B(3, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); } } else { for (; k < K; k++) { @@ -340,11 +372,118 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); } } } else { for (; k < k4; k += 4) { + VECTOR_LOAD_B_K4(0, 0); + VECTOR_LOAD_B_K4(1, 0); + VECTOR_LOAD_B_K4(2, 0); + VECTOR_LOAD_B_K4(3, 0); + TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3); + SCALE_B4_K4(0, 0, 1, 2, 3); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + VECTOR_LOAD_A(pg_true, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1); + VECTOR_LOAD_A(pg_true, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2); + VECTOR_LOAD_A(pg_true, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + VECTOR_LOAD_A(pg_true, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1); + VECTOR_LOAD_A(pg_true, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2); + VECTOR_LOAD_A(pg_true, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3); + } + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); + } + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + VECTOR_STORE(pg_true, 1, 0); + VECTOR_STORE(pg_true, 1, 1); + VECTOR_STORE(pg_true, 1, 2); + VECTOR_STORE(pg_true, 1, 3); + INCR_C_POINTER(0, v_size2); + INCR_C_POINTER(1, v_size2); + INCR_C_POINTER(2, v_size2); + INCR_C_POINTER(3, v_size2); + } + for (; i < v_m1; i += v_size) { + + CREATE_A_POINTER(0, 0); + UPDATE_A_POINTER(v_size); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + + if (LIKELY(packed_b != NULL)) { + for (; k < K; k++) { + + UNPACK_QUADWORD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + } + } else { + for (; k < k4; k += 4) { + VECTOR_LOAD_B_K4(0, 0); VECTOR_LOAD_B_K4(1, 0); VECTOR_LOAD_B_K4(2, 0); @@ -478,6 +617,28 @@ CNAME(BLASLONG M, CREATE_B_POINTER(0, 0); BLASLONG i = 0; + for (; i < v_m2; i += v_size2) { + + CREATE_A_POINTER(0, 0); + CREATE_A_POINTER(1, v_size); + UPDATE_A_POINTER(v_size2); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(1, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 1, 0); + INCR_C_POINTER(0, v_size2); + } for (; i < v_m1; i += v_size) { CREATE_A_POINTER(0, 0); diff --git a/kernel/arm64/sgemm_small_kernel_nt_sve.c b/kernel/arm64/sgemm_small_kernel_nt_sve.c index 9f99c24224..ed7ee6bd63 100644 --- a/kernel/arm64/sgemm_small_kernel_nt_sve.c +++ b/kernel/arm64/sgemm_small_kernel_nt_sve.c @@ -209,6 +209,7 @@ CNAME(BLASLONG M, #endif { const uint64_t v_size = svcntw(); + const uint64_t v_size2 = v_size * 2; const svbool_t pg_true = svptrue_b32(); const svbool_t pg_quad = svwhilelt_b32(0, 4); const svbool_t pg_first = svwhilelt_b32(0, 1); @@ -217,9 +218,10 @@ CNAME(BLASLONG M, const svfloat32_t beta_vec = svdup_f32(beta); #endif const BLASLONG n4 = N & -4; + const BLASLONG v_m2 = M & -v_size2; const BLASLONG v_m1 = M & -v_size; - const int pack_b = M >= v_size && N >= 8 && K >= 8 ? 1 : 0; + const int pack_b = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0; FLOAT* packed_b = (pack_b) ? packed_b = (FLOAT*)malloc(K * 4 * sizeof(FLOAT)) : NULL; @@ -240,16 +242,21 @@ CNAME(BLASLONG M, CREATE_B_POINTER(3, 3); BLASLONG i = 0; - for (; i < v_m1; i += v_size) { + for (; i < v_m2; i += v_size2) { CREATE_A_POINTER(0, 0); - UPDATE_A_POINTER(v_size); + CREATE_A_POINTER(1, v_size); + UPDATE_A_POINTER(v_size2); BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); DECLARE_RESULT_VECTOR(0, 2); DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(1, 1); + DECLARE_RESULT_VECTOR(1, 2); + DECLARE_RESULT_VECTOR(1, 3); if (LIKELY(packed_b != NULL)) { if (i == 0) { @@ -262,6 +269,11 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); } } else { for (; k < K; k++) { @@ -272,11 +284,66 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); } } } else { for (; k < K; k++) { + QUADWORD_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + } + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + VECTOR_STORE(pg_true, 1, 0); + VECTOR_STORE(pg_true, 1, 1); + VECTOR_STORE(pg_true, 1, 2); + VECTOR_STORE(pg_true, 1, 3); + INCR_C_POINTER(0, v_size2); + INCR_C_POINTER(1, v_size2); + INCR_C_POINTER(2, v_size2); + INCR_C_POINTER(3, v_size2); + } + for (; i < v_m1; i += v_size) { + + CREATE_A_POINTER(0, 0); + UPDATE_A_POINTER(v_size); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + + if (LIKELY(packed_b != NULL)) { + for (; k < K; k++) { + + UNPACK_QUADWORD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + } + } else { + for (; k < K; k++) { + QUADWORD_LOAD_B(0, 0); VECTOR_LOAD_A(pg_true, 0, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); @@ -346,6 +413,28 @@ CNAME(BLASLONG M, CREATE_B_POINTER(0, 0); BLASLONG i = 0; + for (; i < v_m2; i += v_size2) { + + CREATE_A_POINTER(0, 0); + CREATE_A_POINTER(1, v_size); + UPDATE_A_POINTER(v_size2); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(1, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + VECTOR_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + VECTOR_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 1, 0); + INCR_C_POINTER(0, v_size2); + } for (; i < v_m1; i += v_size) { CREATE_A_POINTER(0, 0); diff --git a/kernel/arm64/sgemm_small_kernel_tn_sve.c b/kernel/arm64/sgemm_small_kernel_tn_sve.c index 9cbb60d40f..54608a47bb 100644 --- a/kernel/arm64/sgemm_small_kernel_tn_sve.c +++ b/kernel/arm64/sgemm_small_kernel_tn_sve.c @@ -69,7 +69,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #undef C_ELEMENT // #define C_ELEMENT(m, n) C[(i+(m))+(j+(n))*ldc] -#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * v_size + m] +#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * v_size2 + m] #define PACK_ELEMENT(m) PACK_ELEMENT_K(m, 0) // ASIMD @@ -206,6 +206,7 @@ CNAME(BLASLONG M, #endif { const uint64_t v_size = svcntw(); + const uint64_t v_size2 = v_size * 2; const svbool_t pg_true = svptrue_b32(); const svbool_t pg_quad = svwhilelt_b32(0, 4); const svbool_t pg_first = svwhilelt_b32(0, 1); @@ -215,22 +216,25 @@ CNAME(BLASLONG M, #endif const svuint32_t lda_vec = svindex_u32(0LL, lda); + const BLASLONG v_m2 = M & -v_size2; const BLASLONG v_m1 = M & -v_size; const BLASLONG n4 = N & -4; - const int pack_a = M >= v_size && N >= 8 && K >= 8 ? 1 : 0; + const int pack_a = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0; FLOAT* packed_a = - (pack_a) ? packed_a = (FLOAT*)malloc(K * v_size * sizeof(FLOAT)) : NULL; + (pack_a) ? packed_a = (FLOAT*)malloc(K * v_size2 * sizeof(FLOAT)) : NULL; FLOAT* a_offset = A; FLOAT* b_offset = B; FLOAT* c_offset = C; BLASLONG i = 0; - for (; i < v_m1; i += v_size) { + for (; i < v_m2; i += v_size2) { CREATE_C_POINTER(0, 0); + CREATE_C_POINTER(1, v_size); CREATE_A_POINTER(0, 0); + CREATE_A_POINTER(1, v_size); BLASLONG j = 0; for (; j < n4; j += 4) { @@ -246,6 +250,10 @@ CNAME(BLASLONG M, DECLARE_RESULT_VECTOR(0, 1); DECLARE_RESULT_VECTOR(0, 2); DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(1, 1); + DECLARE_RESULT_VECTOR(1, 2); + DECLARE_RESULT_VECTOR(1, 3); if (LIKELY(packed_a != NULL)) { if (j == 0) { @@ -257,10 +265,16 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); BROADCAST_LOAD_B(1, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + GATHER_LOAD_A(pg_true, 1, 0); + VECTOR_PACK_A(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); BROADCAST_LOAD_B(2, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); BROADCAST_LOAD_B(3, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); } } else { for (; k < K; k++) { @@ -270,10 +284,15 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); BROADCAST_LOAD_B(1, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + UNPACK_VECTOR_A(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); BROADCAST_LOAD_B(2, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); BROADCAST_LOAD_B(3, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); } } } else { @@ -284,17 +303,27 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); BROADCAST_LOAD_B(1, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + GATHER_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); BROADCAST_LOAD_B(2, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); BROADCAST_LOAD_B(3, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); } } VECTOR_STORE(pg_true, 0, 0); VECTOR_STORE(pg_true, 0, 1); VECTOR_STORE(pg_true, 0, 2); VECTOR_STORE(pg_true, 0, 3); + VECTOR_STORE(pg_true, 1, 0); + VECTOR_STORE(pg_true, 1, 1); + VECTOR_STORE(pg_true, 1, 2); + VECTOR_STORE(pg_true, 1, 3); INCR_C_POINTER(0, 4); + INCR_C_POINTER(1, 4); } for (; j < N; j++) { @@ -303,6 +332,7 @@ CNAME(BLASLONG M, BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(1, 0); if (LIKELY(packed_a != NULL)) { for (; k < K; k++) { @@ -310,6 +340,8 @@ CNAME(BLASLONG M, BROADCAST_LOAD_B(0, 0); UNPACK_VECTOR_A(0, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + UNPACK_VECTOR_A(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); } } else { for (; k < K; k++) { @@ -317,9 +349,73 @@ CNAME(BLASLONG M, BROADCAST_LOAD_B(0, 0); GATHER_LOAD_A(pg_true, 0, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + GATHER_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); } } VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 1, 0); + INCR_C_POINTER(0, 1); + INCR_C_POINTER(1, 1); + } + + UPDATE_A_POINTER(v_size2); + RESET_B_POINTER(); + UPDATE_C_POINTER(v_size2); + } + for (; i < v_m1; i += v_size) { + + CREATE_C_POINTER(0, 0); + CREATE_A_POINTER(0, 0); + + BLASLONG j = 0; + for (; j < n4; j += 4) { + + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + CREATE_B_POINTER(2, 2); + CREATE_B_POINTER(3, 3); + UPDATE_B_POINTER(4); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + INCR_C_POINTER(0, 4); + } + for (; j < N; j++) { + + CREATE_B_POINTER(0, 0); + UPDATE_B_POINTER(1); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + } + VECTOR_STORE(pg_true, 0, 0); INCR_C_POINTER(0, 1); } diff --git a/kernel/arm64/sgemm_small_kernel_tt_sve.c b/kernel/arm64/sgemm_small_kernel_tt_sve.c index dd9840c378..50dbd7399c 100644 --- a/kernel/arm64/sgemm_small_kernel_tt_sve.c +++ b/kernel/arm64/sgemm_small_kernel_tt_sve.c @@ -69,7 +69,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #undef C_ELEMENT // #define C_ELEMENT(m, n) C[(i+(m))+(j+(n))*ldc] -#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * v_size + m] +#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * v_size2 + m] #define PACK_ELEMENT(m) PACK_ELEMENT_K(m, 0) // ASIMD @@ -207,6 +207,7 @@ CNAME(BLASLONG M, #endif { const uint64_t v_size = svcntw(); + const uint64_t v_size2 = v_size * 2; const svbool_t pg_true = svptrue_b32(); const svbool_t pg_quad = svwhilelt_b32(0, 4); const svbool_t pg_first = svwhilelt_b32(0, 1); @@ -216,22 +217,25 @@ CNAME(BLASLONG M, #endif const svuint32_t lda_vec = svindex_u32(0LL, lda); + const BLASLONG v_m2 = M & -v_size2; const BLASLONG v_m1 = M & -v_size; const BLASLONG n4 = N & -4; - const int pack_a = M >= v_size && N >= 8 && K >= 8 ? 1 : 0; + const int pack_a = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0; FLOAT* packed_a = - (pack_a) ? packed_a = (FLOAT*)malloc(K * v_size * sizeof(FLOAT)) : NULL; + (pack_a) ? packed_a = (FLOAT*)malloc(K * v_size2 * sizeof(FLOAT)) : NULL; FLOAT* a_offset = A; FLOAT* b_offset = B; FLOAT* c_offset = C; BLASLONG i = 0; - for (; i < v_m1; i += v_size) { + for (; i < v_m2; i += v_size2) { CREATE_C_POINTER(0, 0); + CREATE_C_POINTER(1, v_size); CREATE_A_POINTER(0, 0); + CREATE_A_POINTER(1, v_size); BLASLONG j = 0; for (; j < n4; j += 4) { @@ -247,6 +251,10 @@ CNAME(BLASLONG M, DECLARE_RESULT_VECTOR(0, 1); DECLARE_RESULT_VECTOR(0, 2); DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(1, 1); + DECLARE_RESULT_VECTOR(1, 2); + DECLARE_RESULT_VECTOR(1, 3); if (LIKELY(packed_a != NULL)) { if (j == 0) { @@ -259,6 +267,12 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + GATHER_LOAD_A(pg_true, 1, 0); + VECTOR_PACK_A(1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); } } else { for (; k < K; k++) { @@ -269,6 +283,11 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + UNPACK_VECTOR_A(1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); } } } else { @@ -280,13 +299,23 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + GATHER_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); } } VECTOR_STORE(pg_true, 0, 0); VECTOR_STORE(pg_true, 0, 1); VECTOR_STORE(pg_true, 0, 2); VECTOR_STORE(pg_true, 0, 3); + VECTOR_STORE(pg_true, 1, 0); + VECTOR_STORE(pg_true, 1, 1); + VECTOR_STORE(pg_true, 1, 2); + VECTOR_STORE(pg_true, 1, 3); INCR_C_POINTER(0, 4); + INCR_C_POINTER(1, 4); } for (; j < N; j++) { @@ -295,6 +324,7 @@ CNAME(BLASLONG M, BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(1, 0); if (LIKELY(packed_a != NULL)) { for (; k < K; k++) { @@ -302,6 +332,8 @@ CNAME(BLASLONG M, BROADCAST_LOAD_B(0, 0); UNPACK_VECTOR_A(0, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + UNPACK_VECTOR_A(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); } } else { for (; k < K; k++) { @@ -309,9 +341,70 @@ CNAME(BLASLONG M, BROADCAST_LOAD_B(0, 0); GATHER_LOAD_A(pg_true, 0, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + GATHER_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); } } VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 1, 0); + INCR_C_POINTER(0, 1); + INCR_C_POINTER(1, 1); + } + + UPDATE_A_POINTER(v_size2); + RESET_B_POINTER(); + UPDATE_C_POINTER(v_size2); + } + for (; i < v_m1; i += v_size) { + + CREATE_C_POINTER(0, 0); + CREATE_A_POINTER(0, 0); + + BLASLONG j = 0; + for (; j < n4; j += 4) { + + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + CREATE_B_POINTER(2, 2); + CREATE_B_POINTER(3, 3); + UPDATE_B_POINTER(4); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + INCR_C_POINTER(0, 4); + } + for (; j < N; j++) { + + CREATE_B_POINTER(0, 0); + UPDATE_B_POINTER(1); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + } + VECTOR_STORE(pg_true, 0, 0); INCR_C_POINTER(0, 1); } From a9edddb6953cf70020f81e0d97f37b892325b459 Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Thu, 18 Jul 2024 19:03:34 +0000 Subject: [PATCH 211/267] Unroll TN further --- kernel/arm64/sgemm_small_kernel_tn_sve.c | 231 ++++++++++++++++++++++- 1 file changed, 229 insertions(+), 2 deletions(-) diff --git a/kernel/arm64/sgemm_small_kernel_tn_sve.c b/kernel/arm64/sgemm_small_kernel_tn_sve.c index 54608a47bb..03406daa64 100644 --- a/kernel/arm64/sgemm_small_kernel_tn_sve.c +++ b/kernel/arm64/sgemm_small_kernel_tn_sve.c @@ -218,6 +218,7 @@ CNAME(BLASLONG M, const BLASLONG v_m2 = M & -v_size2; const BLASLONG v_m1 = M & -v_size; + const BLASLONG n8 = N & -8; const BLASLONG n4 = N & -4; const int pack_a = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0; @@ -237,23 +238,35 @@ CNAME(BLASLONG M, CREATE_A_POINTER(1, v_size); BLASLONG j = 0; - for (; j < n4; j += 4) { + for (; j < n8; j += 8) { CREATE_B_POINTER(0, 0); CREATE_B_POINTER(1, 1); CREATE_B_POINTER(2, 2); CREATE_B_POINTER(3, 3); - UPDATE_B_POINTER(4); + CREATE_B_POINTER(4, 4); + CREATE_B_POINTER(5, 5); + CREATE_B_POINTER(6, 6); + CREATE_B_POINTER(7, 7); + UPDATE_B_POINTER(8); BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); DECLARE_RESULT_VECTOR(0, 2); DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(0, 4); + DECLARE_RESULT_VECTOR(0, 5); + DECLARE_RESULT_VECTOR(0, 6); + DECLARE_RESULT_VECTOR(0, 7); DECLARE_RESULT_VECTOR(1, 0); DECLARE_RESULT_VECTOR(1, 1); DECLARE_RESULT_VECTOR(1, 2); DECLARE_RESULT_VECTOR(1, 3); + DECLARE_RESULT_VECTOR(1, 4); + DECLARE_RESULT_VECTOR(1, 5); + DECLARE_RESULT_VECTOR(1, 6); + DECLARE_RESULT_VECTOR(1, 7); if (LIKELY(packed_a != NULL)) { if (j == 0) { @@ -275,6 +288,18 @@ CNAME(BLASLONG M, BROADCAST_LOAD_B(3, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); + BROADCAST_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 4, 0); + BROADCAST_LOAD_B(5, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 5, 0); + BROADCAST_LOAD_B(6, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 6, 0); + BROADCAST_LOAD_B(7, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 7, 0); } } else { for (; k < K; k++) { @@ -293,11 +318,109 @@ CNAME(BLASLONG M, BROADCAST_LOAD_B(3, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); + BROADCAST_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 4, 0); + BROADCAST_LOAD_B(5, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 5, 0); + BROADCAST_LOAD_B(6, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 6, 0); + BROADCAST_LOAD_B(7, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 7, 0); } } } else { for (; k < K; k++) { + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + GATHER_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); + BROADCAST_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 4, 0); + BROADCAST_LOAD_B(5, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 5, 0); + BROADCAST_LOAD_B(6, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 6, 0); + BROADCAST_LOAD_B(7, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 7, 0); + } + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + VECTOR_STORE(pg_true, 0, 4); + VECTOR_STORE(pg_true, 0, 5); + VECTOR_STORE(pg_true, 0, 6); + VECTOR_STORE(pg_true, 0, 7); + VECTOR_STORE(pg_true, 1, 0); + VECTOR_STORE(pg_true, 1, 1); + VECTOR_STORE(pg_true, 1, 2); + VECTOR_STORE(pg_true, 1, 3); + VECTOR_STORE(pg_true, 1, 4); + VECTOR_STORE(pg_true, 1, 5); + VECTOR_STORE(pg_true, 1, 6); + VECTOR_STORE(pg_true, 1, 7); + INCR_C_POINTER(0, 8); + INCR_C_POINTER(1, 8); + } + for (; j < n4; j += 4) { + + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + CREATE_B_POINTER(2, 2); + CREATE_B_POINTER(3, 3); + UPDATE_B_POINTER(4); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(1, 1); + DECLARE_RESULT_VECTOR(1, 2); + DECLARE_RESULT_VECTOR(1, 3); + + if (LIKELY(packed_a != NULL)) { + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + UNPACK_VECTOR_A(0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + UNPACK_VECTOR_A(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0); + } + } else { + for (; k < K; k++) { + BROADCAST_LOAD_B(0, 0); GATHER_LOAD_A(pg_true, 0, 0); UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); @@ -369,6 +492,58 @@ CNAME(BLASLONG M, CREATE_A_POINTER(0, 0); BLASLONG j = 0; + for (; j < n8; j += 8) { + + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + CREATE_B_POINTER(2, 2); + CREATE_B_POINTER(3, 3); + CREATE_B_POINTER(4, 4); + CREATE_B_POINTER(5, 5); + CREATE_B_POINTER(6, 6); + CREATE_B_POINTER(7, 7); + UPDATE_B_POINTER(8); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(0, 4); + DECLARE_RESULT_VECTOR(0, 5); + DECLARE_RESULT_VECTOR(0, 6); + DECLARE_RESULT_VECTOR(0, 7); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0); + BROADCAST_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0); + BROADCAST_LOAD_B(5, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0); + BROADCAST_LOAD_B(6, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0); + BROADCAST_LOAD_B(7, 0); + UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + VECTOR_STORE(pg_true, 0, 4); + VECTOR_STORE(pg_true, 0, 5); + VECTOR_STORE(pg_true, 0, 6); + VECTOR_STORE(pg_true, 0, 7); + INCR_C_POINTER(0, 8); + } for (; j < n4; j += 4) { CREATE_B_POINTER(0, 0); @@ -429,6 +604,58 @@ CNAME(BLASLONG M, CREATE_A_POINTER(0, 0); BLASLONG j = 0; + for (; j < n8; j += 8) { + + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + CREATE_B_POINTER(2, 2); + CREATE_B_POINTER(3, 3); + CREATE_B_POINTER(4, 4); + CREATE_B_POINTER(5, 5); + CREATE_B_POINTER(6, 6); + CREATE_B_POINTER(7, 7); + UPDATE_B_POINTER(8); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(0, 4); + DECLARE_RESULT_VECTOR(0, 5); + DECLARE_RESULT_VECTOR(0, 6); + DECLARE_RESULT_VECTOR(0, 7); + + for (; k < K; k++) { + + BROADCAST_LOAD_B(0, 0); + GATHER_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0); + BROADCAST_LOAD_B(1, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 1, 0); + BROADCAST_LOAD_B(2, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 2, 0); + BROADCAST_LOAD_B(3, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 3, 0); + BROADCAST_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 4, 0); + BROADCAST_LOAD_B(5, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 5, 0); + BROADCAST_LOAD_B(6, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 6, 0); + BROADCAST_LOAD_B(7, 0); + UPDATE_RESULT_VECTOR(pg_tail, 0, 7, 0); + } + VECTOR_STORE(pg_tail, 0, 0); + VECTOR_STORE(pg_tail, 0, 1); + VECTOR_STORE(pg_tail, 0, 2); + VECTOR_STORE(pg_tail, 0, 3); + VECTOR_STORE(pg_tail, 0, 4); + VECTOR_STORE(pg_tail, 0, 5); + VECTOR_STORE(pg_tail, 0, 6); + VECTOR_STORE(pg_tail, 0, 7); + INCR_C_POINTER(0, 8); + } for (; j < n4; j += 4) { CREATE_B_POINTER(0, 0); From db98f8753f71e02812b357dc32ae53a6694fa113 Mon Sep 17 00:00:00 2001 From: Hong Bo Peng Date: Fri, 19 Jul 2024 02:08:19 -0400 Subject: [PATCH 212/267] Try to fix LAPACK testing failures on P7. 1. Remove the FADD insn from the GEMV Transpose code. 2. Remove the FADD insn from GEMM and ZGEMM code. 3. Reorder the compution of the Imaginary part in ZGEMM code. --- kernel/power/gemm_kernel_power6.S | 138 ++--- kernel/power/gemv_t.S | 821 +++++++++++++---------------- kernel/power/zgemm_kernel_power6.S | 551 +++++++++---------- 3 files changed, 678 insertions(+), 832 deletions(-) diff --git a/kernel/power/gemm_kernel_power6.S b/kernel/power/gemm_kernel_power6.S index b274f7655a..47771faf80 100644 --- a/kernel/power/gemm_kernel_power6.S +++ b/kernel/power/gemm_kernel_power6.S @@ -864,15 +864,15 @@ LL(22): LFD f22, 10 * SIZE(BO) LFD f23, 11 * SIZE(BO) - FMADD f2, f18, f24, f2 - FMADD f3, f19, f24, f3 - FMADD f6, f18, f25, f6 - FMADD f7, f19, f25, f7 + FMADD f0, f18, f24, f0 + FMADD f1, f19, f24, f1 + FMADD f4, f18, f25, f4 + FMADD f5, f19, f25, f5 - FMADD f10, f18, f26, f10 - FMADD f11, f19, f26, f11 - FMADD f14, f18, f27, f14 - FMADD f15, f19, f27, f15 + FMADD f8, f18, f26, f8 + FMADD f9, f19, f26, f9 + FMADD f12, f18, f27, f12 + FMADD f13, f19, f27, f13 LFD f16, 4 * SIZE(AO) LFD f17, 5 * SIZE(AO) @@ -899,15 +899,15 @@ LL(22): LFD f22, 18 * SIZE(BO) LFD f23, 19 * SIZE(BO) - FMADD f2, f18, f24, f2 - FMADD f3, f19, f24, f3 - FMADD f6, f18, f25, f6 - FMADD f7, f19, f25, f7 + FMADD f0, f18, f24, f0 + FMADD f1, f19, f24, f1 + FMADD f4, f18, f25, f4 + FMADD f5, f19, f25, f5 - FMADD f10, f18, f26, f10 - FMADD f11, f19, f26, f11 - FMADD f14, f18, f27, f14 - FMADD f15, f19, f27, f15 + FMADD f8, f18, f26, f8 + FMADD f9, f19, f26, f9 + FMADD f12, f18, f27, f12 + FMADD f13, f19, f27, f13 LFD f16, 8 * SIZE(AO) LFD f17, 9 * SIZE(AO) @@ -923,14 +923,6 @@ LL(22): addi BO, BO, 16 * SIZE bdnz LL(22) - fadd f0, f2, f0 - fadd f1, f3, f1 - fadd f4, f6, f4 - fadd f5, f7, f5 - fadd f8, f10, f8 - fadd f9, f11, f9 - fadd f12, f14, f12 - fadd f13, f15, f13 .align 4 LL(25): @@ -1161,10 +1153,10 @@ LL(32): LFD f22, 10 * SIZE(BO) LFD f23, 11 * SIZE(BO) - FMADD f1, f17, f24, f1 - FMADD f5, f17, f25, f5 - FMADD f9, f17, f26, f9 - FMADD f13, f17, f27, f13 + FMADD f0, f17, f24, f0 + FMADD f4, f17, f25, f4 + FMADD f8, f17, f26, f8 + FMADD f12, f17, f27, f12 LFD f24, 12 * SIZE(BO) LFD f25, 13 * SIZE(BO) @@ -1181,10 +1173,10 @@ LL(32): LFD f22, 18 * SIZE(BO) LFD f23, 19 * SIZE(BO) - FMADD f1, f19, f24, f1 - FMADD f5, f19, f25, f5 - FMADD f9, f19, f26, f9 - FMADD f13, f19, f27, f13 + FMADD f0, f19, f24, f0 + FMADD f4, f19, f25, f4 + FMADD f8, f19, f26, f8 + FMADD f12, f19, f27, f12 LFD f16, 4 * SIZE(AO) LFD f17, 5 * SIZE(AO) @@ -1200,10 +1192,6 @@ LL(32): addi BO, BO, 16 * SIZE bdnz LL(32) - fadd f0, f1, f0 - fadd f4, f5, f4 - fadd f8, f9, f8 - fadd f12, f13, f12 .align 4 LL(35): @@ -1691,10 +1679,10 @@ LL(52): FMADD f2, f16, f21, f2 FMADD f3, f17, f21, f3 - FMADD f4, f18, f22, f4 - FMADD f5, f19, f22, f5 - FMADD f6, f18, f23, f6 - FMADD f7, f19, f23, f7 + FMADD f0, f18, f22, f0 + FMADD f1, f19, f22, f1 + FMADD f2, f18, f23, f2 + FMADD f3, f19, f23, f3 LFD f16, 4 * SIZE(AO) LFD f17, 5 * SIZE(AO) @@ -1711,10 +1699,10 @@ LL(52): FMADD f2, f16, f25, f2 FMADD f3, f17, f25, f3 - FMADD f4, f18, f26, f4 - FMADD f5, f19, f26, f5 - FMADD f6, f18, f27, f6 - FMADD f7, f19, f27, f7 + FMADD f0, f18, f26, f0 + FMADD f1, f19, f26, f1 + FMADD f2, f18, f27, f2 + FMADD f3, f19, f27, f3 LFD f16, 8 * SIZE(AO) LFD f17, 9 * SIZE(AO) @@ -1775,21 +1763,11 @@ LL(58): LFD f18, 0 * SIZE(CO2) LFD f19, 1 * SIZE(CO2) - FADD f0, f4, f0 - FADD f1, f5, f1 - FADD f2, f6, f2 - FADD f3, f7, f3 - FMADD f0, f0, f30, f16 FMADD f1, f1, f30, f17 FMADD f2, f2, f30, f18 FMADD f3, f3, f30, f19 #else - FADD f0, f4, f0 - FADD f1, f5, f1 - FADD f2, f6, f2 - FADD f3, f7, f3 - FMUL f0, f0, f30 FMUL f1, f1, f30 FMUL f2, f2, f30 @@ -1916,8 +1894,8 @@ LL(60): LL(62): FMADD f0, f16, f20, f0 FMADD f1, f16, f21, f1 - FMADD f2, f17, f22, f2 - FMADD f3, f17, f23, f3 + FMADD f0, f17, f22, f0 + FMADD f1, f17, f23, f1 LFD f20, 8 * SIZE(BO) LFD f21, 9 * SIZE(BO) @@ -1926,8 +1904,8 @@ LL(62): FMADD f0, f18, f24, f0 FMADD f1, f18, f25, f1 - FMADD f2, f19, f26, f2 - FMADD f3, f19, f27, f3 + FMADD f0, f19, f26, f0 + FMADD f1, f19, f27, f1 LFD f16, 4 * SIZE(AO) LFD f17, 5 * SIZE(AO) @@ -1986,15 +1964,9 @@ LL(68): LFD f16, 0 * SIZE(CO1) LFD f18, 0 * SIZE(CO2) - FADD f0, f2, f0 - FADD f1, f3, f1 - FMADD f0, f0, f30, f16 FMADD f1, f1, f30, f18 #else - FADD f0, f2, f0 - FADD f1, f3, f1 - FMUL f0, f0, f30 FMUL f1, f1, f30 #endif @@ -2007,7 +1979,6 @@ LL(68): fmr f4, f0 fmr f5, f0 - #ifdef TRMMKERNEL #if ( defined(LEFT) && defined(TRANSA)) || \ (!defined(LEFT) && !defined(TRANSA)) @@ -2332,8 +2303,8 @@ LL(80): LL(82): FMADD f0, f16, f20, f0 FMADD f1, f17, f20, f1 - FMADD f2, f18, f21, f2 - FMADD f3, f19, f21, f3 + FMADD f0, f18, f21, f0 + FMADD f1, f19, f21, f1 LFD f16, 4 * SIZE(AO) LFD f17, 5 * SIZE(AO) @@ -2342,8 +2313,8 @@ LL(82): FMADD f0, f16, f22, f0 FMADD f1, f17, f22, f1 - FMADD f2, f18, f23, f2 - FMADD f3, f19, f23, f3 + FMADD f0, f18, f23, f0 + FMADD f1, f19, f23, f1 LFD f16, 8 * SIZE(AO) LFD f17, 9 * SIZE(AO) @@ -2401,15 +2372,9 @@ LL(88): LFD f16, 0 * SIZE(CO1) LFD f17, 1 * SIZE(CO1) - FADD f0, f2, f0 - FADD f1, f3, f1 - FMADD f0, f0, f30, f16 FMADD f1, f1, f30, f17 #else - FADD f0, f2, f0 - FADD f1, f3, f1 - FMUL f0, f0, f30 FMUL f1, f1, f30 #endif @@ -2418,9 +2383,6 @@ LL(88): STFD f1, 1 * SIZE(CO1) lfs f0, FZERO - fmr f1, f0 - fmr f2, f0 - fmr f3, f0 addi CO1, CO1, 2 * SIZE @@ -2512,9 +2474,9 @@ LL(90): LL(92): FMADD f0, f16, f20, f0 - FMADD f1, f17, f21, f1 - FMADD f2, f18, f22, f2 - FMADD f3, f19, f23, f3 + FMADD f0, f17, f21, f0 + FMADD f0, f18, f22, f0 + FMADD f0, f19, f23, f0 LFD f16, 4 * SIZE(AO) LFD f17, 5 * SIZE(AO) @@ -2527,9 +2489,9 @@ LL(92): LFD f23, 7 * SIZE(BO) FMADD f0, f16, f20, f0 - FMADD f1, f17, f21, f1 - FMADD f2, f18, f22, f2 - FMADD f3, f19, f23, f3 + FMADD f0, f17, f21, f0 + FMADD f0, f18, f22, f0 + FMADD f0, f19, f23, f0 LFD f16, 8 * SIZE(AO) LFD f17, 9 * SIZE(AO) @@ -2583,16 +2545,8 @@ LL(98): #ifndef TRMMKERNEL LFD f16, 0 * SIZE(CO1) - FADD f0, f1, f0 - FADD f2, f3, f2 - FADD f0, f2, f0 - FMADD f0, f0, f30, f16 #else - FADD f0, f1, f0 - FADD f2, f3, f2 - FADD f0, f2, f0 - FMUL f0, f0, f30 #endif diff --git a/kernel/power/gemv_t.S b/kernel/power/gemv_t.S index accdad7020..0cabe89e03 100644 --- a/kernel/power/gemv_t.S +++ b/kernel/power/gemv_t.S @@ -409,14 +409,6 @@ LL(11): fmr y06, y01 fmr y07, y01 fmr y08, y01 - fmr y09, y01 - fmr y10, y01 - fmr y11, y01 - fmr y12, y01 - fmr y13, y01 - fmr y14, y01 - fmr y15, y01 - fmr y16, y01 DCBT(Y1, PREC) @@ -465,24 +457,24 @@ LL(12): FMADD y08, a8, b1, y08 LFD a8, 2 * SIZE(AO8) - FMADD y09, a1, b2, y09 + FMADD y01, a1, b2, y01 LFD a1, 3 * SIZE(AO1) - FMADD y10, a2, b2, y10 + FMADD y02, a2, b2, y02 LFD a2, 3 * SIZE(AO2) - FMADD y11, a3, b2, y11 + FMADD y03, a3, b2, y03 LFD a3, 3 * SIZE(AO3) - FMADD y12, a4, b2, y12 + FMADD y04, a4, b2, y04 LFD a4, 3 * SIZE(AO4) - FMADD y13, a5, b2, y13 + FMADD y05, a5, b2, y05 LFD a5, 3 * SIZE(AO5) - FMADD y14, a6, b2, y14 + FMADD y06, a6, b2, y06 LFD a6, 3 * SIZE(AO6) - FMADD y15, a7, b2, y15 + FMADD y07, a7, b2, y07 LFD a7, 3 * SIZE(AO7) - FMADD y16, a8, b2, y16 + FMADD y08, a8, b2, y08 LFD a8, 3 * SIZE(AO8) FMADD y01, a1, b3, y01 @@ -505,24 +497,24 @@ LL(12): FMADD y08, a8, b3, y08 LFD a8, 4 * SIZE(AO8) - FMADD y09, a1, b4, y09 + FMADD y01, a1, b4, y01 LFD a1, 5 * SIZE(AO1) - FMADD y10, a2, b4, y10 + FMADD y02, a2, b4, y02 LFD a2, 5 * SIZE(AO2) - FMADD y11, a3, b4, y11 + FMADD y03, a3, b4, y03 LFD a3, 5 * SIZE(AO3) - FMADD y12, a4, b4, y12 + FMADD y04, a4, b4, y04 LFD a4, 5 * SIZE(AO4) - FMADD y13, a5, b4, y13 + FMADD y05, a5, b4, y05 LFD a5, 5 * SIZE(AO5) - FMADD y14, a6, b4, y14 + FMADD y06, a6, b4, y06 LFD a6, 5 * SIZE(AO6) - FMADD y15, a7, b4, y15 + FMADD y07, a7, b4, y07 LFD a7, 5 * SIZE(AO7) - FMADD y16, a8, b4, y16 + FMADD y08, a8, b4, y08 LFD a8, 5 * SIZE(AO8) LFD b1, 9 * SIZE(BO) @@ -550,24 +542,24 @@ LL(12): FMADD y08, a8, b5, y08 LFD a8, 6 * SIZE(AO8) - FMADD y09, a1, b6, y09 + FMADD y01, a1, b6, y01 LFD a1, 7 * SIZE(AO1) - FMADD y10, a2, b6, y10 + FMADD y02, a2, b6, y02 LFD a2, 7 * SIZE(AO2) - FMADD y11, a3, b6, y11 + FMADD y03, a3, b6, y03 LFD a3, 7 * SIZE(AO3) - FMADD y12, a4, b6, y12 + FMADD y04, a4, b6, y04 LFD a4, 7 * SIZE(AO4) - FMADD y13, a5, b6, y13 + FMADD y05, a5, b6, y05 LFD a5, 7 * SIZE(AO5) - FMADD y14, a6, b6, y14 + FMADD y06, a6, b6, y06 LFD a6, 7 * SIZE(AO6) - FMADD y15, a7, b6, y15 + FMADD y07, a7, b6, y07 LFD a7, 7 * SIZE(AO7) - FMADD y16, a8, b6, y16 + FMADD y08, a8, b6, y08 LFD a8, 7 * SIZE(AO8) FMADD y01, a1, b7, y01 @@ -590,24 +582,24 @@ LL(12): FMADD y08, a8, b7, y08 LFD a8, 8 * SIZE(AO8) - FMADD y09, a1, b8, y09 + FMADD y01, a1, b8, y01 LFD a1, 9 * SIZE(AO1) - FMADD y10, a2, b8, y10 + FMADD y02, a2, b8, y02 LFD a2, 9 * SIZE(AO2) - FMADD y11, a3, b8, y11 + FMADD y03, a3, b8, y03 LFD a3, 9 * SIZE(AO3) - FMADD y12, a4, b8, y12 + FMADD y04, a4, b8, y04 LFD a4, 9 * SIZE(AO4) - FMADD y13, a5, b8, y13 + FMADD y05, a5, b8, y05 LFD a5, 9 * SIZE(AO5) - FMADD y14, a6, b8, y14 + FMADD y06, a6, b8, y06 LFD a6, 9 * SIZE(AO6) - FMADD y15, a7, b8, y15 + FMADD y07, a7, b8, y07 LFD a7, 9 * SIZE(AO7) - FMADD y16, a8, b8, y16 + FMADD y08, a8, b8, y08 LFD a8, 9 * SIZE(AO8) LFD b5, 13 * SIZE(BO) @@ -640,24 +632,24 @@ LL(12): FMADD y08, a8, b1, y08 LFD a8, 10 * SIZE(AO8) - FMADD y09, a1, b2, y09 + FMADD y01, a1, b2, y01 LFD a1, 11 * SIZE(AO1) - FMADD y10, a2, b2, y10 + FMADD y02, a2, b2, y02 LFD a2, 11 * SIZE(AO2) - FMADD y11, a3, b2, y11 + FMADD y03, a3, b2, y03 LFD a3, 11 * SIZE(AO3) - FMADD y12, a4, b2, y12 + FMADD y04, a4, b2, y04 LFD a4, 11 * SIZE(AO4) - FMADD y13, a5, b2, y13 + FMADD y05, a5, b2, y05 LFD a5, 11 * SIZE(AO5) - FMADD y14, a6, b2, y14 + FMADD y06, a6, b2, y06 LFD a6, 11 * SIZE(AO6) - FMADD y15, a7, b2, y15 + FMADD y07, a7, b2, y07 LFD a7, 11 * SIZE(AO7) - FMADD y16, a8, b2, y16 + FMADD y08, a8, b2, y08 LFD a8, 11 * SIZE(AO8) FMADD y01, a1, b3, y01 @@ -680,24 +672,24 @@ LL(12): FMADD y08, a8, b3, y08 LFD a8, 12 * SIZE(AO8) - FMADD y09, a1, b4, y09 + FMADD y01, a1, b4, y01 LFD a1, 13 * SIZE(AO1) - FMADD y10, a2, b4, y10 + FMADD y02, a2, b4, y02 LFD a2, 13 * SIZE(AO2) - FMADD y11, a3, b4, y11 + FMADD y03, a3, b4, y03 LFD a3, 13 * SIZE(AO3) - FMADD y12, a4, b4, y12 + FMADD y04, a4, b4, y04 LFD a4, 13 * SIZE(AO4) - FMADD y13, a5, b4, y13 + FMADD y05, a5, b4, y05 LFD a5, 13 * SIZE(AO5) - FMADD y14, a6, b4, y14 + FMADD y06, a6, b4, y06 LFD a6, 13 * SIZE(AO6) - FMADD y15, a7, b4, y15 + FMADD y07, a7, b4, y07 LFD a7, 13 * SIZE(AO7) - FMADD y16, a8, b4, y16 + FMADD y08, a8, b4, y08 LFD a8, 13 * SIZE(AO8) LFD b1, 17 * SIZE(BO) @@ -725,24 +717,24 @@ LL(12): FMADD y08, a8, b5, y08 LFD a8, 14 * SIZE(AO8) - FMADD y09, a1, b6, y09 + FMADD y01, a1, b6, y01 LFD a1, 15 * SIZE(AO1) - FMADD y10, a2, b6, y10 + FMADD y02, a2, b6, y02 LFD a2, 15 * SIZE(AO2) - FMADD y11, a3, b6, y11 + FMADD y03, a3, b6, y03 LFD a3, 15 * SIZE(AO3) - FMADD y12, a4, b6, y12 + FMADD y04, a4, b6, y04 LFD a4, 15 * SIZE(AO4) - FMADD y13, a5, b6, y13 + FMADD y05, a5, b6, y05 LFD a5, 15 * SIZE(AO5) - FMADD y14, a6, b6, y14 + FMADD y06, a6, b6, y06 LFD a6, 15 * SIZE(AO6) - FMADD y15, a7, b6, y15 + FMADD y07, a7, b6, y07 LFD a7, 15 * SIZE(AO7) - FMADD y16, a8, b6, y16 + FMADD y08, a8, b6, y08 LFD a8, 15 * SIZE(AO8) FMADD y01, a1, b7, y01 @@ -765,14 +757,14 @@ LL(12): FMADD y08, a8, b7, y08 LFD a8, 16 * SIZE(AO8) - FMADD y09, a1, b8, y09 + FMADD y01, a1, b8, y01 LFD a1, 17 * SIZE(AO1) - FMADD y10, a2, b8, y10 + FMADD y02, a2, b8, y02 LFD a2, 17 * SIZE(AO2) - FMADD y11, a3, b8, y11 + FMADD y03, a3, b8, y03 LFD a3, 17 * SIZE(AO3) - FMADD y12, a4, b8, y12 + FMADD y04, a4, b8, y04 LFD a4, 17 * SIZE(AO4) addi AO1, AO1, 16 * SIZE @@ -780,14 +772,14 @@ LL(12): addi AO3, AO3, 16 * SIZE addi AO4, AO4, 16 * SIZE - FMADD y13, a5, b8, y13 + FMADD y05, a5, b8, y05 LFD a5, 17 * SIZE(AO5) - FMADD y14, a6, b8, y14 + FMADD y06, a6, b8, y06 LFD a6, 17 * SIZE(AO6) - FMADD y15, a7, b8, y15 + FMADD y07, a7, b8, y07 LFD a7, 17 * SIZE(AO7) - FMADD y16, a8, b8, y16 + FMADD y08, a8, b8, y08 LFD a8, 17 * SIZE(AO8) LFD b5, 21 * SIZE(BO) @@ -830,24 +822,24 @@ LL(13): FMADD y08, a8, b1, y08 LFD a8, 2 * SIZE(AO8) - FMADD y09, a1, b2, y09 + FMADD y01, a1, b2, y01 LFD a1, 3 * SIZE(AO1) - FMADD y10, a2, b2, y10 + FMADD y02, a2, b2, y02 LFD a2, 3 * SIZE(AO2) - FMADD y11, a3, b2, y11 + FMADD y03, a3, b2, y03 LFD a3, 3 * SIZE(AO3) - FMADD y12, a4, b2, y12 + FMADD y04, a4, b2, y04 LFD a4, 3 * SIZE(AO4) - FMADD y13, a5, b2, y13 + FMADD y05, a5, b2, y05 LFD a5, 3 * SIZE(AO5) - FMADD y14, a6, b2, y14 + FMADD y06, a6, b2, y06 LFD a6, 3 * SIZE(AO6) - FMADD y15, a7, b2, y15 + FMADD y07, a7, b2, y07 LFD a7, 3 * SIZE(AO7) - FMADD y16, a8, b2, y16 + FMADD y08, a8, b2, y08 LFD a8, 3 * SIZE(AO8) FMADD y01, a1, b3, y01 @@ -870,24 +862,24 @@ LL(13): FMADD y08, a8, b3, y08 LFD a8, 4 * SIZE(AO8) - FMADD y09, a1, b4, y09 + FMADD y01, a1, b4, y01 LFD a1, 5 * SIZE(AO1) - FMADD y10, a2, b4, y10 + FMADD y02, a2, b4, y02 LFD a2, 5 * SIZE(AO2) - FMADD y11, a3, b4, y11 + FMADD y03, a3, b4, y03 LFD a3, 5 * SIZE(AO3) - FMADD y12, a4, b4, y12 + FMADD y04, a4, b4, y04 LFD a4, 5 * SIZE(AO4) - FMADD y13, a5, b4, y13 + FMADD y05, a5, b4, y05 LFD a5, 5 * SIZE(AO5) - FMADD y14, a6, b4, y14 + FMADD y06, a6, b4, y06 LFD a6, 5 * SIZE(AO6) - FMADD y15, a7, b4, y15 + FMADD y07, a7, b4, y07 LFD a7, 5 * SIZE(AO7) - FMADD y16, a8, b4, y16 + FMADD y08, a8, b4, y08 LFD a8, 5 * SIZE(AO8) LFD b1, 9 * SIZE(BO) @@ -915,24 +907,24 @@ LL(13): FMADD y08, a8, b5, y08 LFD a8, 6 * SIZE(AO8) - FMADD y09, a1, b6, y09 + FMADD y01, a1, b6, y01 LFD a1, 7 * SIZE(AO1) - FMADD y10, a2, b6, y10 + FMADD y02, a2, b6, y02 LFD a2, 7 * SIZE(AO2) - FMADD y11, a3, b6, y11 + FMADD y03, a3, b6, y03 LFD a3, 7 * SIZE(AO3) - FMADD y12, a4, b6, y12 + FMADD y04, a4, b6, y04 LFD a4, 7 * SIZE(AO4) - FMADD y13, a5, b6, y13 + FMADD y05, a5, b6, y05 LFD a5, 7 * SIZE(AO5) - FMADD y14, a6, b6, y14 + FMADD y06, a6, b6, y06 LFD a6, 7 * SIZE(AO6) - FMADD y15, a7, b6, y15 + FMADD y07, a7, b6, y07 LFD a7, 7 * SIZE(AO7) - FMADD y16, a8, b6, y16 + FMADD y08, a8, b6, y08 LFD a8, 7 * SIZE(AO8) FMADD y01, a1, b7, y01 @@ -955,24 +947,24 @@ LL(13): FMADD y08, a8, b7, y08 LFD a8, 8 * SIZE(AO8) - FMADD y09, a1, b8, y09 + FMADD y01, a1, b8, y01 LFD a1, 9 * SIZE(AO1) - FMADD y10, a2, b8, y10 + FMADD y02, a2, b8, y02 LFD a2, 9 * SIZE(AO2) - FMADD y11, a3, b8, y11 + FMADD y03, a3, b8, y03 LFD a3, 9 * SIZE(AO3) - FMADD y12, a4, b8, y12 + FMADD y04, a4, b8, y04 LFD a4, 9 * SIZE(AO4) - FMADD y13, a5, b8, y13 + FMADD y05, a5, b8, y05 LFD a5, 9 * SIZE(AO5) - FMADD y14, a6, b8, y14 + FMADD y06, a6, b8, y06 LFD a6, 9 * SIZE(AO6) - FMADD y15, a7, b8, y15 + FMADD y07, a7, b8, y07 LFD a7, 9 * SIZE(AO7) - FMADD y16, a8, b8, y16 + FMADD y08, a8, b8, y08 LFD a8, 9 * SIZE(AO8) LFD b5, 13 * SIZE(BO) @@ -1000,24 +992,24 @@ LL(13): FMADD y08, a8, b1, y08 LFD a8, 10 * SIZE(AO8) - FMADD y09, a1, b2, y09 + FMADD y01, a1, b2, y01 LFD a1, 11 * SIZE(AO1) - FMADD y10, a2, b2, y10 + FMADD y02, a2, b2, y02 LFD a2, 11 * SIZE(AO2) - FMADD y11, a3, b2, y11 + FMADD y03, a3, b2, y03 LFD a3, 11 * SIZE(AO3) - FMADD y12, a4, b2, y12 + FMADD y04, a4, b2, y04 LFD a4, 11 * SIZE(AO4) - FMADD y13, a5, b2, y13 + FMADD y05, a5, b2, y05 LFD a5, 11 * SIZE(AO5) - FMADD y14, a6, b2, y14 + FMADD y06, a6, b2, y06 LFD a6, 11 * SIZE(AO6) - FMADD y15, a7, b2, y15 + FMADD y07, a7, b2, y07 LFD a7, 11 * SIZE(AO7) - FMADD y16, a8, b2, y16 + FMADD y08, a8, b2, y08 LFD a8, 11 * SIZE(AO8) FMADD y01, a1, b3, y01 @@ -1040,24 +1032,24 @@ LL(13): FMADD y08, a8, b3, y08 LFD a8, 12 * SIZE(AO8) - FMADD y09, a1, b4, y09 + FMADD y01, a1, b4, y01 LFD a1, 13 * SIZE(AO1) - FMADD y10, a2, b4, y10 + FMADD y02, a2, b4, y02 LFD a2, 13 * SIZE(AO2) - FMADD y11, a3, b4, y11 + FMADD y03, a3, b4, y03 LFD a3, 13 * SIZE(AO3) - FMADD y12, a4, b4, y12 + FMADD y04, a4, b4, y04 LFD a4, 13 * SIZE(AO4) - FMADD y13, a5, b4, y13 + FMADD y05, a5, b4, y05 LFD a5, 13 * SIZE(AO5) - FMADD y14, a6, b4, y14 + FMADD y06, a6, b4, y06 LFD a6, 13 * SIZE(AO6) - FMADD y15, a7, b4, y15 + FMADD y07, a7, b4, y07 LFD a7, 13 * SIZE(AO7) - FMADD y16, a8, b4, y16 + FMADD y08, a8, b4, y08 LFD a8, 13 * SIZE(AO8) FMADD y01, a1, b5, y01 @@ -1080,24 +1072,24 @@ LL(13): FMADD y08, a8, b5, y08 LFD a8, 14 * SIZE(AO8) - FMADD y09, a1, b6, y09 + FMADD y01, a1, b6, y01 LFD a1, 15 * SIZE(AO1) - FMADD y10, a2, b6, y10 + FMADD y02, a2, b6, y02 LFD a2, 15 * SIZE(AO2) - FMADD y11, a3, b6, y11 + FMADD y03, a3, b6, y03 LFD a3, 15 * SIZE(AO3) - FMADD y12, a4, b6, y12 + FMADD y04, a4, b6, y04 LFD a4, 15 * SIZE(AO4) - FMADD y13, a5, b6, y13 + FMADD y05, a5, b6, y05 LFD a5, 15 * SIZE(AO5) - FMADD y14, a6, b6, y14 + FMADD y06, a6, b6, y06 LFD a6, 15 * SIZE(AO6) - FMADD y15, a7, b6, y15 + FMADD y07, a7, b6, y07 LFD a7, 15 * SIZE(AO7) - FMADD y16, a8, b6, y16 + FMADD y08, a8, b6, y08 LFD a8, 15 * SIZE(AO8) FMADD y01, a1, b7, y01 @@ -1120,20 +1112,20 @@ LL(13): FMADD y08, a8, b7, y08 LFD a8, 16 * SIZE(AO8) - FMADD y09, a1, b8, y09 - FMADD y10, a2, b8, y10 - FMADD y11, a3, b8, y11 - FMADD y12, a4, b8, y12 + FMADD y01, a1, b8, y01 + FMADD y02, a2, b8, y02 + FMADD y03, a3, b8, y03 + FMADD y04, a4, b8, y04 addi AO1, AO1, 16 * SIZE addi AO2, AO2, 16 * SIZE addi AO3, AO3, 16 * SIZE addi AO4, AO4, 16 * SIZE - FMADD y13, a5, b8, y13 - FMADD y14, a6, b8, y14 - FMADD y15, a7, b8, y15 - FMADD y16, a8, b8, y16 + FMADD y05, a5, b8, y05 + FMADD y06, a6, b8, y06 + FMADD y07, a7, b8, y07 + FMADD y08, a8, b8, y08 addi AO5, AO5, 16 * SIZE addi AO6, AO6, 16 * SIZE @@ -1180,21 +1172,21 @@ LL(14): FMADD y08, a8, b1, y08 LFD a8, 2 * SIZE(AO8) - FMADD y09, a1, b2, y09 + FMADD y01, a1, b2, y01 LFD a1, 3 * SIZE(AO1) - FMADD y10, a2, b2, y10 + FMADD y02, a2, b2, y02 LFD a2, 3 * SIZE(AO2) - FMADD y11, a3, b2, y11 + FMADD y03, a3, b2, y03 LFD a3, 3 * SIZE(AO3) - FMADD y12, a4, b2, y12 + FMADD y04, a4, b2, y04 LFD a4, 3 * SIZE(AO4) - FMADD y13, a5, b2, y13 + FMADD y05, a5, b2, y05 LFD a5, 3 * SIZE(AO5) - FMADD y14, a6, b2, y14 + FMADD y06, a6, b2, y06 LFD a6, 3 * SIZE(AO6) - FMADD y15, a7, b2, y15 + FMADD y07, a7, b2, y07 LFD a7, 3 * SIZE(AO7) - FMADD y16, a8, b2, y16 + FMADD y08, a8, b2, y08 LFD a8, 3 * SIZE(AO8) LFD b5, 5 * SIZE(BO) @@ -1219,21 +1211,21 @@ LL(14): FMADD y08, a8, b3, y08 LFD a8, 4 * SIZE(AO8) - FMADD y09, a1, b4, y09 + FMADD y01, a1, b4, y01 LFD a1, 5 * SIZE(AO1) - FMADD y10, a2, b4, y10 + FMADD y02, a2, b4, y02 LFD a2, 5 * SIZE(AO2) - FMADD y11, a3, b4, y11 + FMADD y03, a3, b4, y03 LFD a3, 5 * SIZE(AO3) - FMADD y12, a4, b4, y12 + FMADD y04, a4, b4, y04 LFD a4, 5 * SIZE(AO4) - FMADD y13, a5, b4, y13 + FMADD y05, a5, b4, y05 LFD a5, 5 * SIZE(AO5) - FMADD y14, a6, b4, y14 + FMADD y06, a6, b4, y06 LFD a6, 5 * SIZE(AO6) - FMADD y15, a7, b4, y15 + FMADD y07, a7, b4, y07 LFD a7, 5 * SIZE(AO7) - FMADD y16, a8, b4, y16 + FMADD y08, a8, b4, y08 LFD a8, 5 * SIZE(AO8) FMADD y01, a1, b5, y01 @@ -1253,21 +1245,21 @@ LL(14): FMADD y08, a8, b5, y08 LFD a8, 6 * SIZE(AO8) - FMADD y09, a1, b6, y09 + FMADD y01, a1, b6, y01 LFD a1, 7 * SIZE(AO1) - FMADD y10, a2, b6, y10 + FMADD y02, a2, b6, y02 LFD a2, 7 * SIZE(AO2) - FMADD y11, a3, b6, y11 + FMADD y03, a3, b6, y03 LFD a3, 7 * SIZE(AO3) - FMADD y12, a4, b6, y12 + FMADD y04, a4, b6, y04 LFD a4, 7 * SIZE(AO4) - FMADD y13, a5, b6, y13 + FMADD y05, a5, b6, y05 LFD a5, 7 * SIZE(AO5) - FMADD y14, a6, b6, y14 + FMADD y06, a6, b6, y06 LFD a6, 7 * SIZE(AO6) - FMADD y15, a7, b6, y15 + FMADD y07, a7, b6, y07 LFD a7, 7 * SIZE(AO7) - FMADD y16, a8, b6, y16 + FMADD y08, a8, b6, y08 LFD a8, 7 * SIZE(AO8) FMADD y01, a1, b7, y01 @@ -1287,21 +1279,21 @@ LL(14): FMADD y08, a8, b7, y08 LFD a8, 8 * SIZE(AO8) - FMADD y09, a1, b8, y09 + FMADD y01, a1, b8, y01 addi AO1, AO1, 8 * SIZE - FMADD y10, a2, b8, y10 + FMADD y02, a2, b8, y02 addi AO2, AO2, 8 * SIZE - FMADD y11, a3, b8, y11 + FMADD y03, a3, b8, y03 addi AO3, AO3, 8 * SIZE - FMADD y12, a4, b8, y12 + FMADD y04, a4, b8, y04 addi AO4, AO4, 8 * SIZE - FMADD y13, a5, b8, y13 + FMADD y05, a5, b8, y05 addi AO5, AO5, 8 * SIZE - FMADD y14, a6, b8, y14 + FMADD y06, a6, b8, y06 addi AO6, AO6, 8 * SIZE - FMADD y15, a7, b8, y15 + FMADD y07, a7, b8, y07 addi AO7, AO7, 8 * SIZE - FMADD y16, a8, b8, y16 + FMADD y08, a8, b8, y08 addi AO8, AO8, 8 * SIZE addi BO, BO, 8 * SIZE .align 4 @@ -1341,21 +1333,21 @@ LL(15): FMADD y08, a8, b1, y08 LFD a8, 2 * SIZE(AO8) - FMADD y09, a1, b2, y09 + FMADD y01, a1, b2, y01 LFD a1, 3 * SIZE(AO1) - FMADD y10, a2, b2, y10 + FMADD y02, a2, b2, y02 LFD a2, 3 * SIZE(AO2) - FMADD y11, a3, b2, y11 + FMADD y03, a3, b2, y03 LFD a3, 3 * SIZE(AO3) - FMADD y12, a4, b2, y12 + FMADD y04, a4, b2, y04 LFD a4, 3 * SIZE(AO4) - FMADD y13, a5, b2, y13 + FMADD y05, a5, b2, y05 LFD a5, 3 * SIZE(AO5) - FMADD y14, a6, b2, y14 + FMADD y06, a6, b2, y06 LFD a6, 3 * SIZE(AO6) - FMADD y15, a7, b2, y15 + FMADD y07, a7, b2, y07 LFD a7, 3 * SIZE(AO7) - FMADD y16, a8, b2, y16 + FMADD y08, a8, b2, y08 LFD a8, 3 * SIZE(AO8) FMADD y01, a1, b3, y01 @@ -1376,21 +1368,21 @@ LL(15): FMADD y08, a8, b3, y08 LFD a8, 4 * SIZE(AO8) - FMADD y09, a1, b4, y09 + FMADD y01, a1, b4, y01 addi AO1, AO1, 4 * SIZE - FMADD y10, a2, b4, y10 + FMADD y02, a2, b4, y02 addi AO2, AO2, 4 * SIZE - FMADD y11, a3, b4, y11 + FMADD y03, a3, b4, y03 addi AO3, AO3, 4 * SIZE - FMADD y12, a4, b4, y12 + FMADD y04, a4, b4, y04 addi AO4, AO4, 4 * SIZE - FMADD y13, a5, b4, y13 + FMADD y05, a5, b4, y05 addi AO5, AO5, 4 * SIZE - FMADD y14, a6, b4, y14 + FMADD y06, a6, b4, y06 addi AO6, AO6, 4 * SIZE - FMADD y15, a7, b4, y15 + FMADD y07, a7, b4, y07 addi AO7, AO7, 4 * SIZE - FMADD y16, a8, b4, y16 + FMADD y08, a8, b4, y08 addi AO8, AO8, 4 * SIZE addi BO, BO, 4 * SIZE .align 4 @@ -1428,22 +1420,22 @@ LL(16): FMADD y08, a8, b1, y08 LFD a8, 2 * SIZE(AO8) - FMADD y09, a1, b2, y09 + FMADD y01, a1, b2, y01 addi AO1, AO1, 2 * SIZE addi AO2, AO2, 2 * SIZE - FMADD y10, a2, b2, y10 + FMADD y02, a2, b2, y02 addi AO3, AO3, 2 * SIZE addi AO4, AO4, 2 * SIZE - FMADD y11, a3, b2, y11 - FMADD y12, a4, b2, y12 + FMADD y03, a3, b2, y03 + FMADD y04, a4, b2, y04 addi AO5, AO5, 2 * SIZE addi AO6, AO6, 2 * SIZE - FMADD y13, a5, b2, y13 - FMADD y14, a6, b2, y14 + FMADD y05, a5, b2, y05 + FMADD y06, a6, b2, y06 addi AO7, AO7, 2 * SIZE addi AO8, AO8, 2 * SIZE - FMADD y15, a7, b2, y15 - FMADD y16, a8, b2, y16 + FMADD y07, a7, b2, y07 + FMADD y08, a8, b2, y08 addi BO, BO, 2 * SIZE .align 4 @@ -1486,15 +1478,6 @@ LL(18): LFD a7, 7 * SIZE(CO) LFD a8, 8 * SIZE(CO) - FADD y01, y09, y01 - FADD y02, y10, y02 - FADD y03, y11, y03 - FADD y04, y12, y04 - FADD y05, y13, y05 - FADD y06, y14, y06 - FADD y07, y15, y07 - FADD y08, y16, y08 - FMADD a1, alpha, y01, a1 FMADD a2, alpha, y02, a2 FMADD a3, alpha, y03, a3 @@ -1530,15 +1513,6 @@ LL(19): LFDUX a7, CO, INCY LFDUX a8, CO, INCY - FADD y01, y09, y01 - FADD y02, y10, y02 - FADD y03, y11, y03 - FADD y04, y12, y04 - FADD y05, y13, y05 - FADD y06, y14, y06 - FADD y07, y15, y07 - FADD y08, y16, y08 - FMADD a1, alpha, f0, a1 FMADD a2, alpha, f1, a2 FMADD a3, alpha, f2, a3 @@ -1580,10 +1554,6 @@ LL(20): fmr y02, y01 fmr y03, y01 fmr y04, y01 - fmr y09, y01 - fmr y10, y01 - fmr y11, y01 - fmr y12, y01 DCBT(Y1, PREC) @@ -1621,13 +1591,13 @@ LL(22): FMADD y04, a4, b1, y04 LFD a4, 3 * SIZE(AO4) - FMADD y09, a5, b2, y09 + FMADD y01, a5, b2, y01 LFD a5, 4 * SIZE(AO1) - FMADD y10, a6, b2, y10 + FMADD y02, a6, b2, y02 LFD a6, 4 * SIZE(AO2) - FMADD y11, a7, b2, y11 + FMADD y03, a7, b2, y03 LFD a7, 4 * SIZE(AO3) - FMADD y12, a8, b2, y12 + FMADD y04, a8, b2, y04 LFD a8, 4 * SIZE(AO4) FMADD y01, a1, b3, y01 @@ -1639,13 +1609,13 @@ LL(22): FMADD y04, a4, b3, y04 LFD a4, 5 * SIZE(AO4) - FMADD y09, a5, b4, y09 + FMADD y01, a5, b4, y01 LFD a5, 6 * SIZE(AO1) - FMADD y10, a6, b4, y10 + FMADD y02, a6, b4, y02 LFD a6, 6 * SIZE(AO2) - FMADD y11, a7, b4, y11 + FMADD y03, a7, b4, y03 LFD a7, 6 * SIZE(AO3) - FMADD y12, a8, b4, y12 + FMADD y04, a8, b4, y04 LFD a8, 6 * SIZE(AO4) LFD b1, 9 * SIZE(BO) @@ -1662,13 +1632,13 @@ LL(22): FMADD y04, a4, b5, y04 LFD a4, 7 * SIZE(AO4) - FMADD y09, a5, b6, y09 + FMADD y01, a5, b6, y01 LFD a5, 8 * SIZE(AO1) - FMADD y10, a6, b6, y10 + FMADD y02, a6, b6, y02 LFD a6, 8 * SIZE(AO2) - FMADD y11, a7, b6, y11 + FMADD y03, a7, b6, y03 LFD a7, 8 * SIZE(AO3) - FMADD y12, a8, b6, y12 + FMADD y04, a8, b6, y04 LFD a8, 8 * SIZE(AO4) FMADD y01, a1, b7, y01 @@ -1680,13 +1650,13 @@ LL(22): FMADD y04, a4, b7, y04 LFD a4, 9 * SIZE(AO4) - FMADD y09, a5, b8, y09 + FMADD y01, a5, b8, y01 LFD a5, 10 * SIZE(AO1) - FMADD y10, a6, b8, y10 + FMADD y02, a6, b8, y02 LFD a6, 10 * SIZE(AO2) - FMADD y11, a7, b8, y11 + FMADD y03, a7, b8, y03 LFD a7, 10 * SIZE(AO3) - FMADD y12, a8, b8, y12 + FMADD y04, a8, b8, y04 LFD a8, 10 * SIZE(AO4) LFD b5, 13 * SIZE(BO) @@ -1703,13 +1673,13 @@ LL(22): FMADD y04, a4, b1, y04 LFD a4, 11 * SIZE(AO4) - FMADD y09, a5, b2, y09 + FMADD y01, a5, b2, y01 LFD a5, 12 * SIZE(AO1) - FMADD y10, a6, b2, y10 + FMADD y02, a6, b2, y02 LFD a6, 12 * SIZE(AO2) - FMADD y11, a7, b2, y11 + FMADD y03, a7, b2, y03 LFD a7, 12 * SIZE(AO3) - FMADD y12, a8, b2, y12 + FMADD y04, a8, b2, y04 LFD a8, 12 * SIZE(AO4) FMADD y01, a1, b3, y01 @@ -1721,13 +1691,13 @@ LL(22): FMADD y04, a4, b3, y04 LFD a4, 13 * SIZE(AO4) - FMADD y09, a5, b4, y09 + FMADD y01, a5, b4, y01 LFD a5, 14 * SIZE(AO1) - FMADD y10, a6, b4, y10 + FMADD y02, a6, b4, y02 LFD a6, 14 * SIZE(AO2) - FMADD y11, a7, b4, y11 + FMADD y03, a7, b4, y03 LFD a7, 14 * SIZE(AO3) - FMADD y12, a8, b4, y12 + FMADD y04, a8, b4, y04 LFD a8, 14 * SIZE(AO4) LFD b1, 17 * SIZE(BO) @@ -1744,13 +1714,13 @@ LL(22): FMADD y04, a4, b5, y04 LFD a4, 15 * SIZE(AO4) - FMADD y09, a5, b6, y09 + FMADD y01, a5, b6, y01 LFD a5, 16 * SIZE(AO1) - FMADD y10, a6, b6, y10 + FMADD y02, a6, b6, y02 LFD a6, 16 * SIZE(AO2) - FMADD y11, a7, b6, y11 + FMADD y03, a7, b6, y03 LFD a7, 16 * SIZE(AO3) - FMADD y12, a8, b6, y12 + FMADD y04, a8, b6, y04 LFD a8, 16 * SIZE(AO4) FMADD y01, a1, b7, y01 @@ -1762,13 +1732,13 @@ LL(22): FMADD y04, a4, b7, y04 LFD a4, 17 * SIZE(AO4) - FMADD y09, a5, b8, y09 + FMADD y01, a5, b8, y01 LFD a5, 18 * SIZE(AO1) - FMADD y10, a6, b8, y10 + FMADD y02, a6, b8, y02 LFD a6, 18 * SIZE(AO2) - FMADD y11, a7, b8, y11 + FMADD y03, a7, b8, y03 LFD a7, 18 * SIZE(AO3) - FMADD y12, a8, b8, y12 + FMADD y04, a8, b8, y04 LFD a8, 18 * SIZE(AO4) LFD b5, 21 * SIZE(BO) @@ -1800,13 +1770,13 @@ LL(23): FMADD y04, a4, b1, y04 LFD a4, 3 * SIZE(AO4) - FMADD y09, a5, b2, y09 + FMADD y01, a5, b2, y01 LFD a5, 4 * SIZE(AO1) - FMADD y10, a6, b2, y10 + FMADD y02, a6, b2, y02 LFD a6, 4 * SIZE(AO2) - FMADD y11, a7, b2, y11 + FMADD y03, a7, b2, y03 LFD a7, 4 * SIZE(AO3) - FMADD y12, a8, b2, y12 + FMADD y04, a8, b2, y04 LFD a8, 4 * SIZE(AO4) FMADD y01, a1, b3, y01 @@ -1818,13 +1788,13 @@ LL(23): FMADD y04, a4, b3, y04 LFD a4, 5 * SIZE(AO4) - FMADD y09, a5, b4, y09 + FMADD y01, a5, b4, y01 LFD a5, 6 * SIZE(AO1) - FMADD y10, a6, b4, y10 + FMADD y02, a6, b4, y02 LFD a6, 6 * SIZE(AO2) - FMADD y11, a7, b4, y11 + FMADD y03, a7, b4, y03 LFD a7, 6 * SIZE(AO3) - FMADD y12, a8, b4, y12 + FMADD y04, a8, b4, y04 LFD a8, 6 * SIZE(AO4) LFD b1, 9 * SIZE(BO) @@ -1841,13 +1811,13 @@ LL(23): FMADD y04, a4, b5, y04 LFD a4, 7 * SIZE(AO4) - FMADD y09, a5, b6, y09 + FMADD y01, a5, b6, y01 LFD a5, 8 * SIZE(AO1) - FMADD y10, a6, b6, y10 + FMADD y02, a6, b6, y02 LFD a6, 8 * SIZE(AO2) - FMADD y11, a7, b6, y11 + FMADD y03, a7, b6, y03 LFD a7, 8 * SIZE(AO3) - FMADD y12, a8, b6, y12 + FMADD y04, a8, b6, y04 LFD a8, 8 * SIZE(AO4) FMADD y01, a1, b7, y01 @@ -1859,13 +1829,13 @@ LL(23): FMADD y04, a4, b7, y04 LFD a4, 9 * SIZE(AO4) - FMADD y09, a5, b8, y09 + FMADD y01, a5, b8, y01 LFD a5, 10 * SIZE(AO1) - FMADD y10, a6, b8, y10 + FMADD y02, a6, b8, y02 LFD a6, 10 * SIZE(AO2) - FMADD y11, a7, b8, y11 + FMADD y03, a7, b8, y03 LFD a7, 10 * SIZE(AO3) - FMADD y12, a8, b8, y12 + FMADD y04, a8, b8, y04 LFD a8, 10 * SIZE(AO4) LFD b5, 13 * SIZE(BO) @@ -1882,13 +1852,13 @@ LL(23): FMADD y04, a4, b1, y04 LFD a4, 11 * SIZE(AO4) - FMADD y09, a5, b2, y09 + FMADD y01, a5, b2, y01 LFD a5, 12 * SIZE(AO1) - FMADD y10, a6, b2, y10 + FMADD y02, a6, b2, y02 LFD a6, 12 * SIZE(AO2) - FMADD y11, a7, b2, y11 + FMADD y03, a7, b2, y03 LFD a7, 12 * SIZE(AO3) - FMADD y12, a8, b2, y12 + FMADD y04, a8, b2, y04 LFD a8, 12 * SIZE(AO4) FMADD y01, a1, b3, y01 @@ -1900,13 +1870,13 @@ LL(23): FMADD y04, a4, b3, y04 LFD a4, 13 * SIZE(AO4) - FMADD y09, a5, b4, y09 + FMADD y01, a5, b4, y01 LFD a5, 14 * SIZE(AO1) - FMADD y10, a6, b4, y10 + FMADD y02, a6, b4, y02 LFD a6, 14 * SIZE(AO2) - FMADD y11, a7, b4, y11 + FMADD y03, a7, b4, y03 LFD a7, 14 * SIZE(AO3) - FMADD y12, a8, b4, y12 + FMADD y04, a8, b4, y04 LFD a8, 14 * SIZE(AO4) FMADD y01, a1, b5, y01 @@ -1918,13 +1888,13 @@ LL(23): FMADD y04, a4, b5, y04 LFD a4, 15 * SIZE(AO4) - FMADD y09, a5, b6, y09 + FMADD y01, a5, b6, y01 LFD a5, 16 * SIZE(AO1) - FMADD y10, a6, b6, y10 + FMADD y02, a6, b6, y02 LFD a6, 16 * SIZE(AO2) - FMADD y11, a7, b6, y11 + FMADD y03, a7, b6, y03 LFD a7, 16 * SIZE(AO3) - FMADD y12, a8, b6, y12 + FMADD y04, a8, b6, y04 LFD a8, 16 * SIZE(AO4) FMADD y01, a1, b7, y01 @@ -1932,10 +1902,10 @@ LL(23): FMADD y03, a3, b7, y03 FMADD y04, a4, b7, y04 - FMADD y09, a5, b8, y09 - FMADD y10, a6, b8, y10 - FMADD y11, a7, b8, y11 - FMADD y12, a8, b8, y12 + FMADD y01, a5, b8, y01 + FMADD y02, a6, b8, y02 + FMADD y03, a7, b8, y03 + FMADD y04, a8, b8, y04 addi AO1, AO1, 16 * SIZE addi AO2, AO2, 16 * SIZE @@ -1975,13 +1945,13 @@ LL(24): FMADD y04, a4, b1, y04 LFD a4, 3 * SIZE(AO4) - FMADD y09, a5, b2, y09 + FMADD y01, a5, b2, y01 LFD a5, 4 * SIZE(AO1) - FMADD y10, a6, b2, y10 + FMADD y02, a6, b2, y02 LFD a6, 4 * SIZE(AO2) - FMADD y11, a7, b2, y11 + FMADD y03, a7, b2, y03 LFD a7, 4 * SIZE(AO3) - FMADD y12, a8, b2, y12 + FMADD y04, a8, b2, y04 LFD a8, 4 * SIZE(AO4) FMADD y01, a1, b3, y01 @@ -1993,13 +1963,13 @@ LL(24): FMADD y04, a4, b3, y04 LFD a4, 5 * SIZE(AO4) - FMADD y09, a5, b4, y09 + FMADD y01, a5, b4, y01 LFD a5, 6 * SIZE(AO1) - FMADD y10, a6, b4, y10 + FMADD y02, a6, b4, y02 LFD a6, 6 * SIZE(AO2) - FMADD y11, a7, b4, y11 + FMADD y03, a7, b4, y03 LFD a7, 6 * SIZE(AO3) - FMADD y12, a8, b4, y12 + FMADD y04, a8, b4, y04 LFD a8, 6 * SIZE(AO4) LFD b1, 5 * SIZE(BO) @@ -2016,13 +1986,13 @@ LL(24): FMADD y04, a4, b1, y04 LFD a4, 7 * SIZE(AO4) - FMADD y09, a5, b2, y09 + FMADD y01, a5, b2, y01 LFD a5, 8 * SIZE(AO1) - FMADD y10, a6, b2, y10 + FMADD y02, a6, b2, y02 LFD a6, 8 * SIZE(AO2) - FMADD y11, a7, b2, y11 + FMADD y03, a7, b2, y03 LFD a7, 8 * SIZE(AO3) - FMADD y12, a8, b2, y12 + FMADD y04, a8, b2, y04 LFD a8, 8 * SIZE(AO4) FMADD y01, a1, b3, y01 @@ -2030,13 +2000,13 @@ LL(24): FMADD y03, a3, b3, y03 FMADD y04, a4, b3, y04 - FMADD y09, a5, b4, y09 + FMADD y01, a5, b4, y01 addi AO1, AO1, 8 * SIZE - FMADD y10, a6, b4, y10 + FMADD y02, a6, b4, y02 addi AO2, AO2, 8 * SIZE - FMADD y11, a7, b4, y11 + FMADD y03, a7, b4, y03 addi AO3, AO3, 8 * SIZE - FMADD y12, a8, b4, y12 + FMADD y04, a8, b4, y04 addi AO4, AO4, 8 * SIZE addi BO, BO, 8 * SIZE @@ -2070,13 +2040,13 @@ LL(25): FMADD y04, a4, b1, y04 LFD a4, 3 * SIZE(AO4) - FMADD y09, a5, b2, y09 + FMADD y01, a5, b2, y01 LFD a5, 4 * SIZE(AO1) - FMADD y10, a6, b2, y10 + FMADD y02, a6, b2, y02 LFD a6, 4 * SIZE(AO2) - FMADD y11, a7, b2, y11 + FMADD y03, a7, b2, y03 LFD a7, 4 * SIZE(AO3) - FMADD y12, a8, b2, y12 + FMADD y04, a8, b2, y04 LFD a8, 4 * SIZE(AO4) FMADD y01, a1, b3, y01 @@ -2084,13 +2054,13 @@ LL(25): FMADD y03, a3, b3, y03 FMADD y04, a4, b3, y04 - FMADD y09, a5, b4, y09 + FMADD y01, a5, b4, y01 addi AO1, AO1, 4 * SIZE - FMADD y10, a6, b4, y10 + FMADD y02, a6, b4, y02 addi AO2, AO2, 4 * SIZE - FMADD y11, a7, b4, y11 + FMADD y03, a7, b4, y03 addi AO3, AO3, 4 * SIZE - FMADD y12, a8, b4, y12 + FMADD y04, a8, b4, y04 addi AO4, AO4, 4 * SIZE addi BO, BO, 4 * SIZE .align 4 @@ -2117,13 +2087,13 @@ LL(26): FMADD y03, a3, b1, y03 FMADD y04, a4, b1, y04 - FMADD y09, a5, b2, y09 + FMADD y01, a5, b2, y01 addi AO1, AO1, 2 * SIZE - FMADD y10, a6, b2, y10 + FMADD y02, a6, b2, y02 addi AO2, AO2, 2 * SIZE - FMADD y11, a7, b2, y11 + FMADD y03, a7, b2, y03 addi AO3, AO3, 2 * SIZE - FMADD y12, a8, b2, y12 + FMADD y04, a8, b2, y04 addi AO4, AO4, 2 * SIZE addi BO, BO, 2 * SIZE .align 4 @@ -2156,11 +2126,6 @@ LL(28): LFD a3, 3 * SIZE(CO) LFD a4, 4 * SIZE(CO) - FADD y01, y09, y01 - FADD y02, y10, y02 - FADD y03, y11, y03 - FADD y04, y12, y04 - FMADD a1, alpha, y01, a1 FMADD a2, alpha, y02, a2 FMADD a3, alpha, y03, a3 @@ -2181,11 +2146,6 @@ LL(29): LFDUX a3, CO, INCY LFDUX a4, CO, INCY - FADD y01, y09, y01 - FADD y02, y10, y02 - FADD y03, y11, y03 - FADD y04, y12, y04 - FMADD a1, alpha, f0, a1 FMADD a2, alpha, f1, a2 FMADD a3, alpha, f2, a3 @@ -2209,12 +2169,6 @@ LL(30): lfd y01, FZERO fmr y02, y01 - fmr y03, y01 - fmr y04, y01 - fmr y09, y01 - fmr y10, y01 - fmr y11, y01 - fmr y12, y01 DCBT(Y1, PREC) @@ -2247,18 +2201,18 @@ LL(32): LFD a1, 5 * SIZE(AO1) FMADD y02, a2, b1, y02 LFD a2, 5 * SIZE(AO2) - FMADD y03, a3, b2, y03 + FMADD y01, a3, b2, y01 LFD a3, 6 * SIZE(AO1) - FMADD y04, a4, b2, y04 + FMADD y02, a4, b2, y02 LFD a4, 6 * SIZE(AO2) - FMADD y09, a5, b3, y09 + FMADD y01, a5, b3, y01 LFD a5, 7 * SIZE(AO1) - FMADD y10, a6, b3, y10 + FMADD y02, a6, b3, y02 LFD a6, 7 * SIZE(AO2) - FMADD y11, a7, b4, y11 + FMADD y01, a7, b4, y01 LFD a7, 8 * SIZE(AO1) - FMADD y12, a8, b4, y12 + FMADD y02, a8, b4, y02 LFD a8, 8 * SIZE(AO2) LFD b1, 9 * SIZE(BO) @@ -2270,18 +2224,18 @@ LL(32): LFD a1, 9 * SIZE(AO1) FMADD y02, a2, b5, y02 LFD a2, 9 * SIZE(AO2) - FMADD y03, a3, b6, y03 + FMADD y01, a3, b6, y01 LFD a3, 10 * SIZE(AO1) - FMADD y04, a4, b6, y04 + FMADD y02, a4, b6, y02 LFD a4, 10 * SIZE(AO2) - FMADD y09, a5, b7, y09 + FMADD y01, a5, b7, y01 LFD a5, 11 * SIZE(AO1) - FMADD y10, a6, b7, y10 + FMADD y02, a6, b7, y02 LFD a6, 11 * SIZE(AO2) - FMADD y11, a7, b8, y11 + FMADD y01, a7, b8, y01 LFD a7, 12 * SIZE(AO1) - FMADD y12, a8, b8, y12 + FMADD y02, a8, b8, y02 LFD a8, 12 * SIZE(AO2) LFD b5, 13 * SIZE(BO) @@ -2293,18 +2247,18 @@ LL(32): LFD a1, 13 * SIZE(AO1) FMADD y02, a2, b1, y02 LFD a2, 13 * SIZE(AO2) - FMADD y03, a3, b2, y03 + FMADD y01, a3, b2, y01 LFD a3, 14 * SIZE(AO1) - FMADD y04, a4, b2, y04 + FMADD y02, a4, b2, y02 LFD a4, 14 * SIZE(AO2) - FMADD y09, a5, b3, y09 + FMADD y01, a5, b3, y01 LFD a5, 15 * SIZE(AO1) - FMADD y10, a6, b3, y10 + FMADD y02, a6, b3, y02 LFD a6, 15 * SIZE(AO2) - FMADD y11, a7, b4, y11 + FMADD y01, a7, b4, y01 LFD a7, 16 * SIZE(AO1) - FMADD y12, a8, b4, y12 + FMADD y02, a8, b4, y02 LFD a8, 16 * SIZE(AO2) LFD b1, 17 * SIZE(BO) @@ -2316,18 +2270,18 @@ LL(32): LFD a1, 17 * SIZE(AO1) FMADD y02, a2, b5, y02 LFD a2, 17 * SIZE(AO2) - FMADD y03, a3, b6, y03 + FMADD y01, a3, b6, y01 LFD a3, 18 * SIZE(AO1) - FMADD y04, a4, b6, y04 + FMADD y02, a4, b6, y02 LFD a4, 18 * SIZE(AO2) - FMADD y09, a5, b7, y09 + FMADD y01, a5, b7, y01 LFD a5, 19 * SIZE(AO1) - FMADD y10, a6, b7, y10 + FMADD y02, a6, b7, y02 LFD a6, 19 * SIZE(AO2) - FMADD y11, a7, b8, y11 + FMADD y01, a7, b8, y01 LFD a7, 20 * SIZE(AO1) - FMADD y12, a8, b8, y12 + FMADD y02, a8, b8, y02 LFD a8, 20 * SIZE(AO2) LFD b5, 21 * SIZE(BO) @@ -2349,18 +2303,18 @@ LL(33): LFD a1, 5 * SIZE(AO1) FMADD y02, a2, b1, y02 LFD a2, 5 * SIZE(AO2) - FMADD y03, a3, b2, y03 + FMADD y01, a3, b2, y01 LFD a3, 6 * SIZE(AO1) - FMADD y04, a4, b2, y04 + FMADD y02, a4, b2, y02 LFD a4, 6 * SIZE(AO2) - FMADD y09, a5, b3, y09 + FMADD y01, a5, b3, y01 LFD a5, 7 * SIZE(AO1) - FMADD y10, a6, b3, y10 + FMADD y02, a6, b3, y02 LFD a6, 7 * SIZE(AO2) - FMADD y11, a7, b4, y11 + FMADD y01, a7, b4, y01 LFD a7, 8 * SIZE(AO1) - FMADD y12, a8, b4, y12 + FMADD y02, a8, b4, y02 LFD a8, 8 * SIZE(AO2) LFD b1, 9 * SIZE(BO) @@ -2372,18 +2326,18 @@ LL(33): LFD a1, 9 * SIZE(AO1) FMADD y02, a2, b5, y02 LFD a2, 9 * SIZE(AO2) - FMADD y03, a3, b6, y03 + FMADD y01, a3, b6, y01 LFD a3, 10 * SIZE(AO1) - FMADD y04, a4, b6, y04 + FMADD y02, a4, b6, y02 LFD a4, 10 * SIZE(AO2) - FMADD y09, a5, b7, y09 + FMADD y01, a5, b7, y01 LFD a5, 11 * SIZE(AO1) - FMADD y10, a6, b7, y10 + FMADD y02, a6, b7, y02 LFD a6, 11 * SIZE(AO2) - FMADD y11, a7, b8, y11 + FMADD y01, a7, b8, y01 LFD a7, 12 * SIZE(AO1) - FMADD y12, a8, b8, y12 + FMADD y02, a8, b8, y02 LFD a8, 12 * SIZE(AO2) LFD b5, 13 * SIZE(BO) @@ -2395,29 +2349,29 @@ LL(33): LFD a1, 13 * SIZE(AO1) FMADD y02, a2, b1, y02 LFD a2, 13 * SIZE(AO2) - FMADD y03, a3, b2, y03 + FMADD y01, a3, b2, y01 LFD a3, 14 * SIZE(AO1) - FMADD y04, a4, b2, y04 + FMADD y02, a4, b2, y02 LFD a4, 14 * SIZE(AO2) - FMADD y09, a5, b3, y09 + FMADD y01, a5, b3, y01 LFD a5, 15 * SIZE(AO1) - FMADD y10, a6, b3, y10 + FMADD y02, a6, b3, y02 LFD a6, 15 * SIZE(AO2) - FMADD y11, a7, b4, y11 + FMADD y01, a7, b4, y01 LFD a7, 16 * SIZE(AO1) - FMADD y12, a8, b4, y12 + FMADD y02, a8, b4, y02 LFD a8, 16 * SIZE(AO2) FMADD y01, a1, b5, y01 FMADD y02, a2, b5, y02 - FMADD y03, a3, b6, y03 - FMADD y04, a4, b6, y04 + FMADD y01, a3, b6, y01 + FMADD y02, a4, b6, y02 - FMADD y09, a5, b7, y09 - FMADD y10, a6, b7, y10 - FMADD y11, a7, b8, y11 - FMADD y12, a8, b8, y12 + FMADD y01, a5, b7, y01 + FMADD y02, a6, b7, y02 + FMADD y01, a7, b8, y01 + FMADD y02, a8, b8, y02 addi AO1, AO1, 16 * SIZE addi AO2, AO2, 16 * SIZE @@ -2454,32 +2408,32 @@ LL(34): LFD a1, 5 * SIZE(AO1) FMADD y02, a2, b1, y02 LFD a2, 5 * SIZE(AO2) - FMADD y09, a3, b2, y09 + FMADD y01, a3, b2, y01 LFD a3, 6 * SIZE(AO1) - FMADD y10, a4, b2, y10 + FMADD y02, a4, b2, y02 LFD a4, 6 * SIZE(AO2) FMADD y01, a5, b3, y01 LFD a5, 7 * SIZE(AO1) FMADD y02, a6, b3, y02 LFD a6, 7 * SIZE(AO2) - FMADD y09, a7, b4, y09 + FMADD y01, a7, b4, y01 LFD a7, 8 * SIZE(AO1) - FMADD y10, a8, b4, y10 + FMADD y02, a8, b4, y02 LFD a8, 8 * SIZE(AO2) FMADD y01, a1, b5, y01 FMADD y02, a2, b5, y02 - FMADD y09, a3, b6, y09 - FMADD y10, a4, b6, y10 + FMADD y01, a3, b6, y01 + FMADD y02, a4, b6, y02 FMADD y01, a5, b7, y01 addi AO1, AO1, 8 * SIZE FMADD y02, a6, b7, y02 addi AO2, AO2, 8 * SIZE - FMADD y09, a7, b8, y09 + FMADD y01, a7, b8, y01 addi BO, BO, 8 * SIZE - FMADD y10, a8, b8, y10 + FMADD y02, a8, b8, y02 nop .align 4 @@ -2504,17 +2458,17 @@ LL(35): FMADD y01, a1, b1, y01 FMADD y02, a2, b1, y02 - FMADD y09, a3, b2, y09 - FMADD y10, a4, b2, y10 + FMADD y01, a3, b2, y01 + FMADD y02, a4, b2, y02 FMADD y01, a5, b3, y01 addi AO1, AO1, 4 * SIZE FMADD y02, a6, b3, y02 addi AO2, AO2, 4 * SIZE - FMADD y09, a7, b4, y09 + FMADD y01, a7, b4, y01 addi BO, BO, 4 * SIZE - FMADD y10, a8, b4, y10 + FMADD y02, a8, b4, y02 .align 4 LL(36): @@ -2531,8 +2485,8 @@ LL(36): FMADD y01, a1, b1, y01 FMADD y02, a2, b1, y02 - FMADD y09, a3, b2, y09 - FMADD y10, a4, b2, y10 + FMADD y01, a3, b2, y01 + FMADD y02, a4, b2, y02 addi AO1, AO1, 2 * SIZE addi AO2, AO2, 2 * SIZE @@ -2560,14 +2514,6 @@ LL(38): LFD a1, 1 * SIZE(CO) LFD a2, 2 * SIZE(CO) - FADD y01, y03, y01 - FADD y02, y04, y02 - FADD y09, y11, y09 - FADD y10, y12, y10 - - FADD y01, y09, y01 - FADD y02, y10, y02 - FMADD a1, alpha, y01, a1 FMADD a2, alpha, y02, a2 @@ -2582,14 +2528,6 @@ LL(39): LFDUX a1, CO, INCY LFDUX a2, CO, INCY - FADD y01, y03, y01 - FADD y02, y04, y02 - FADD y09, y11, y09 - FADD y10, y12, y10 - - FADD y01, y09, y01 - FADD y02, y10, y02 - FMADD a1, alpha, f0, a1 FMADD a2, alpha, f1, a2 @@ -2606,13 +2544,6 @@ LL(40): mr BO, XP lfd y01, FZERO - fmr y02, y01 - fmr y03, y01 - fmr y04, y01 - fmr y09, y01 - fmr y10, y01 - fmr y11, y01 - fmr y12, y01 DCBT(Y1, PREC) @@ -2646,17 +2577,17 @@ LL(42): LFD a1, 9 * SIZE(AO1) LFD b1, 9 * SIZE(BO) - FMADD y02, a2, b2, y02 + FMADD y01, a2, b2, y01 nop LFD a2, 10 * SIZE(AO1) LFD b2, 10 * SIZE(BO) - FMADD y03, a3, b3, y03 + FMADD y01, a3, b3, y01 nop LFD a3, 11 * SIZE(AO1) LFD b3, 11 * SIZE(BO) - FMADD y04, a4, b4, y04 + FMADD y01, a4, b4, y01 nop LFD a4, 12 * SIZE(AO1) LFD b4, 12 * SIZE(BO) @@ -2666,17 +2597,17 @@ LL(42): LFD a5, 13 * SIZE(AO1) LFD b5, 13 * SIZE(BO) - FMADD y02, a6, b6, y02 + FMADD y01, a6, b6, y01 nop LFD a6, 14 * SIZE(AO1) LFD b6, 14 * SIZE(BO) - FMADD y03, a7, b7, y03 + FMADD y01, a7, b7, y01 nop LFD a7, 15 * SIZE(AO1) LFD b7, 15 * SIZE(BO) - FMADD y04, a8, b8, y04 + FMADD y01, a8, b8, y01 nop LFD a8, 16 * SIZE(AO1) LFD b8, 16 * SIZE(BO) @@ -2686,17 +2617,17 @@ LL(42): LFD a1, 17 * SIZE(AO1) LFD b1, 17 * SIZE(BO) - FMADD y02, a2, b2, y02 + FMADD y01, a2, b2, y01 nop LFD a2, 18 * SIZE(AO1) LFD b2, 18 * SIZE(BO) - FMADD y03, a3, b3, y03 + FMADD y01, a3, b3, y01 nop LFD a3, 19 * SIZE(AO1) LFD b3, 19 * SIZE(BO) - FMADD y04, a4, b4, y04 + FMADD y01, a4, b4, y01 nop LFD a4, 20 * SIZE(AO1) LFD b4, 20 * SIZE(BO) @@ -2706,17 +2637,17 @@ LL(42): LFD a5, 21 * SIZE(AO1) LFD b5, 21 * SIZE(BO) - FMADD y02, a6, b6, y02 + FMADD y01, a6, b6, y01 nop LFD a6, 22 * SIZE(AO1) LFD b6, 22 * SIZE(BO) - FMADD y03, a7, b7, y03 + FMADD y01, a7, b7, y01 nop LFD a7, 23 * SIZE(AO1) LFD b7, 23 * SIZE(BO) - FMADD y04, a8, b8, y04 + FMADD y01, a8, b8, y01 nop LFD a8, 24 * SIZE(AO1) LFD b8, 24 * SIZE(BO) @@ -2733,17 +2664,17 @@ LL(43): LFD a1, 9 * SIZE(AO1) LFD b1, 9 * SIZE(BO) - FMADD y02, a2, b2, y02 + FMADD y01, a2, b2, y01 nop LFD a2, 10 * SIZE(AO1) LFD b2, 10 * SIZE(BO) - FMADD y03, a3, b3, y03 + FMADD y01, a3, b3, y01 nop LFD a3, 11 * SIZE(AO1) LFD b3, 11 * SIZE(BO) - FMADD y04, a4, b4, y04 + FMADD y01, a4, b4, y01 nop LFD a4, 12 * SIZE(AO1) LFD b4, 12 * SIZE(BO) @@ -2753,34 +2684,34 @@ LL(43): LFD a5, 13 * SIZE(AO1) LFD b5, 13 * SIZE(BO) - FMADD y02, a6, b6, y02 + FMADD y01, a6, b6, y01 nop LFD a6, 14 * SIZE(AO1) LFD b6, 14 * SIZE(BO) - FMADD y03, a7, b7, y03 + FMADD y01, a7, b7, y01 nop LFD a7, 15 * SIZE(AO1) LFD b7, 15 * SIZE(BO) - FMADD y04, a8, b8, y04 + FMADD y01, a8, b8, y01 nop LFD a8, 16 * SIZE(AO1) LFD b8, 16 * SIZE(BO) FMADD y01, a1, b1, y01 - FMADD y02, a2, b2, y02 - FMADD y03, a3, b3, y03 - FMADD y04, a4, b4, y04 + FMADD y01, a2, b2, y01 + FMADD y01, a3, b3, y01 + FMADD y01, a4, b4, y01 FMADD y01, a5, b5, y01 addi AO1, AO1, 16 * SIZE - FMADD y02, a6, b6, y02 + FMADD y01, a6, b6, y01 addi BO, BO, 16 * SIZE - FMADD y03, a7, b7, y03 + FMADD y01, a7, b7, y01 nop - FMADD y04, a8, b8, y04 + FMADD y01, a8, b8, y01 nop .align 4 @@ -2811,17 +2742,17 @@ LL(44): LFD b8, 8 * SIZE(BO) FMADD y01, a1, b1, y01 - FMADD y02, a2, b2, y02 - FMADD y03, a3, b3, y03 - FMADD y04, a4, b4, y04 + FMADD y01, a2, b2, y01 + FMADD y01, a3, b3, y01 + FMADD y01, a4, b4, y01 FMADD y01, a5, b5, y01 addi AO1, AO1, 8 * SIZE - FMADD y02, a6, b6, y02 + FMADD y01, a6, b6, y01 addi BO, BO, 8 * SIZE - FMADD y03, a7, b7, y03 + FMADD y01, a7, b7, y01 nop - FMADD y04, a8, b8, y04 + FMADD y01, a8, b8, y01 nop .align 4 @@ -2841,12 +2772,12 @@ LL(45): FMADD y01, a1, b1, y01 addi AO1, AO1, 4 * SIZE - FMADD y02, a2, b2, y02 + FMADD y01, a2, b2, y01 addi AO2, AO2, 4 * SIZE - FMADD y03, a3, b3, y03 + FMADD y01, a3, b3, y01 addi BO, BO, 4 * SIZE - FMADD y04, a4, b4, y04 + FMADD y01, a4, b4, y01 nop .align 4 @@ -2861,7 +2792,7 @@ LL(46): FMADD y01, a1, b1, y01 addi AO1, AO1, 2 * SIZE - FMADD y02, a2, b2, y02 + FMADD y01, a2, b2, y01 addi BO, BO, 2 * SIZE .align 4 @@ -2882,10 +2813,6 @@ LL(48): LFD a1, 1 * SIZE(CO) - FADD y01, y02, y01 - FADD y03, y04, y03 - FADD y01, y03, y01 - FMADD a1, alpha, y01, a1 STFD a1, 1 * SIZE(CO) b LL(99) @@ -2893,9 +2820,7 @@ LL(48): LL(49): LFDUX a1, CO, INCY - FADD y01, y02, y01 - FADD y03, y04, y03 - FADD y01, y03, y01 + FMADD a1, alpha, f0, a1 STFDUX a1, BO, INCY .align 4 diff --git a/kernel/power/zgemm_kernel_power6.S b/kernel/power/zgemm_kernel_power6.S index 9b47b9fc15..c513285df9 100644 --- a/kernel/power/zgemm_kernel_power6.S +++ b/kernel/power/zgemm_kernel_power6.S @@ -1159,9 +1159,9 @@ LL(20): LL(22): FMA1 f0, f16, f20, f0 - FMA4 f3, f17, f20, f3 FMA2 f1, f16, f21, f1 - FMA3 f2, f17, f21, f2 + FMA4 f1, f17, f20, f1 + FMA3 f0, f17, f21, f0 LFD f28, 4 * SIZE(AO) LFD f29, 5 * SIZE(AO) @@ -1169,9 +1169,9 @@ LL(22): LFD f31, 7 * SIZE(AO) FMA1 f4, f16, f22, f4 - FMA4 f7, f17, f22, f7 FMA2 f5, f16, f23, f5 - FMA3 f6, f17, f23, f6 + FMA4 f5, f17, f22, f5 + FMA3 f4, f17, f23, f4 LFD f20, 8 * SIZE(BO) LFD f21, 9 * SIZE(BO) @@ -1179,14 +1179,14 @@ LL(22): LFD f23, 11 * SIZE(BO) FMA1 f8, f16, f24, f8 - FMA4 f11, f17, f24, f11 FMA2 f9, f16, f25, f9 - FMA3 f10, f17, f25, f10 + FMA4 f9, f17, f24, f9 + FMA3 f8, f17, f25, f8 FMA1 f12, f16, f26, f12 - FMA4 f15, f17, f26, f15 FMA2 f13, f16, f27, f13 - FMA3 f14, f17, f27, f14 + FMA4 f13, f17, f26, f13 + FMA3 f12, f17, f27, f12 LFD f24, 12 * SIZE(BO) LFD f25, 13 * SIZE(BO) @@ -1194,14 +1194,14 @@ LL(22): LFD f27, 15 * SIZE(BO) FMA1 f0, f18, f20, f0 - FMA4 f3, f19, f20, f3 FMA2 f1, f18, f21, f1 - FMA3 f2, f19, f21, f2 + FMA4 f1, f19, f20, f1 + FMA3 f0, f19, f21, f0 FMA1 f4, f18, f22, f4 - FMA4 f7, f19, f22, f7 FMA2 f5, f18, f23, f5 - FMA3 f6, f19, f23, f6 + FMA4 f5, f19, f22, f5 + FMA3 f4, f19, f23, f4 LFD f20, 16 * SIZE(BO) LFD f21, 17 * SIZE(BO) @@ -1209,14 +1209,14 @@ LL(22): LFD f23, 19 * SIZE(BO) FMA1 f8, f18, f24, f8 - FMA4 f11, f19, f24, f11 FMA2 f9, f18, f25, f9 - FMA3 f10, f19, f25, f10 + FMA4 f9, f19, f24, f9 + FMA3 f8, f19, f25, f8 FMA1 f12, f18, f26, f12 - FMA4 f15, f19, f26, f15 FMA2 f13, f18, f27, f13 - FMA3 f14, f19, f27, f14 + FMA4 f13, f19, f26, f13 + FMA3 f12, f19, f27, f12 LFD f24, 20 * SIZE(BO) LFD f25, 21 * SIZE(BO) @@ -1224,9 +1224,9 @@ LL(22): LFD f27, 23 * SIZE(BO) FMA1 f0, f28, f20, f0 - FMA4 f3, f29, f20, f3 FMA2 f1, f28, f21, f1 - FMA3 f2, f29, f21, f2 + FMA4 f1, f29, f20, f1 + FMA3 f0, f29, f21, f0 LFD f16, 8 * SIZE(AO) LFD f17, 9 * SIZE(AO) @@ -1234,9 +1234,9 @@ LL(22): LFD f19, 11 * SIZE(AO) FMA1 f4, f28, f22, f4 - FMA4 f7, f29, f22, f7 FMA2 f5, f28, f23, f5 - FMA3 f6, f29, f23, f6 + FMA4 f5, f29, f22, f5 + FMA3 f4, f29, f23, f4 LFD f20, 24 * SIZE(BO) LFD f21, 25 * SIZE(BO) @@ -1244,14 +1244,14 @@ LL(22): LFD f23, 27 * SIZE(BO) FMA1 f8, f28, f24, f8 - FMA4 f11, f29, f24, f11 FMA2 f9, f28, f25, f9 - FMA3 f10, f29, f25, f10 + FMA4 f9, f29, f24, f9 + FMA3 f8, f29, f25, f8 FMA1 f12, f28, f26, f12 - FMA4 f15, f29, f26, f15 FMA2 f13, f28, f27, f13 - FMA3 f14, f29, f27, f14 + FMA4 f13, f29, f26, f13 + FMA3 f12, f29, f27, f12 LFD f24, 28 * SIZE(BO) LFD f25, 29 * SIZE(BO) @@ -1259,14 +1259,14 @@ LL(22): LFD f27, 31 * SIZE(BO) FMA1 f0, f30, f20, f0 - FMA4 f3, f31, f20, f3 FMA2 f1, f30, f21, f1 - FMA3 f2, f31, f21, f2 + FMA4 f1, f31, f20, f1 + FMA3 f0, f31, f21, f0 FMA1 f4, f30, f22, f4 - FMA4 f7, f31, f22, f7 FMA2 f5, f30, f23, f5 - FMA3 f6, f31, f23, f6 + FMA4 f5, f31, f22, f5 + FMA3 f4, f31, f23, f4 LFD f20, 32 * SIZE(BO) LFD f21, 33 * SIZE(BO) @@ -1274,14 +1274,14 @@ LL(22): LFD f23, 35 * SIZE(BO) FMA1 f8, f30, f24, f8 - FMA4 f11, f31, f24, f11 FMA2 f9, f30, f25, f9 - FMA3 f10, f31, f25, f10 + FMA4 f9, f31, f24, f9 + FMA3 f8, f31, f25, f8 FMA1 f12, f30, f26, f12 - FMA4 f15, f31, f26, f15 FMA2 f13, f30, f27, f13 - FMA3 f14, f31, f27, f14 + FMA4 f13, f31, f26, f13 + FMA3 f12, f31, f27, f12 LFD f24, 36 * SIZE(BO) LFD f25, 37 * SIZE(BO) @@ -1318,14 +1318,14 @@ LL(25): LL(26): FMA1 f0, f16, f20, f0 - FMA4 f3, f17, f20, f3 FMA2 f1, f16, f21, f1 - FMA3 f2, f17, f21, f2 + FMA4 f1, f17, f20, f1 + FMA3 f0, f17, f21, f0 FMA1 f4, f16, f22, f4 - FMA4 f7, f17, f22, f7 FMA2 f5, f16, f23, f5 - FMA3 f6, f17, f23, f6 + FMA4 f5, f17, f22, f5 + FMA3 f4, f17, f23, f4 LFD f20, 8 * SIZE(BO) LFD f21, 9 * SIZE(BO) @@ -1333,14 +1333,14 @@ LL(26): LFD f23, 11 * SIZE(BO) FMA1 f8, f16, f24, f8 - FMA4 f11, f17, f24, f11 FMA2 f9, f16, f25, f9 - FMA3 f10, f17, f25, f10 + FMA4 f9, f17, f24, f9 + FMA3 f8, f17, f25, f8 FMA1 f12, f16, f26, f12 - FMA4 f15, f17, f26, f15 FMA2 f13, f16, f27, f13 - FMA3 f14, f17, f27, f14 + FMA4 f13, f17, f26, f13 + FMA3 f12, f17, f27, f12 LFD f16, 2 * SIZE(AO) LFD f17, 3 * SIZE(AO) @@ -1363,47 +1363,42 @@ LL(28): LFD f18, 0 * SIZE(CO2) LFD f19, 1 * SIZE(CO2) - FADD f0, f0, f2 - FADD f1, f1, f3 - FADD f4, f4, f6 - FADD f5, f5, f7 - LFD f20, 0 * SIZE(CO3) LFD f21, 1 * SIZE(CO3) LFD f22, 0 * SIZE(CO4) LFD f23, 1 * SIZE(CO4) - FADD f8, f8, f10 - FADD f9, f9, f11 - FADD f12, f12, f14 - FADD f13, f13, f15 + fmr f2, f0 + fmr f3, f1 + fmr f6, f4 + fmr f7, f5 - FNMSUB f24, f31, f1, f16 - FMADD f25, f31, f0, f17 - FNMSUB f26, f31, f5, f18 - FMADD f27, f31, f4, f19 + FMADD f24, f30, f0, f16 + FMADD f25, f30, f1, f17 + FMADD f26, f30, f4, f18 + FMADD f27, f30, f5, f19 - FMADD f0, f30, f0, f24 - FMADD f1, f30, f1, f25 - FMADD f4, f30, f4, f26 - FMADD f5, f30, f5, f27 + FNMSUB f0, f31, f3, f24 + FMADD f1, f31, f2, f25 + FNMSUB f4, f31, f7, f26 + FMADD f5, f31, f6, f27 - FNMSUB f24, f31, f9, f20 - FMADD f25, f31, f8, f21 - FNMSUB f26, f31, f13, f22 - FMADD f27, f31, f12, f23 + fmr f10, f8 + fmr f11, f9 + fmr f14, f12 + fmr f15, f13 - FMADD f8, f30, f8, f24 - FMADD f9, f30, f9, f25 - FMADD f12, f30, f12, f26 - FMADD f13, f30, f13, f27 + FMADD f24, f30, f8, f20 + FMADD f25, f30, f9, f21 + FMADD f26, f30, f12, f22 + FMADD f27, f30, f13, f23 -#else - FADD f0, f0, f2 - FADD f1, f1, f3 - FADD f4, f4, f6 - FADD f5, f5, f7 + FNMSUB f8, f31, f11, f24 + FMADD f9, f31, f10, f25 + FNMSUB f12, f31, f15, f26 + FMADD f13, f31, f14, f27 +#else FMUL f16, f31, f1 FMUL f17, f31, f0 FMUL f18, f31, f5 @@ -1414,11 +1409,6 @@ LL(28): FMSUB f4, f30, f4, f18 FMADD f5, f30, f5, f19 - FADD f8, f8, f10 - FADD f9, f9, f11 - FADD f12, f12, f14 - FADD f13, f13, f15 - FMUL f20, f31, f9 FMUL f21, f31, f8 FMUL f22, f31, f13 @@ -1616,15 +1606,15 @@ LL(32): FMA2 f5, f16, f23, f5 FMA2 f7, f18, f23, f7 - FMA4 f9, f17, f20, f9 - FMA4 f11, f19, f20, f11 - FMA3 f8, f17, f21, f8 - FMA3 f10, f19, f21, f10 + FMA4 f1, f17, f20, f1 + FMA4 f3, f19, f20, f3 + FMA3 f0, f17, f21, f0 + FMA3 f2, f19, f21, f2 - FMA4 f13, f17, f22, f13 - FMA4 f15, f19, f22, f15 - FMA3 f12, f17, f23, f12 - FMA3 f14, f19, f23, f14 + FMA4 f5, f17, f22, f5 + FMA4 f7, f19, f22, f7 + FMA3 f4, f17, f23, f4 + FMA3 f6, f19, f23, f6 LFD f20, 8 * SIZE(BO) LFD f21, 9 * SIZE(BO) @@ -1646,15 +1636,15 @@ LL(32): FMA2 f5, f28, f27, f5 FMA2 f7, f30, f27, f7 - FMA4 f9, f29, f24, f9 - FMA4 f11, f31, f24, f11 - FMA3 f8, f29, f25, f8 - FMA3 f10, f31, f25, f10 + FMA4 f1, f29, f24, f1 + FMA4 f3, f31, f24, f3 + FMA3 f0, f29, f25, f0 + FMA3 f2, f31, f25, f2 - FMA4 f13, f29, f26, f13 - FMA4 f15, f31, f26, f15 - FMA3 f12, f29, f27, f12 - FMA3 f14, f31, f27, f14 + FMA4 f5, f29, f26, f5 + FMA4 f7, f31, f26, f7 + FMA3 f4, f29, f27, f4 + FMA3 f6, f31, f27, f6 LFD f24, 12 * SIZE(BO) LFD f25, 13 * SIZE(BO) @@ -1676,15 +1666,15 @@ LL(32): FMA2 f5, f16, f23, f5 FMA2 f7, f18, f23, f7 - FMA4 f9, f17, f20, f9 - FMA4 f11, f19, f20, f11 - FMA3 f8, f17, f21, f8 - FMA3 f10, f19, f21, f10 + FMA4 f1, f17, f20, f1 + FMA4 f3, f19, f20, f3 + FMA3 f0, f17, f21, f0 + FMA3 f2, f19, f21, f2 - FMA4 f13, f17, f22, f13 - FMA4 f15, f19, f22, f15 - FMA3 f12, f17, f23, f12 - FMA3 f14, f19, f23, f14 + FMA4 f5, f17, f22, f5 + FMA4 f7, f19, f22, f7 + FMA3 f4, f17, f23, f4 + FMA3 f6, f19, f23, f6 LFD f20, 16 * SIZE(BO) LFD f21, 17 * SIZE(BO) @@ -1706,15 +1696,15 @@ LL(32): FMA2 f5, f28, f27, f5 FMA2 f7, f30, f27, f7 - FMA4 f9, f29, f24, f9 - FMA4 f11, f31, f24, f11 - FMA3 f8, f29, f25, f8 - FMA3 f10, f31, f25, f10 + FMA4 f1, f29, f24, f1 + FMA4 f3, f31, f24, f3 + FMA3 f0, f29, f25, f0 + FMA3 f2, f31, f25, f2 - FMA4 f13, f29, f26, f13 - FMA4 f15, f31, f26, f15 - FMA3 f12, f29, f27, f12 - FMA3 f14, f31, f27, f14 + FMA4 f5, f29, f26, f5 + FMA4 f7, f31, f26, f7 + FMA3 f4, f29, f27, f4 + FMA3 f6, f31, f27, f6 LFD f24, 20 * SIZE(BO) LFD f25, 21 * SIZE(BO) @@ -1736,15 +1726,15 @@ LL(32): FMA2 f5, f16, f23, f5 FMA2 f7, f18, f23, f7 - FMA4 f9, f17, f20, f9 - FMA4 f11, f19, f20, f11 - FMA3 f8, f17, f21, f8 - FMA3 f10, f19, f21, f10 + FMA4 f1, f17, f20, f1 + FMA4 f3, f19, f20, f3 + FMA3 f0, f17, f21, f0 + FMA3 f2, f19, f21, f2 - FMA4 f13, f17, f22, f13 - FMA4 f15, f19, f22, f15 - FMA3 f12, f17, f23, f12 - FMA3 f14, f19, f23, f14 + FMA4 f5, f17, f22, f5 + FMA4 f7, f19, f22, f7 + FMA3 f4, f17, f23, f4 + FMA3 f6, f19, f23, f6 LFD f20, 24 * SIZE(BO) LFD f21, 25 * SIZE(BO) @@ -1766,15 +1756,15 @@ LL(32): FMA2 f5, f28, f27, f5 FMA2 f7, f30, f27, f7 - FMA4 f9, f29, f24, f9 - FMA4 f11, f31, f24, f11 - FMA3 f8, f29, f25, f8 - FMA3 f10, f31, f25, f10 + FMA4 f1, f29, f24, f1 + FMA4 f3, f31, f24, f3 + FMA3 f0, f29, f25, f0 + FMA3 f2, f31, f25, f2 - FMA4 f13, f29, f26, f13 - FMA4 f15, f31, f26, f15 - FMA3 f12, f29, f27, f12 - FMA3 f14, f31, f27, f14 + FMA4 f5, f29, f26, f5 + FMA4 f7, f31, f26, f7 + FMA3 f4, f29, f27, f4 + FMA3 f6, f31, f27, f6 LFD f24, 28 * SIZE(BO) LFD f25, 29 * SIZE(BO) @@ -1796,15 +1786,15 @@ LL(32): FMA2 f5, f16, f23, f5 FMA2 f7, f18, f23, f7 - FMA4 f9, f17, f20, f9 - FMA4 f11, f19, f20, f11 - FMA3 f8, f17, f21, f8 - FMA3 f10, f19, f21, f10 + FMA4 f1, f17, f20, f1 + FMA4 f3, f19, f20, f3 + FMA3 f0, f17, f21, f0 + FMA3 f2, f19, f21, f2 - FMA4 f13, f17, f22, f13 - FMA4 f15, f19, f22, f15 - FMA3 f12, f17, f23, f12 - FMA3 f14, f19, f23, f14 + FMA4 f5, f17, f22, f5 + FMA4 f7, f19, f22, f7 + FMA3 f4, f17, f23, f4 + FMA3 f6, f19, f23, f6 LFD f20, 32 * SIZE(BO) LFD f21, 33 * SIZE(BO) @@ -1826,15 +1816,15 @@ LL(32): FMA2 f5, f28, f27, f5 FMA2 f7, f30, f27, f7 - FMA4 f9, f29, f24, f9 - FMA4 f11, f31, f24, f11 - FMA3 f8, f29, f25, f8 - FMA3 f10, f31, f25, f10 + FMA4 f1, f29, f24, f1 + FMA4 f3, f31, f24, f3 + FMA3 f0, f29, f25, f0 + FMA3 f2, f31, f25, f2 - FMA4 f13, f29, f26, f13 - FMA4 f15, f31, f26, f15 - FMA3 f12, f29, f27, f12 - FMA3 f14, f31, f27, f14 + FMA4 f5, f29, f26, f5 + FMA4 f7, f31, f26, f7 + FMA3 f4, f29, f27, f4 + FMA3 f6, f31, f27, f6 LFD f24, 36 * SIZE(BO) LFD f25, 37 * SIZE(BO) @@ -1883,20 +1873,20 @@ LL(36): FMA2 f5, f16, f23, f5 FMA2 f7, f18, f23, f7 - FMA4 f9, f17, f20, f9 - FMA4 f11, f19, f20, f11 - FMA3 f8, f17, f21, f8 - FMA3 f10, f19, f21, f10 + FMA4 f1, f17, f20, f1 + FMA4 f3, f19, f20, f3 + FMA3 f0, f17, f21, f0 + FMA3 f2, f19, f21, f2 LFD f16, 4 * SIZE(AO) LFD f18, 6 * SIZE(AO) LFD f20, 4 * SIZE(BO) LFD f21, 5 * SIZE(BO) - FMA4 f13, f17, f22, f13 - FMA4 f15, f19, f22, f15 - FMA3 f12, f17, f23, f12 - FMA3 f14, f19, f23, f14 + FMA4 f5, f17, f22, f5 + FMA4 f7, f19, f22, f7 + FMA3 f4, f17, f23, f4 + FMA3 f6, f19, f23, f6 LFD f17, 5 * SIZE(AO) LFD f19, 7 * SIZE(AO) @@ -1916,52 +1906,42 @@ LL(38): LFD f18, 2 * SIZE(CO1) LFD f19, 3 * SIZE(CO1) - FADD f0, f0, f8 - FADD f1, f1, f9 - FADD f2, f2, f10 - FADD f3, f3, f11 - LFD f20, 0 * SIZE(CO2) LFD f21, 1 * SIZE(CO2) LFD f22, 2 * SIZE(CO2) LFD f23, 3 * SIZE(CO2) - FADD f4, f4, f12 - FADD f5, f5, f13 - FADD f6, f6, f14 - FADD f7, f7, f15 + fmr f8, f0 + fmr f9, f1 + fmr f10, f2 + fmr f11, f3 - FNMSUB f24, f31, f1, f16 - FMADD f25, f31, f0, f17 - FNMSUB f26, f31, f3, f18 - FMADD f27, f31, f2, f19 + FMADD f24, f30, f0, f16 + FMADD f25, f30, f1, f17 + FMADD f26, f30, f2, f18 + FMADD f27, f30, f3, f19 - FMADD f0, f30, f0, f24 - FMADD f1, f30, f1, f25 - FMADD f2, f30, f2, f26 - FMADD f3, f30, f3, f27 + FNMSUB f0, f31, f9, f24 + FMADD f1, f31, f8, f25 + FNMSUB f2, f31, f11, f26 + FMADD f3, f31, f10, f27 - FNMSUB f24, f31, f5, f20 - FMADD f25, f31, f4, f21 - FNMSUB f26, f31, f7, f22 - FMADD f27, f31, f6, f23 + fmr f12, f4 + fmr f13, f5 + fmr f14, f6 + fmr f15, f7 - FMADD f4, f30, f4, f24 - FMADD f5, f30, f5, f25 - FMADD f6, f30, f6, f26 - FMADD f7, f30, f7, f27 + FMADD f24, f30, f4, f20 + FMADD f25, f30, f5, f21 + FMADD f26, f30, f6, f22 + FMADD f27, f30, f7, f23 -#else - FADD f0, f0, f8 - FADD f1, f1, f9 - FADD f2, f2, f10 - FADD f3, f3, f11 - - FADD f4, f4, f12 - FADD f5, f5, f13 - FADD f6, f6, f14 - FADD f7, f7, f15 + FNMSUB f4, f31, f13, f24 + FMADD f5, f31, f12, f25 + FNMSUB f6, f31, f15, f26 + FMADD f7, f31, f14, f27 +#else FMUL f16, f31, f1 FMUL f17, f31, f0 FMUL f18, f31, f3 @@ -2101,14 +2081,14 @@ LL(40): LL(42): FMA1 f0, f16, f20, f0 - FMA4 f3, f17, f20, f3 FMA2 f1, f16, f21, f1 - FMA3 f2, f17, f21, f2 + FMA4 f1, f17, f20, f1 + FMA3 f0, f17, f21, f0 FMA1 f4, f16, f22, f4 - FMA4 f7, f17, f22, f7 FMA2 f5, f16, f23, f5 - FMA3 f6, f17, f23, f6 + FMA4 f5, f17, f22, f5 + FMA3 f4, f17, f23, f4 LFD f16, 2 * SIZE(AO) LFD f17, 3 * SIZE(AO) @@ -2119,14 +2099,14 @@ LL(42): LFD f23, 7 * SIZE(BO) FMA1 f0, f16, f20, f0 - FMA4 f3, f17, f20, f3 FMA2 f1, f16, f21, f1 - FMA3 f2, f17, f21, f2 + FMA4 f1, f17, f20, f1 + FMA3 f0, f17, f21, f0 FMA1 f4, f16, f22, f4 - FMA4 f7, f17, f22, f7 FMA2 f5, f16, f23, f5 - FMA3 f6, f17, f23, f6 + FMA4 f5, f17, f22, f5 + FMA3 f4, f17, f23, f4 LFD f16, 4 * SIZE(AO) LFD f17, 5 * SIZE(AO) @@ -2137,14 +2117,14 @@ LL(42): LFD f23, 11 * SIZE(BO) FMA1 f0, f16, f20, f0 - FMA4 f3, f17, f20, f3 FMA2 f1, f16, f21, f1 - FMA3 f2, f17, f21, f2 + FMA4 f1, f17, f20, f1 + FMA3 f0, f17, f21, f0 FMA1 f4, f16, f22, f4 - FMA4 f7, f17, f22, f7 FMA2 f5, f16, f23, f5 - FMA3 f6, f17, f23, f6 + FMA4 f5, f17, f22, f5 + FMA3 f4, f17, f23, f4 LFD f16, 6 * SIZE(AO) LFD f17, 7 * SIZE(AO) @@ -2155,14 +2135,14 @@ LL(42): LFD f23, 15 * SIZE(BO) FMA1 f0, f16, f20, f0 - FMA4 f3, f17, f20, f3 FMA2 f1, f16, f21, f1 - FMA3 f2, f17, f21, f2 + FMA4 f1, f17, f20, f1 + FMA3 f0, f17, f21, f0 FMA1 f4, f16, f22, f4 - FMA4 f7, f17, f22, f7 FMA2 f5, f16, f23, f5 - FMA3 f6, f17, f23, f6 + FMA4 f5, f17, f22, f5 + FMA3 f4, f17, f23, f4 LFD f16, 8 * SIZE(AO) LFD f17, 9 * SIZE(AO) @@ -2202,14 +2182,14 @@ LL(45): LL(46): FMA1 f0, f16, f20, f0 - FMA4 f3, f17, f20, f3 FMA2 f1, f16, f21, f1 - FMA3 f2, f17, f21, f2 + FMA4 f1, f17, f20, f1 + FMA3 f0, f17, f21, f0 FMA1 f4, f16, f22, f4 - FMA4 f7, f17, f22, f7 FMA2 f5, f16, f23, f5 - FMA3 f6, f17, f23, f6 + FMA4 f5, f17, f22, f5 + FMA3 f4, f17, f23, f4 LFD f16, 2 * SIZE(AO) LFD f17, 3 * SIZE(AO) @@ -2231,27 +2211,22 @@ LL(48): LFD f20, 0 * SIZE(CO2) LFD f21, 1 * SIZE(CO2) - FADD f0, f0, f2 - FADD f1, f1, f3 - FADD f4, f4, f6 - FADD f5, f5, f7 + fmr f2, f0 + fmr f3, f1 + fmr f6, f4 + fmr f7, f5 - FNMSUB f24, f31, f1, f16 - FMADD f25, f31, f0, f17 - FNMSUB f26, f31, f5, f20 - FMADD f27, f31, f4, f21 + FMADD f24, f30, f0, f16 + FMADD f25, f30, f1, f17 + FMADD f26, f30, f4, f20 + FMADD f27, f30, f5, f21 - FMADD f0, f30, f0, f24 - FMADD f1, f30, f1, f25 - FMADD f4, f30, f4, f26 - FMADD f5, f30, f5, f27 + FNMSUB f0, f31, f3, f24 + FMADD f1, f31, f2, f25 + FNMSUB f4, f31, f7, f26 + FMADD f5, f31, f6, f27 #else - FADD f0, f0, f2 - FADD f1, f1, f3 - FADD f4, f4, f6 - FADD f5, f5, f7 - FMUL f16, f31, f1 FMUL f17, f31, f0 FMUL f18, f31, f5 @@ -2401,10 +2376,10 @@ LL(52): FMA2 f1, f16, f21, f1 FMA2 f3, f18, f21, f3 - FMA4 f9, f17, f20, f9 - FMA4 f11, f19, f20, f11 - FMA3 f8, f17, f21, f8 - FMA3 f10, f19, f21, f10 + FMA4 f1, f17, f20, f1 + FMA4 f3, f19, f20, f3 + FMA3 f0, f17, f21, f0 + FMA3 f2, f19, f21, f2 LFD f16, 4 * SIZE(AO) LFD f17, 5 * SIZE(AO) @@ -2416,10 +2391,10 @@ LL(52): FMA2 f1, f16, f23, f1 FMA2 f3, f18, f23, f3 - FMA4 f9, f17, f22, f9 - FMA4 f11, f19, f22, f11 - FMA3 f8, f17, f23, f8 - FMA3 f10, f19, f23, f10 + FMA4 f1, f17, f22, f1 + FMA4 f3, f19, f22, f3 + FMA3 f0, f17, f23, f0 + FMA3 f2, f19, f23, f2 LFD f16, 8 * SIZE(AO) LFD f17, 9 * SIZE(AO) @@ -2436,10 +2411,10 @@ LL(52): FMA2 f1, f16, f21, f1 FMA2 f3, f18, f21, f3 - FMA4 f9, f17, f20, f9 - FMA4 f11, f19, f20, f11 - FMA3 f8, f17, f21, f8 - FMA3 f10, f19, f21, f10 + FMA4 f1, f17, f20, f1 + FMA4 f3, f19, f20, f3 + FMA3 f0, f17, f21, f0 + FMA3 f2, f19, f21, f2 LFD f16, 12 * SIZE(AO) LFD f17, 13 * SIZE(AO) @@ -2451,10 +2426,10 @@ LL(52): FMA2 f1, f16, f23, f1 FMA2 f3, f18, f23, f3 - FMA4 f9, f17, f22, f9 - FMA4 f11, f19, f22, f11 - FMA3 f8, f17, f23, f8 - FMA3 f10, f19, f23, f10 + FMA4 f1, f17, f22, f1 + FMA4 f3, f19, f22, f3 + FMA3 f0, f17, f23, f0 + FMA3 f2, f19, f23, f2 LFD f16, 16 * SIZE(AO) LFD f17, 17 * SIZE(AO) @@ -2471,10 +2446,10 @@ LL(52): FMA2 f1, f16, f21, f1 FMA2 f3, f18, f21, f3 - FMA4 f9, f17, f20, f9 - FMA4 f11, f19, f20, f11 - FMA3 f8, f17, f21, f8 - FMA3 f10, f19, f21, f10 + FMA4 f1, f17, f20, f1 + FMA4 f3, f19, f20, f3 + FMA3 f0, f17, f21, f0 + FMA3 f2, f19, f21, f2 LFD f16, 20 * SIZE(AO) LFD f17, 21 * SIZE(AO) @@ -2486,10 +2461,10 @@ LL(52): FMA2 f1, f16, f23, f1 FMA2 f3, f18, f23, f3 - FMA4 f9, f17, f22, f9 - FMA4 f11, f19, f22, f11 - FMA3 f8, f17, f23, f8 - FMA3 f10, f19, f23, f10 + FMA4 f1, f17, f22, f1 + FMA4 f3, f19, f22, f3 + FMA3 f0, f17, f23, f0 + FMA3 f2, f19, f23, f2 LFD f16, 24 * SIZE(AO) LFD f17, 25 * SIZE(AO) @@ -2506,10 +2481,10 @@ LL(52): FMA2 f1, f16, f21, f1 FMA2 f3, f18, f21, f3 - FMA4 f9, f17, f20, f9 - FMA4 f11, f19, f20, f11 - FMA3 f8, f17, f21, f8 - FMA3 f10, f19, f21, f10 + FMA4 f1, f17, f20, f1 + FMA4 f3, f19, f20, f3 + FMA3 f0, f17, f21, f0 + FMA3 f2, f19, f21, f2 LFD f16, 28 * SIZE(AO) LFD f17, 29 * SIZE(AO) @@ -2521,10 +2496,10 @@ LL(52): FMA2 f1, f16, f23, f1 FMA2 f3, f18, f23, f3 - FMA4 f9, f17, f22, f9 - FMA4 f11, f19, f22, f11 - FMA3 f8, f17, f23, f8 - FMA3 f10, f19, f23, f10 + FMA4 f1, f17, f22, f1 + FMA4 f3, f19, f22, f3 + FMA3 f0, f17, f23, f0 + FMA3 f2, f19, f23, f2 LFD f16, 32 * SIZE(AO) LFD f17, 33 * SIZE(AO) @@ -2573,10 +2548,10 @@ LL(56): LFD f16, 4 * SIZE(AO) LFD f18, 6 * SIZE(AO) - FMA4 f9, f17, f20, f9 - FMA4 f11, f19, f20, f11 - FMA3 f8, f17, f21, f8 - FMA3 f10, f19, f21, f10 + FMA4 f1, f17, f20, f1 + FMA4 f3, f19, f20, f3 + FMA3 f0, f17, f21, f0 + FMA3 f2, f19, f21, f2 LFD f17, 5 * SIZE(AO) LFD f19, 7 * SIZE(AO) @@ -2595,27 +2570,22 @@ LL(58): LFD f18, 2 * SIZE(CO1) LFD f19, 3 * SIZE(CO1) - FADD f0, f0, f8 - FADD f1, f1, f9 - FADD f2, f2, f10 - FADD f3, f3, f11 + fmr f8, f0 + fmr f9, f1 + fmr f10, f2 + fmr f11, f3 - FNMSUB f24, f31, f1, f16 - FMADD f25, f31, f0, f17 - FNMSUB f26, f31, f3, f18 - FMADD f27, f31, f2, f19 + FMADD f24, f30, f0, f16 + FMADD f25, f30, f1, f17 + FMADD f26, f30, f2, f18 + FMADD f27, f30, f3, f19 - FMADD f0, f30, f0, f24 - FMADD f1, f30, f1, f25 - FMADD f2, f30, f2, f26 - FMADD f3, f30, f3, f27 + FNMSUB f0, f31, f9, f24 + FMADD f1, f31, f8, f25 + FNMSUB f2, f31, f11, f26 + FMADD f3, f31, f10, f27 #else - FADD f0, f0, f8 - FADD f1, f1, f9 - FADD f2, f2, f10 - FADD f3, f3, f11 - FMUL f16, f31, f1 FMUL f17, f31, f0 FMUL f18, f31, f3 @@ -2735,9 +2705,9 @@ LL(60): LL(62): FMA1 f0, f16, f20, f0 - FMA4 f3, f17, f20, f3 FMA2 f1, f16, f21, f1 - FMA3 f2, f17, f21, f2 + FMA4 f1, f17, f20, f1 + FMA3 f0, f17, f21, f0 LFD f16, 4 * SIZE(AO) LFD f17, 5 * SIZE(AO) @@ -2745,9 +2715,9 @@ LL(62): LFD f21, 5 * SIZE(BO) FMA1 f0, f18, f22, f0 - FMA4 f3, f19, f22, f3 FMA2 f1, f18, f23, f1 - FMA3 f2, f19, f23, f2 + FMA4 f1, f19, f22, f1 + FMA3 f0, f19, f23, f0 LFD f18, 6 * SIZE(AO) LFD f19, 7 * SIZE(AO) @@ -2755,9 +2725,9 @@ LL(62): LFD f23, 7 * SIZE(BO) FMA1 f0, f16, f20, f0 - FMA4 f3, f17, f20, f3 FMA2 f1, f16, f21, f1 - FMA3 f2, f17, f21, f2 + FMA4 f1, f17, f20, f1 + FMA3 f0, f17, f21, f0 LFD f16, 8 * SIZE(AO) LFD f17, 9 * SIZE(AO) @@ -2765,9 +2735,9 @@ LL(62): LFD f21, 9 * SIZE(BO) FMA1 f0, f18, f22, f0 - FMA4 f3, f19, f22, f3 FMA2 f1, f18, f23, f1 - FMA3 f2, f19, f23, f2 + FMA4 f1, f19, f22, f1 + FMA3 f0, f19, f23, f0 LFD f18, 10 * SIZE(AO) LFD f19, 11 * SIZE(AO) @@ -2803,11 +2773,11 @@ LL(65): LL(66): FMA1 f0, f16, f20, f0 - FMA4 f3, f17, f20, f3 - LFD f20, 2 * SIZE(BO) FMA2 f1, f16, f21, f1 LFD f16, 2 * SIZE(AO) - FMA3 f2, f17, f21, f2 + FMA4 f1, f17, f20, f1 + LFD f20, 2 * SIZE(BO) + FMA3 f0, f17, f21, f0 LFD f17, 3 * SIZE(AO) LFD f21, 3 * SIZE(BO) @@ -2821,20 +2791,17 @@ LL(68): LFD f16, 0 * SIZE(CO1) LFD f17, 1 * SIZE(CO1) - FADD f0, f0, f2 - FADD f1, f1, f3 + fmr f2, f0 + fmr f3, f1 - FNMSUB f24, f31, f1, f16 - FMADD f25, f31, f0, f17 + FMADD f24, f30, f0, f16 + FMADD f25, f30, f1, f17 - FMADD f0, f30, f0, f24 - FMADD f1, f30, f1, f25 + FNMSUB f0, f31, f3, f24 + FMADD f1, f31, f2, f25 #else - FADD f0, f0, f2 - FADD f1, f1, f3 - FMUL f16, f31, f1 FMUL f17, f31, f0 From dd6c33d34d9925a7c02284c47a86afa4bf117237 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 19 Jul 2024 16:14:55 +0200 Subject: [PATCH 213/267] make NAN handling depend on dummy2 parameter --- kernel/x86/scal.S | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/kernel/x86/scal.S b/kernel/x86/scal.S index b0c232b1bd..7e12a52ab6 100644 --- a/kernel/x86/scal.S +++ b/kernel/x86/scal.S @@ -57,19 +57,24 @@ #ifdef XDOUBLE movl 44(%esp),%edi movl 48(%esp),%esi + movl 64(%esp),%ecx #elif defined(DOUBLE) movl 36(%esp),%edi movl 40(%esp),%esi + movl 56(%esp),%ecx #else movl 32(%esp),%edi movl 36(%esp),%esi + movl 54(%esp),%ecx #endif ftst fnstsw %ax andb $68, %ah -// je .L300 # Alpha != ZERO - jmp .L300 + je .L300 # Alpha != ZERO + + cmpl $1,%ecx # dummy2 flag + je .L300 /* Alpha == ZERO */ cmpl $1,%esi From 7311d9301650adc0672c2990e1997702630b9d47 Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Fri, 19 Jul 2024 16:50:50 +0000 Subject: [PATCH 214/267] Unroll TT further --- kernel/arm64/sgemm_small_kernel_tt_sve.c | 204 ++++++++++++++++++++++- 1 file changed, 202 insertions(+), 2 deletions(-) diff --git a/kernel/arm64/sgemm_small_kernel_tt_sve.c b/kernel/arm64/sgemm_small_kernel_tt_sve.c index 50dbd7399c..c66330fb57 100644 --- a/kernel/arm64/sgemm_small_kernel_tt_sve.c +++ b/kernel/arm64/sgemm_small_kernel_tt_sve.c @@ -219,6 +219,7 @@ CNAME(BLASLONG M, const BLASLONG v_m2 = M & -v_size2; const BLASLONG v_m1 = M & -v_size; + const BLASLONG n8 = N & -8; const BLASLONG n4 = N & -4; const int pack_a = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0; @@ -238,23 +239,35 @@ CNAME(BLASLONG M, CREATE_A_POINTER(1, v_size); BLASLONG j = 0; - for (; j < n4; j += 4) { + for (; j < n8; j += 8) { CREATE_B_POINTER(0, 0); CREATE_B_POINTER(1, 1); CREATE_B_POINTER(2, 2); CREATE_B_POINTER(3, 3); - UPDATE_B_POINTER(4); + CREATE_B_POINTER(4, 4); + CREATE_B_POINTER(5, 5); + CREATE_B_POINTER(6, 6); + CREATE_B_POINTER(7, 7); + UPDATE_B_POINTER(8); BLASLONG k = 0; DECLARE_RESULT_VECTOR(0, 0); DECLARE_RESULT_VECTOR(0, 1); DECLARE_RESULT_VECTOR(0, 2); DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(0, 4); + DECLARE_RESULT_VECTOR(0, 5); + DECLARE_RESULT_VECTOR(0, 6); + DECLARE_RESULT_VECTOR(0, 7); DECLARE_RESULT_VECTOR(1, 0); DECLARE_RESULT_VECTOR(1, 1); DECLARE_RESULT_VECTOR(1, 2); DECLARE_RESULT_VECTOR(1, 3); + DECLARE_RESULT_VECTOR(1, 4); + DECLARE_RESULT_VECTOR(1, 5); + DECLARE_RESULT_VECTOR(1, 6); + DECLARE_RESULT_VECTOR(1, 7); if (LIKELY(packed_a != NULL)) { if (j == 0) { @@ -267,12 +280,21 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + QUADWORD_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); GATHER_LOAD_A(pg_true, 1, 0); VECTOR_PACK_A(1, 0); UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); } } else { for (; k < K; k++) { @@ -283,16 +305,102 @@ CNAME(BLASLONG M, UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + QUADWORD_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); UNPACK_VECTOR_A(1, 0); UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); } } } else { for (; k < K; k++) { + QUADWORD_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + QUADWORD_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + GATHER_LOAD_A(pg_true, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0); + } + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + VECTOR_STORE(pg_true, 0, 4); + VECTOR_STORE(pg_true, 0, 5); + VECTOR_STORE(pg_true, 0, 6); + VECTOR_STORE(pg_true, 0, 7); + VECTOR_STORE(pg_true, 1, 0); + VECTOR_STORE(pg_true, 1, 1); + VECTOR_STORE(pg_true, 1, 2); + VECTOR_STORE(pg_true, 1, 3); + VECTOR_STORE(pg_true, 1, 4); + VECTOR_STORE(pg_true, 1, 5); + VECTOR_STORE(pg_true, 1, 6); + VECTOR_STORE(pg_true, 1, 7); + INCR_C_POINTER(0, 8); + INCR_C_POINTER(1, 8); + } + for (; j < n4; j += 4) { + + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + CREATE_B_POINTER(2, 2); + CREATE_B_POINTER(3, 3); + UPDATE_B_POINTER(4); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(1, 0); + DECLARE_RESULT_VECTOR(1, 1); + DECLARE_RESULT_VECTOR(1, 2); + DECLARE_RESULT_VECTOR(1, 3); + + if (LIKELY(packed_a != NULL)) { + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + UNPACK_VECTOR_A(0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + UNPACK_VECTOR_A(1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0); + } + } else { + for (; k < K; k++) { + QUADWORD_LOAD_B(0, 0); GATHER_LOAD_A(pg_true, 0, 0); UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); @@ -361,6 +469,52 @@ CNAME(BLASLONG M, CREATE_A_POINTER(0, 0); BLASLONG j = 0; + for (; j < n8; j += 8) { + + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + CREATE_B_POINTER(2, 2); + CREATE_B_POINTER(3, 3); + CREATE_B_POINTER(4, 4); + CREATE_B_POINTER(5, 5); + CREATE_B_POINTER(6, 6); + CREATE_B_POINTER(7, 7); + UPDATE_B_POINTER(8); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(0, 4); + DECLARE_RESULT_VECTOR(0, 5); + DECLARE_RESULT_VECTOR(0, 6); + DECLARE_RESULT_VECTOR(0, 7); + + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + GATHER_LOAD_A(pg_true, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + QUADWORD_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + } + VECTOR_STORE(pg_true, 0, 0); + VECTOR_STORE(pg_true, 0, 1); + VECTOR_STORE(pg_true, 0, 2); + VECTOR_STORE(pg_true, 0, 3); + VECTOR_STORE(pg_true, 0, 4); + VECTOR_STORE(pg_true, 0, 5); + VECTOR_STORE(pg_true, 0, 6); + VECTOR_STORE(pg_true, 0, 7); + INCR_C_POINTER(0, 8); + } for (; j < n4; j += 4) { CREATE_B_POINTER(0, 0); @@ -418,6 +572,52 @@ CNAME(BLASLONG M, CREATE_A_POINTER(0, 0); BLASLONG j = 0; + for (; j < n8; j += 8) { + + CREATE_B_POINTER(0, 0); + CREATE_B_POINTER(1, 1); + CREATE_B_POINTER(2, 2); + CREATE_B_POINTER(3, 3); + CREATE_B_POINTER(4, 4); + CREATE_B_POINTER(5, 5); + CREATE_B_POINTER(6, 6); + CREATE_B_POINTER(7, 7); + UPDATE_B_POINTER(8); + + BLASLONG k = 0; + DECLARE_RESULT_VECTOR(0, 0); + DECLARE_RESULT_VECTOR(0, 1); + DECLARE_RESULT_VECTOR(0, 2); + DECLARE_RESULT_VECTOR(0, 3); + DECLARE_RESULT_VECTOR(0, 4); + DECLARE_RESULT_VECTOR(0, 5); + DECLARE_RESULT_VECTOR(0, 6); + DECLARE_RESULT_VECTOR(0, 7); + + for (; k < K; k++) { + + QUADWORD_LOAD_B(0, 0); + GATHER_LOAD_A(pg_tail, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0); + QUADWORD_LOAD_B(4, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0); + UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0); + } + VECTOR_STORE(pg_tail, 0, 0); + VECTOR_STORE(pg_tail, 0, 1); + VECTOR_STORE(pg_tail, 0, 2); + VECTOR_STORE(pg_tail, 0, 3); + VECTOR_STORE(pg_tail, 0, 4); + VECTOR_STORE(pg_tail, 0, 5); + VECTOR_STORE(pg_tail, 0, 6); + VECTOR_STORE(pg_tail, 0, 7); + INCR_C_POINTER(0, 8); + } for (; j < n4; j += 4) { CREATE_B_POINTER(0, 0); From ea4ab3b31081a4344905929a9c911b0a53eeca45 Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Sat, 20 Jul 2024 13:39:22 +0000 Subject: [PATCH 215/267] Better header guard around bridge --- kernel/arm64/dgemm_small_kernel_nn_sve.c | 3 ++- kernel/arm64/dgemm_small_kernel_nt_sve.c | 3 ++- kernel/arm64/dgemm_small_kernel_tn_sve.c | 3 ++- kernel/arm64/dgemm_small_kernel_tt_sve.c | 3 ++- kernel/arm64/sgemm_small_kernel_nn_sve.c | 3 ++- kernel/arm64/sgemm_small_kernel_nt_sve.c | 3 ++- kernel/arm64/sgemm_small_kernel_tn_sve.c | 3 ++- kernel/arm64/sgemm_small_kernel_tt_sve.c | 3 ++- 8 files changed, 16 insertions(+), 8 deletions(-) diff --git a/kernel/arm64/dgemm_small_kernel_nn_sve.c b/kernel/arm64/dgemm_small_kernel_nn_sve.c index 4176334713..fa39103d0e 100644 --- a/kernel/arm64/dgemm_small_kernel_nn_sve.c +++ b/kernel/arm64/dgemm_small_kernel_nn_sve.c @@ -29,7 +29,8 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -#ifdef __ARM_NEON_SVE_BRIDGE +#if defined(__ARM_NEON_SVE_BRIDGE) && defined(__has_include) && \ + __has_include() #include #else #define svdup_neonq_f32(fixed_reg) \ diff --git a/kernel/arm64/dgemm_small_kernel_nt_sve.c b/kernel/arm64/dgemm_small_kernel_nt_sve.c index 241d96a6c9..0b306e7542 100644 --- a/kernel/arm64/dgemm_small_kernel_nt_sve.c +++ b/kernel/arm64/dgemm_small_kernel_nt_sve.c @@ -29,7 +29,8 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -#ifdef __ARM_NEON_SVE_BRIDGE +#if defined(__ARM_NEON_SVE_BRIDGE) && defined(__has_include) && \ + __has_include() #include #else #define svdup_neonq_f32(fixed_reg) \ diff --git a/kernel/arm64/dgemm_small_kernel_tn_sve.c b/kernel/arm64/dgemm_small_kernel_tn_sve.c index 1b0fada2ae..daca8e1bef 100644 --- a/kernel/arm64/dgemm_small_kernel_tn_sve.c +++ b/kernel/arm64/dgemm_small_kernel_tn_sve.c @@ -29,7 +29,8 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -#ifdef __ARM_NEON_SVE_BRIDGE +#if defined(__ARM_NEON_SVE_BRIDGE) && defined(__has_include) && \ + __has_include() #include #else #define svdup_neonq_f32(fixed_reg) \ diff --git a/kernel/arm64/dgemm_small_kernel_tt_sve.c b/kernel/arm64/dgemm_small_kernel_tt_sve.c index aa5bf27512..efe11a9f9b 100644 --- a/kernel/arm64/dgemm_small_kernel_tt_sve.c +++ b/kernel/arm64/dgemm_small_kernel_tt_sve.c @@ -29,7 +29,8 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -#ifdef __ARM_NEON_SVE_BRIDGE +#if defined(__ARM_NEON_SVE_BRIDGE) && defined(__has_include) && \ + __has_include() #include #else #define svdup_neonq_f32(fixed_reg) \ diff --git a/kernel/arm64/sgemm_small_kernel_nn_sve.c b/kernel/arm64/sgemm_small_kernel_nn_sve.c index 0af073a149..8ea9cf5a7c 100644 --- a/kernel/arm64/sgemm_small_kernel_nn_sve.c +++ b/kernel/arm64/sgemm_small_kernel_nn_sve.c @@ -29,7 +29,8 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -#ifdef __ARM_NEON_SVE_BRIDGE +#if defined(__ARM_NEON_SVE_BRIDGE) && defined(__has_include) && \ + __has_include() #include #else #define svdup_neonq_f32(fixed_reg) \ diff --git a/kernel/arm64/sgemm_small_kernel_nt_sve.c b/kernel/arm64/sgemm_small_kernel_nt_sve.c index ed7ee6bd63..ac7e067cda 100644 --- a/kernel/arm64/sgemm_small_kernel_nt_sve.c +++ b/kernel/arm64/sgemm_small_kernel_nt_sve.c @@ -29,7 +29,8 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -#ifdef __ARM_NEON_SVE_BRIDGE +#if defined(__ARM_NEON_SVE_BRIDGE) && defined(__has_include) && \ + __has_include() #include #else #define svdup_neonq_f32(fixed_reg) \ diff --git a/kernel/arm64/sgemm_small_kernel_tn_sve.c b/kernel/arm64/sgemm_small_kernel_tn_sve.c index 03406daa64..1146409504 100644 --- a/kernel/arm64/sgemm_small_kernel_tn_sve.c +++ b/kernel/arm64/sgemm_small_kernel_tn_sve.c @@ -29,7 +29,8 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -#ifdef __ARM_NEON_SVE_BRIDGE +#if defined(__ARM_NEON_SVE_BRIDGE) && defined(__has_include) && \ + __has_include() #include #else #define svdup_neonq_f32(fixed_reg) \ diff --git a/kernel/arm64/sgemm_small_kernel_tt_sve.c b/kernel/arm64/sgemm_small_kernel_tt_sve.c index c66330fb57..731c9861b8 100644 --- a/kernel/arm64/sgemm_small_kernel_tt_sve.c +++ b/kernel/arm64/sgemm_small_kernel_tt_sve.c @@ -29,7 +29,8 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -#ifdef __ARM_NEON_SVE_BRIDGE +#if defined(__ARM_NEON_SVE_BRIDGE) && defined(__has_include) && \ + __has_include() #include #else #define svdup_neonq_f32(fixed_reg) \ From c2ffd90e8c2cfc93c61d72bcef6fd19bb169e072 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 20 Jul 2024 17:31:00 +0200 Subject: [PATCH 216/267] make NAN handling depend on dummy2 parameter --- kernel/x86_64/scal_sse.S | 6 +++++- kernel/x86_64/scal_sse2.S | 7 ++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/kernel/x86_64/scal_sse.S b/kernel/x86_64/scal_sse.S index 91149af3ff..88ef4a3eda 100644 --- a/kernel/x86_64/scal_sse.S +++ b/kernel/x86_64/scal_sse.S @@ -60,8 +60,10 @@ #ifdef WINDOWS_ABI movq 40(%rsp), X movq 48(%rsp), INCX - + movq 64(%rsp), %r9 movaps %xmm3, %xmm0 +#else + movq 24(%rsp), %r9 #endif SAVEREGISTERS @@ -76,6 +78,8 @@ shufps $0, %xmm0, %xmm0 jne .L100 # Alpha != ZERO + + cmpq $1, %r9 je .L100 /* Alpha == ZERO */ cmpq $SIZE, INCX diff --git a/kernel/x86_64/scal_sse2.S b/kernel/x86_64/scal_sse2.S index b778895ba3..485e6ef462 100644 --- a/kernel/x86_64/scal_sse2.S +++ b/kernel/x86_64/scal_sse2.S @@ -48,6 +48,7 @@ #define X ARG2 #define INCX ARG3 #endif +#define FLAG %r9 #define XX %r10 #define I %rax @@ -60,8 +61,10 @@ #ifdef WINDOWS_ABI movq 40(%rsp), X movq 48(%rsp), INCX - + movq 64(%rsp), FLAG movaps %xmm3, %xmm0 +#else + movq 24(%rsp), FLAG #endif SAVEREGISTERS @@ -75,6 +78,8 @@ comisd %xmm0, %xmm1 jne .L100 # Alpha != ZERO jp .L100 # For Alpha = NaN + + cmpq $1, FLAG je .L100 # disable the Alpha=zero path as it does not handle x=inf or nan /* Alpha == ZERO */ cmpq $SIZE, INCX From c064319ecb47838babe32999adf2149ed4f5cd83 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 20 Jul 2024 17:42:31 +0200 Subject: [PATCH 217/267] fix alpha=NAN case --- kernel/x86_64/zscal_sse.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/x86_64/zscal_sse.S b/kernel/x86_64/zscal_sse.S index 8505c67bfc..acd6c36549 100644 --- a/kernel/x86_64/zscal_sse.S +++ b/kernel/x86_64/zscal_sse.S @@ -76,7 +76,7 @@ pxor %xmm15, %xmm15 comiss %xmm0, %xmm15 jne .L100 # Alpha_r != ZERO - + jp .L100 # Alpha_r == NAN comiss %xmm1, %xmm15 jne .L100 # Alpha_i != ZERO From dfbc2348a88cbc20ec8016ed65a740639c3e45ae Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 20 Jul 2024 18:27:15 +0200 Subject: [PATCH 218/267] fix NAN handling --- kernel/x86_64/scal_atom.S | 8 +++++++- kernel/x86_64/zscal_atom.S | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/kernel/x86_64/scal_atom.S b/kernel/x86_64/scal_atom.S index 11350ea191..284ea4515a 100644 --- a/kernel/x86_64/scal_atom.S +++ b/kernel/x86_64/scal_atom.S @@ -60,8 +60,10 @@ #ifdef WINDOWS_ABI movq 40(%rsp), X movq 48(%rsp), INCX - + movq 64(%rsp), %r9 movaps %xmm3, %xmm0 +#else + movq 24(%rsp), %r9 #endif SAVEREGISTERS @@ -73,6 +75,10 @@ lea (, INCX, SIZE), INCX comisd %xmm0, %xmm1 jne .L100 + jp .L100 + + cmpq $1, %r9 + je .L100 /* Alpha == ZERO */ cmpq $SIZE, INCX diff --git a/kernel/x86_64/zscal_atom.S b/kernel/x86_64/zscal_atom.S index 1649b855b8..7713626c9a 100644 --- a/kernel/x86_64/zscal_atom.S +++ b/kernel/x86_64/zscal_atom.S @@ -74,7 +74,7 @@ pxor %xmm15, %xmm15 comisd %xmm0, %xmm15 jne .L30 # Alpha_r != ZERO - + jp .L30 comisd %xmm1, %xmm15 jne .L30 # Alpha_i != ZERO From 73f8866ffba84ca7c8f75b7e59030ce0611e5528 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sun, 21 Jul 2024 13:42:47 +0200 Subject: [PATCH 219/267] make NAN handling depend on DUMMY2 parameter --- kernel/power/dscal.c | 34 +++++++++++++++++++++++++++++++--- kernel/power/sscal.c | 44 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 71 insertions(+), 7 deletions(-) diff --git a/kernel/power/dscal.c b/kernel/power/dscal.c index 2bbc1ea6d7..da80839768 100644 --- a/kernel/power/dscal.c +++ b/kernel/power/dscal.c @@ -73,6 +73,15 @@ static void dscal_kernel_8_zero (BLASLONG n, FLOAT *x) for( i=0; i= 16 ) { BLASLONG align = ((32 - ((uintptr_t)x & (uintptr_t)0x1F)) >> 3) & 0x3; + if (dummy2 == 0) + for (j = 0; j < align; j++) { + x [j] = 0.0; + } + else for (j = 0; j < align; j++) { if (isfinite(x[j])) x[j] = 0.0; @@ -151,7 +166,13 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS j=n1; } #endif - + if (dummy2 == 0) + while(j < n) + { + x[j]=0.0; + j++; + } + else while(j < n) { if (!isfinite(x[j])) @@ -202,7 +223,14 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS if ( da == 0.0 ) { - + if (dummy2 == 0) + while(j < n) + { + x[i]=0.0; + i += inc_x; + j++; + } + else while(j < n) { if (!isfinite(x[i])) diff --git a/kernel/power/sscal.c b/kernel/power/sscal.c index 12246b0a38..54047a856d 100644 --- a/kernel/power/sscal.c +++ b/kernel/power/sscal.c @@ -74,7 +74,24 @@ static void sscal_kernel_16_zero( BLASLONG n, FLOAT *x ) for( i=0; i= 32 ) { BLASLONG align = ((32 - ((uintptr_t)x & (uintptr_t)0x1F)) >> 2) & 0x7; + if (dummy2 == 0) + for (j = 0; j < align; j++){ + x[j] = 0.0; + } + else for (j = 0; j < align; j++) { if (isfinite(x[j])) x[j] = 0.0; @@ -153,9 +176,15 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS j=n1; } #endif - + if (dummy2 == 0) while(j < n) { + x[j] = 0.0; + j++; + } + else + while(j < n) + { if (isfinite(x[j])) x[j]=0.0; else @@ -204,7 +233,14 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS if ( da == 0.0 ) { - + if (dummy2 == 0) + while(j < n) + { + x[i]=0.0; + i += inc_x; + j++; + } + else while(j < n) { if (isfinite(x[i])) From 29f3e759b9796ea427029ab362fea407c04fc757 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 23 Jul 2024 11:20:48 +0200 Subject: [PATCH 220/267] work around a gcc14.1 bug observed on Loongarch --- utest/test_potrs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utest/test_potrs.c b/utest/test_potrs.c index f39287d6f3..642ce1e376 100644 --- a/utest/test_potrs.c +++ b/utest/test_potrs.c @@ -32,7 +32,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************************/ #include "openblas_utest.h" - +#pragma GCC optimize("no-gcse") /* void BLASFUNC(cpotrf)(char*, BLASINT*, complex float*, BLASINT*, BLASINT*); void BLASFUNC(zpotrs_(char*, BLASINT*, BLASINT*, complex double*, From 821ef34635db922f875b5c40dba4a12bd2b94fa5 Mon Sep 17 00:00:00 2001 From: yamazaki-mitsufumi Date: Tue, 23 Jul 2024 20:44:39 +0900 Subject: [PATCH 221/267] Add A64FX to the list of CPUs supported by DYNAMIC_ARCH --- Makefile.system | 1 + cmake/arch.cmake | 2 +- cmake/prebuild.cmake | 31 +++++++++++++++++++++++++++++++ cmake/system.cmake | 12 ++++++++++++ driver/others/dynamic_arm64.c | 21 ++++++++++++++++++++- 5 files changed, 65 insertions(+), 2 deletions(-) diff --git a/Makefile.system b/Makefile.system index 847bab1798..6c1f01ba73 100644 --- a/Makefile.system +++ b/Makefile.system @@ -689,6 +689,7 @@ ifneq ($(NO_SVE), 1) DYNAMIC_CORE += NEOVERSEV1 DYNAMIC_CORE += NEOVERSEN2 DYNAMIC_CORE += ARMV8SVE +DYNAMIC_CORE += A64FX endif DYNAMIC_CORE += THUNDERX DYNAMIC_CORE += THUNDERX2T99 diff --git a/cmake/arch.cmake b/cmake/arch.cmake index eb974456b7..27c5650abd 100644 --- a/cmake/arch.cmake +++ b/cmake/arch.cmake @@ -46,7 +46,7 @@ if (DYNAMIC_ARCH) if (ARM64) set(DYNAMIC_CORE ARMV8 CORTEXA53 CORTEXA57 THUNDERX THUNDERX2T99 TSV110 EMAG8180 NEOVERSEN1 THUNDERX3T110) if (${CMAKE_C_COMPILER_VERSION} VERSION_GREATER 9.99) - set(DYNAMIC_CORE ${DYNAMIC_CORE} NEOVERSEV1 NEOVERSEN2 ARMV8SVE) + set(DYNAMIC_CORE ${DYNAMIC_CORE} NEOVERSEV1 NEOVERSEN2 ARMV8SVE A64FX) endif () if (DYNAMIC_LIST) set(DYNAMIC_CORE ARMV8 ${DYNAMIC_LIST}) diff --git a/cmake/prebuild.cmake b/cmake/prebuild.cmake index e64352f4ae..609fbe2417 100644 --- a/cmake/prebuild.cmake +++ b/cmake/prebuild.cmake @@ -1218,6 +1218,37 @@ endif () set(ZGEMM_UNROLL_M 4) set(ZGEMM_UNROLL_N 4) set(SYMV_P 16) + elseif ("${TCORE}" STREQUAL "A64FX") + file(APPEND ${TARGET_CONF_TEMP} + "#define L1_CODE_SIZE\t65536\n" + "#define L1_CODE_LINESIZE\t256\n" + "#define L1_CODE_ASSOCIATIVE\t8\n" + "#define L1_DATA_SIZE\t32768\n" + "#define L1_DATA_LINESIZE\t256\n" + "#define L1_DATA_ASSOCIATIVE\t8\n" + "#define L2_SIZE\t8388608\n\n" + "#define L2_LINESIZE\t256\n" + "#define L2_ASSOCIATIVE\t8\n" + "#define L3_SIZE\t0\n\n" + "#define L3_LINESIZE\t0\n\n" + "#define L3_ASSOCIATIVE\t0\n\n" + "#define DTB_DEFAULT_ENTRIES\t64\n" + "#define DTB_SIZE\t4096\n" + "#define HAVE_VFPV4\n" + "#define HAVE_VFPV3\n" + "#define HAVE_VFP\n" + "#define HAVE_NEON\n" + "#define HAVE_SVE\n" + "#define ARMV8\n") + set(SGEMM_UNROLL_M 4) + set(SGEMM_UNROLL_N 8) + set(DGEMM_UNROLL_M 2) + set(DGEMM_UNROLL_N 8) + set(CGEMM_UNROLL_M 2) + set(CGEMM_UNROLL_N 4) + set(ZGEMM_UNROLL_M 2) + set(ZGEMM_UNROLL_N 4) + set(SYMV_P 16) elseif ("${TCORE}" STREQUAL "P5600") file(APPEND ${TARGET_CONF_TEMP} "#define L2_SIZE 1048576\n" diff --git a/cmake/system.cmake b/cmake/system.cmake index e4778249f3..b682c3af81 100644 --- a/cmake/system.cmake +++ b/cmake/system.cmake @@ -310,6 +310,18 @@ if (${TARGET} STREQUAL NEOVERSEV1) set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=armv8.2-a+sve") endif() endif() + if (${TARGET} STREQUAL A64FX) + if (${CMAKE_C_COMPILER_ID} STREQUAL "PGI" AND NOT NO_SVE) + set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -Msve-intrinsics -march=armv8.2-a+sve -mtune=a64fx") + else () + execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) + if (${GCC_VERSION} VERSION_GREATER 10.4 OR ${GCC_VERSION} VERSION_EQUAL 10.4) + set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=armv8.2-a+sve -mtune=a64fx") + else () + message(FATAL_ERROR "Compiler $${CMAKE_C_COMPILER} {GCC_VERSION} does not support A64FX.") + endif() + endif() + endif() endif() diff --git a/driver/others/dynamic_arm64.c b/driver/others/dynamic_arm64.c index 6b21028d12..dc88d816fb 100644 --- a/driver/others/dynamic_arm64.c +++ b/driver/others/dynamic_arm64.c @@ -120,6 +120,11 @@ extern gotoblas_t gotoblas_CORTEXA55; #else #define gotoblas_CORTEXA55 gotoblas_ARMV8 #endif +#ifdef DYN_A64FX +extern gotoblas_t gotoblas_A64FX; +#else +#define gotoblas_A64FX gotoblas_ARMV8 +#endif #else extern gotoblas_t gotoblas_CORTEXA53; #define gotoblas_CORTEXA55 gotoblas_CORTEXA53 @@ -136,10 +141,12 @@ extern gotoblas_t gotoblas_NEOVERSEN1; extern gotoblas_t gotoblas_NEOVERSEV1; extern gotoblas_t gotoblas_NEOVERSEN2; extern gotoblas_t gotoblas_ARMV8SVE; +extern gotoblas_t gotoblas_A64FX; #else #define gotoblas_NEOVERSEV1 gotoblas_ARMV8 #define gotoblas_NEOVERSEN2 gotoblas_ARMV8 #define gotoblas_ARMV8SVE gotoblas_ARMV8 +#define gotoblas_A64FX gotoblas_ARMV8 #endif extern gotoblas_t gotoblas_THUNDERX3T110; #endif @@ -149,7 +156,7 @@ extern void openblas_warning(int verbose, const char * msg); #define FALLBACK_VERBOSE 1 #define NEOVERSEN1_FALLBACK "OpenBLAS : Your OS does not support SVE instructions. OpenBLAS is using Neoverse N1 kernels as a fallback, which may give poorer performance.\n" -#define NUM_CORETYPES 17 +#define NUM_CORETYPES 18 /* * In case asm/hwcap.h is outdated on the build system, make sure @@ -184,6 +191,7 @@ static char *corename[] = { "thunderx3t110", "cortexa55", "armv8sve", + "a64fx", "unknown" }; @@ -205,6 +213,7 @@ char *gotoblas_corename(void) { if (gotoblas == &gotoblas_THUNDERX3T110) return corename[14]; if (gotoblas == &gotoblas_CORTEXA55) return corename[15]; if (gotoblas == &gotoblas_ARMV8SVE) return corename[16]; + if (gotoblas == &gotoblas_A64FX) return corename[17]; return corename[NUM_CORETYPES]; } @@ -241,6 +250,7 @@ static gotoblas_t *force_coretype(char *coretype) { case 14: return (&gotoblas_THUNDERX3T110); case 15: return (&gotoblas_CORTEXA55); case 16: return (&gotoblas_ARMV8SVE); + case 17: return (&gotoblas_A64FX); } snprintf(message, 128, "Core not found: %s\n", coretype); openblas_warning(1, message); @@ -346,6 +356,15 @@ static gotoblas_t *get_coretype(void) { return &gotoblas_THUNDERX3T110; } break; + case 0x46: // Fujitsu + switch (part) + { +#ifndef NO_SVE + case 0x001: // A64FX + return &gotoblas_A64FX; +#endif + } + break; case 0x48: // HiSilicon switch (part) { From 0096482f0383aa96f2652aa76ee98be79c920717 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 23 Jul 2024 15:01:26 +0200 Subject: [PATCH 222/267] fix incompatible definitions of MAXLOC --- lapack-netlib/SRC/cpstf2.c | 2 +- lapack-netlib/SRC/cpstrf.c | 2 +- lapack-netlib/SRC/spstf2.c | 2 +- lapack-netlib/SRC/spstrf.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lapack-netlib/SRC/cpstf2.c b/lapack-netlib/SRC/cpstf2.c index ae7fb05c4e..f983f879fb 100644 --- a/lapack-netlib/SRC/cpstf2.c +++ b/lapack-netlib/SRC/cpstf2.c @@ -256,7 +256,7 @@ static char junk[] = "\n@(#)LIBF77 VERSION 19990503\n"; #define myceiling_(w) {ceil(w)} #define myhuge_(w) {HUGE_VAL} //#define mymaxloc_(w,s,e,n) {if (sizeof(*(w)) == sizeof(double)) dmaxloc_((w),*(s),*(e),n); else dmaxloc_((w),*(s),*(e),n);} -#define mymaxloc_(w,s,e,n) dmaxloc_(w,*(s),*(e),n) +#define mymaxloc_(w,s,e,n) smaxloc_(w,*(s),*(e),n) /* procedure parameter types for -A and -C++ */ diff --git a/lapack-netlib/SRC/cpstrf.c b/lapack-netlib/SRC/cpstrf.c index ab4811273d..a271835c25 100644 --- a/lapack-netlib/SRC/cpstrf.c +++ b/lapack-netlib/SRC/cpstrf.c @@ -256,7 +256,7 @@ static char junk[] = "\n@(#)LIBF77 VERSION 19990503\n"; #define myceiling_(w) {ceil(w)} #define myhuge_(w) {HUGE_VAL} //#define mymaxloc_(w,s,e,n) {if (sizeof(*(w)) == sizeof(double)) dmaxloc_((w),*(s),*(e),n); else dmaxloc_((w),*(s),*(e),n);} -#define mymaxloc_(w,s,e,n) dmaxloc_(w,*(s),*(e),n) +#define mymaxloc_(w,s,e,n) smaxloc_(w,*(s),*(e),n) /* procedure parameter types for -A and -C++ */ diff --git a/lapack-netlib/SRC/spstf2.c b/lapack-netlib/SRC/spstf2.c index 7b2abce036..c9e01583a2 100644 --- a/lapack-netlib/SRC/spstf2.c +++ b/lapack-netlib/SRC/spstf2.c @@ -256,7 +256,7 @@ static char junk[] = "\n@(#)LIBF77 VERSION 19990503\n"; #define myceiling_(w) {ceil(w)} #define myhuge_(w) {HUGE_VAL} //#define mymaxloc_(w,s,e,n) {if (sizeof(*(w)) == sizeof(double)) dmaxloc_((w),*(s),*(e),n); else dmaxloc_((w),*(s),*(e),n);} -#define mymaxloc_(w,s,e,n) dmaxloc_(w,*(s),*(e),n) +#define mymaxloc_(w,s,e,n) smaxloc_(w,*(s),*(e),n) /* procedure parameter types for -A and -C++ */ diff --git a/lapack-netlib/SRC/spstrf.c b/lapack-netlib/SRC/spstrf.c index 75e1d65b7e..ebb4fe9345 100644 --- a/lapack-netlib/SRC/spstrf.c +++ b/lapack-netlib/SRC/spstrf.c @@ -256,7 +256,7 @@ static char junk[] = "\n@(#)LIBF77 VERSION 19990503\n"; #define myceiling_(w) {ceil(w)} #define myhuge_(w) {HUGE_VAL} //#define mymaxloc_(w,s,e,n) {if (sizeof(*(w)) == sizeof(double)) dmaxloc_((w),*(s),*(e),n); else dmaxloc_((w),*(s),*(e),n);} -#define mymaxloc_(w,s,e,n) dmaxloc_(w,*(s),*(e),n) +#define mymaxloc_(w,s,e,n) smaxloc_(w,*(s),*(e),n) /* procedure parameter types for -A and -C++ */ From b613754143d68a2635f0232f29a04318f15e34d3 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 24 Jul 2024 14:31:29 +0200 Subject: [PATCH 223/267] Update scal..c --- kernel/arm/scal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/arm/scal.c b/kernel/arm/scal.c index 1f96f9b958..6a2c37631e 100644 --- a/kernel/arm/scal.c +++ b/kernel/arm/scal.c @@ -43,7 +43,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS if ( (n <= 0) || (inc_x <= 0)) return(0); - if (dummy2 == 0) + if (dummy2 == 0) { while(j < n) { From 88caf02f6286c7e84948b788c01bed2cb79ba86d Mon Sep 17 00:00:00 2001 From: yamazaki-mitsufumi Date: Thu, 25 Jul 2024 22:43:13 +0900 Subject: [PATCH 224/267] Fix ambiguous error on Mac OS --- kernel/arm64/gemv_n_sve.c | 4 ++-- kernel/arm64/gemv_t_sve.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/arm64/gemv_n_sve.c b/kernel/arm64/gemv_n_sve.c index d3aa57ae32..2950555615 100644 --- a/kernel/arm64/gemv_n_sve.c +++ b/kernel/arm64/gemv_n_sve.c @@ -37,13 +37,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define SV_COUNT svcntd #define SV_TYPE svfloat64_t #define SV_TRUE svptrue_b64 -#define SV_WHILE svwhilelt_b64 +#define SV_WHILE svwhilelt_b64_s64 #define SV_DUP svdup_f64 #else #define SV_COUNT svcntw #define SV_TYPE svfloat32_t #define SV_TRUE svptrue_b32 -#define SV_WHILE svwhilelt_b32 +#define SV_WHILE svwhilelt_b32_s64 #define SV_DUP svdup_f32 #endif diff --git a/kernel/arm64/gemv_t_sve.c b/kernel/arm64/gemv_t_sve.c index bff08b2579..ab700a3748 100644 --- a/kernel/arm64/gemv_t_sve.c +++ b/kernel/arm64/gemv_t_sve.c @@ -37,13 +37,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define SV_COUNT svcntd #define SV_TYPE svfloat64_t #define SV_TRUE svptrue_b64 -#define SV_WHILE svwhilelt_b64 +#define SV_WHILE svwhilelt_b64_s64 #define SV_DUP svdup_f64 #else #define SV_COUNT svcntw #define SV_TYPE svfloat32_t #define SV_TRUE svptrue_b32 -#define SV_WHILE svwhilelt_b32 +#define SV_WHILE svwhilelt_b32_s64 #define SV_DUP svdup_f32 #endif From 24acdd6bbbd0f9726d867851f62e75ed4c35644a Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 26 Jul 2024 09:49:24 +0200 Subject: [PATCH 225/267] correct offset --- kernel/x86/scal.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/x86/scal.S b/kernel/x86/scal.S index 7e12a52ab6..6620d31693 100644 --- a/kernel/x86/scal.S +++ b/kernel/x86/scal.S @@ -65,7 +65,7 @@ #else movl 32(%esp),%edi movl 36(%esp),%esi - movl 54(%esp),%ecx + movl 52(%esp),%ecx #endif ftst From a875304eb0da7cc7966cf45317cf57392192430f Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 26 Jul 2024 09:50:20 +0200 Subject: [PATCH 226/267] fix inverted conditional for NAN handling --- kernel/riscv64/scal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/riscv64/scal.c b/kernel/riscv64/scal.c index bebbed67eb..bd53fcff9f 100644 --- a/kernel/riscv64/scal.c +++ b/kernel/riscv64/scal.c @@ -43,7 +43,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS if ( (n <= 0) || (inc_x <= 0)) return(0); - if (dummy2 == 0) { + if (dummy2 == 1) { while(j < n) { From d9ae4609fb4c5623b30b06abcfef5f5e1ae30ab1 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 26 Jul 2024 11:15:33 +0200 Subject: [PATCH 227/267] remove C99 requirement --- utest/test_gemv.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/utest/test_gemv.c b/utest/test_gemv.c index c85ef3f389..d5b8d2c6c4 100644 --- a/utest/test_gemv.c +++ b/utest/test_gemv.c @@ -12,6 +12,7 @@ CTEST(sgemv, 0_nan_inf) { + int i; blasint N = 17; blasint incX = 1; blasint incY = 1; @@ -24,19 +25,20 @@ CTEST(sgemv, 0_nan_inf) memset(A, 0, sizeof(A)); memset(X, 0, sizeof(X)); - for (int i = 0; i < (N - 1); i += 2) + for (i = 0; i < (N - 1); i += 2) { Y[i] = NAN; Y[i + 1] = INFINITY; } Y[N - 1] = NAN; BLASFUNC(sgemv)(&trans, &N, &N, &alpha, A, &N, X, &incX, &beta, Y, &incY); - for (int i = 0; i < N; i ++) + for (i = 0; i < N; i ++) ASSERT_TRUE(Y[i] == 0.0); } CTEST(sgemv, 0_nan_inf_incy_2) { + int i; blasint N = 17; blasint Ny = 33; blasint incX = 1; @@ -52,7 +54,7 @@ CTEST(sgemv, 0_nan_inf_incy_2) memset(A, 0, sizeof(A)); memset(X, 0, sizeof(X)); memset(Y, 0, sizeof(Y)); - for (int i = 0; i < (N - 1); i += 2) + for (i = 0; i < (N - 1); i += 2) { ay[0] = NAN; ay += 2; @@ -61,7 +63,7 @@ CTEST(sgemv, 0_nan_inf_incy_2) } Y[Ny - 1] = NAN; BLASFUNC(sgemv)(&trans, &N, &N, &alpha, A, &N, X, &incX, &beta, Y, &incY); - for (int i = 0; i < Ny; i ++) + for (i = 0; i < Ny; i ++) ASSERT_TRUE(Y[i] == 0.0); } @@ -70,6 +72,7 @@ CTEST(sgemv, 0_nan_inf_incy_2) #ifdef BUILD_DOUBLE CTEST(dgemv, 0_nan_inf) { + int i; blasint N = 17; blasint incX = 1; blasint incY = 1; @@ -82,19 +85,20 @@ CTEST(dgemv, 0_nan_inf) memset(A, 0, sizeof(A)); memset(X, 0, sizeof(X)); - for (int i = 0; i < (N - 1); i += 2) + for (i = 0; i < (N - 1); i += 2) { Y[i] = NAN; Y[i + 1] = INFINITY; } Y[N - 1] = NAN; BLASFUNC(dgemv)(&trans, &N, &N, &alpha, A, &N, X, &incX, &beta, Y, &incY); - for (int i = 0; i < N; i ++) + for (i = 0; i < N; i ++) ASSERT_TRUE(Y[i] == 0.0); } CTEST(dgemv, 0_nan_inf_incy_2) { + int i; blasint N = 17; blasint Ny = 33; blasint incX = 1; @@ -110,7 +114,7 @@ CTEST(dgemv, 0_nan_inf_incy_2) memset(A, 0, sizeof(A)); memset(X, 0, sizeof(X)); memset(Y, 0, sizeof(Y)); - for (int i = 0; i < (N - 1); i += 2) + for (i = 0; i < (N - 1); i += 2) { ay[0] = NAN; ay += 2; @@ -119,7 +123,7 @@ CTEST(dgemv, 0_nan_inf_incy_2) } Y[Ny - 1] = NAN; BLASFUNC(dgemv)(&trans, &N, &N, &alpha, A, &N, X, &incX, &beta, Y, &incY); - for (int i = 0; i < Ny; i ++) + for (i = 0; i < Ny; i ++) ASSERT_TRUE(Y[i] == 0.0); } From db5328e85bb1a9e11781ab92ce15273c28dffec0 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 26 Jul 2024 12:45:39 +0200 Subject: [PATCH 228/267] make array dimensions constant --- utest/test_gemv.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/utest/test_gemv.c b/utest/test_gemv.c index d5b8d2c6c4..deae6d2a87 100644 --- a/utest/test_gemv.c +++ b/utest/test_gemv.c @@ -7,13 +7,14 @@ #ifndef INFINITY #define INFINITY 1.0/0.0 #endif +#define N 17 +#define Ny 33 #ifdef BUILD_SINGLE CTEST(sgemv, 0_nan_inf) { int i; - blasint N = 17; blasint incX = 1; blasint incY = 1; float alpha = 0.0; @@ -39,8 +40,6 @@ CTEST(sgemv, 0_nan_inf) CTEST(sgemv, 0_nan_inf_incy_2) { int i; - blasint N = 17; - blasint Ny = 33; blasint incX = 1; blasint incY = 2; float alpha = 0.0; @@ -73,7 +72,6 @@ CTEST(sgemv, 0_nan_inf_incy_2) CTEST(dgemv, 0_nan_inf) { int i; - blasint N = 17; blasint incX = 1; blasint incY = 1; double alpha = 0.0; @@ -99,8 +97,6 @@ CTEST(dgemv, 0_nan_inf) CTEST(dgemv, 0_nan_inf_incy_2) { int i; - blasint N = 17; - blasint Ny = 33; blasint incX = 1; blasint incY = 2; double alpha = 0.0; From 700649286351e763ff1263867e962a20e60987fc Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 26 Jul 2024 12:49:57 +0200 Subject: [PATCH 229/267] replace "Preview" in the MSVC vcvarsall path with "Community" --- docs/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/install.md b/docs/install.md index 1b04165c73..ffb4659d82 100644 --- a/docs/install.md +++ b/docs/install.md @@ -245,7 +245,7 @@ newer installed. On Windows 11 with Visual Studio 2022, this would be done by invoking: ```shell - "c:\Program Files\Microsoft Visual Studio\2022\Preview\vc\Auxiliary\Build\vcvars64.bat" + "c:\Program Files\Microsoft Visual Studio\2022\Community\vc\Auxiliary\Build\vcvars64.bat" ``` With VS2019, the command should be the same (except for the year number of course). From a090011fbff0d15b63a7edcc0156ed790e87f4b1 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 26 Jul 2024 12:56:12 +0200 Subject: [PATCH 230/267] just use numeric constants in dimensions --- utest/test_gemv.c | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/utest/test_gemv.c b/utest/test_gemv.c index deae6d2a87..dab6d2f11f 100644 --- a/utest/test_gemv.c +++ b/utest/test_gemv.c @@ -7,22 +7,21 @@ #ifndef INFINITY #define INFINITY 1.0/0.0 #endif -#define N 17 -#define Ny 33 #ifdef BUILD_SINGLE CTEST(sgemv, 0_nan_inf) { int i; + blasint N = 17; blasint incX = 1; blasint incY = 1; float alpha = 0.0; float beta = 0.0; char trans = 'N'; - float A[N * N]; - float X[N]; - float Y[N]; + float A[17 * 17]; + float X[17]; + float Y[17]; memset(A, 0, sizeof(A)); memset(X, 0, sizeof(X)); @@ -40,14 +39,16 @@ CTEST(sgemv, 0_nan_inf) CTEST(sgemv, 0_nan_inf_incy_2) { int i; + blasint N = 17; + blasint Ny = 33; blasint incX = 1; blasint incY = 2; float alpha = 0.0; float beta = 0.0; char trans = 'N'; - float A[N * N]; - float X[N]; - float Y[Ny]; + float A[17 * 17]; + float X[17]; + float Y[33]; float *ay = Y; memset(A, 0, sizeof(A)); @@ -72,14 +73,15 @@ CTEST(sgemv, 0_nan_inf_incy_2) CTEST(dgemv, 0_nan_inf) { int i; + blasint N = 17; blasint incX = 1; blasint incY = 1; double alpha = 0.0; double beta = 0.0; char trans = 'N'; - double A[N * N]; - double X[N]; - double Y[N]; + double A[17 * 17]; + double X[17]; + double Y[17]; memset(A, 0, sizeof(A)); memset(X, 0, sizeof(X)); @@ -97,14 +99,16 @@ CTEST(dgemv, 0_nan_inf) CTEST(dgemv, 0_nan_inf_incy_2) { int i; + blasint N = 17; + blasint Ny = 33; blasint incX = 1; blasint incY = 2; double alpha = 0.0; double beta = 0.0; char trans = 'N'; - double A[N * N]; - double X[N]; - double Y[Ny]; + double A[17 * 17]; + double X[17]; + double Y[33]; double *ay = Y; memset(A, 0, sizeof(A)); From 4460d3ee7f2324d2642350a9d1650776d4fac981 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 26 Jul 2024 15:07:52 +0200 Subject: [PATCH 231/267] re-enable the sgesdd benchmark --- benchmark/pybench/benchmarks/bench_blas.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/benchmark/pybench/benchmarks/bench_blas.py b/benchmark/pybench/benchmarks/bench_blas.py index 628c0cb2a6..8127dd0c7d 100644 --- a/benchmark/pybench/benchmarks/bench_blas.py +++ b/benchmark/pybench/benchmarks/bench_blas.py @@ -234,14 +234,10 @@ def test_gesdd(benchmark, mn, variant): gesdd = ow.get_func('gesdd', variant) u, s, vt, info = benchmark(run_gesdd, a, lwork, gesdd) - if variant != 's': - # On entry to SLASCL parameter number 4 had an illegal value - # under codspeed (cannot repro locally or on CI w/o codspeed) - # https://github.com/OpenMathLib/OpenBLAS/issues/4776 - assert info == 0 - - atol = {'s': 1e-5, 'd': 1e-13} - np.testing.assert_allclose(u @ np.diag(s) @ vt, a, atol=atol[variant]) + assert info == 0 + + atol = {'s': 1e-5, 'd': 1e-13} + np.testing.assert_allclose(u @ np.diag(s) @ vt, a, atol=atol[variant]) # linalg.eigh From 175008caf8c58a35688a08a6200872772da146e0 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 27 Jul 2024 19:08:02 +0200 Subject: [PATCH 232/267] harden against a dashed suffix to the gcc version number --- c_check | 3 +++ 1 file changed, 3 insertions(+) diff --git a/c_check b/c_check index 114eed60c2..c2b52c81b0 100755 --- a/c_check +++ b/c_check @@ -356,6 +356,9 @@ if [ "$compiler" = "GCC" ]; then no_avx2=0 oldgcc=0 data=`$compiler_name -dumpversion` + case "$data" in *-*) + data="${data%-*}" + esac case "$data" in *.*.*) data="${data%.*}" esac From 85ca003ae7174ac00857c9c75ddfd53589c4918e Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Mon, 29 Jul 2024 11:14:25 +0100 Subject: [PATCH 233/267] Add fallback compile options for A64FX target --- Makefile.arm64 | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile.arm64 b/Makefile.arm64 index 6ba63e94e0..2c68d215f5 100644 --- a/Makefile.arm64 +++ b/Makefile.arm64 @@ -276,12 +276,19 @@ endif endif endif -ifeq (1, $(filter 1,$(GCCVERSIONGTEQ11) $(ISCLANG))) ifeq ($(CORE), A64FX) +ifeq (1, $(filter 1,$(GCCVERSIONGTEQ10) $(ISCLANG))) +ifeq (1, $(filter 1,$(GCCMINORVERSIONGTEQ3))) CCOMMON_OPT += -march=armv8.2-a+sve -mtune=a64fx ifneq ($(F_COMPILER), NAG) FCOMMON_OPT += -march=armv8.2-a+sve -mtune=a64fx endif +else +CCOMMON_OPT += -march=armv8.4-a+sve -mtune=neoverse-n1 +ifneq ($(F_COMPILER), NAG) +FCOMMON_OPT += -march=armv8.4-a -mtune=neoverse-n1 +endif +endif endif endif From 3ed226d3f8a724b163a69986a679e329b2fce056 Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Mon, 29 Jul 2024 11:32:59 +0100 Subject: [PATCH 234/267] Re-add ISCLANG filter --- Makefile.arm64 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.arm64 b/Makefile.arm64 index 2c68d215f5..d3ea7522d9 100644 --- a/Makefile.arm64 +++ b/Makefile.arm64 @@ -278,7 +278,7 @@ endif ifeq ($(CORE), A64FX) ifeq (1, $(filter 1,$(GCCVERSIONGTEQ10) $(ISCLANG))) -ifeq (1, $(filter 1,$(GCCMINORVERSIONGTEQ3))) +ifeq (1, $(filter 1,$(GCCMINORVERSIONGTEQ3) $(ISCLANG))) CCOMMON_OPT += -march=armv8.2-a+sve -mtune=a64fx ifneq ($(F_COMPILER), NAG) FCOMMON_OPT += -march=armv8.2-a+sve -mtune=a64fx From 54ce33e85110911ab56f1b67c681e84cfe9b07fc Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Mon, 29 Jul 2024 15:28:59 +0100 Subject: [PATCH 235/267] Fix GCC11 check for A64FX target --- Makefile.arm64 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.arm64 b/Makefile.arm64 index d3ea7522d9..fccc0d0d0f 100644 --- a/Makefile.arm64 +++ b/Makefile.arm64 @@ -278,7 +278,7 @@ endif ifeq ($(CORE), A64FX) ifeq (1, $(filter 1,$(GCCVERSIONGTEQ10) $(ISCLANG))) -ifeq (1, $(filter 1,$(GCCMINORVERSIONGTEQ3) $(ISCLANG))) +ifeq (1, $(filter 1,$(GCCMINORVERSIONGTEQ3) $(GCCVERSIONGTEQ11) $(ISCLANG))) CCOMMON_OPT += -march=armv8.2-a+sve -mtune=a64fx ifneq ($(F_COMPILER), NAG) FCOMMON_OPT += -march=armv8.2-a+sve -mtune=a64fx From a13015b65689b71136fc71c0de8aac9f78be4b85 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 30 Jul 2024 19:10:18 +0200 Subject: [PATCH 236/267] try requesting ubuntu22 instead of latest --- Jenkinsfile.pwr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile.pwr b/Jenkinsfile.pwr index 96e18b8ad6..ea27ef98f3 100644 --- a/Jenkinsfile.pwr +++ b/Jenkinsfile.pwr @@ -1,7 +1,7 @@ pipeline { agent { docker { - image 'osuosl/ubuntu-ppc64le' + image 'osuosl/ubuntu-ppc64le:22.04' } } stages { From 86c15f028b37bc4285f0235c635b62aa639a6c82 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 30 Jul 2024 21:21:34 +0200 Subject: [PATCH 237/267] Update Jenkinsfile.pwr --- Jenkinsfile.pwr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile.pwr b/Jenkinsfile.pwr index ea27ef98f3..b2f8ce2e5d 100644 --- a/Jenkinsfile.pwr +++ b/Jenkinsfile.pwr @@ -1,7 +1,7 @@ pipeline { agent { docker { - image 'osuosl/ubuntu-ppc64le:22.04' + image 'osuosl/ubuntu-ppc64le:18.04' } } stages { From 3db5dbc88e62ff6676202eb80645fe41e901f08d Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Mon, 20 May 2024 22:40:04 +0200 Subject: [PATCH 238/267] forward to GEMV when one argument is actually a vector --- interface/gemm.c | 49 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/interface/gemm.c b/interface/gemm.c index 4537b6a78f..c3118672f5 100644 --- a/interface/gemm.c +++ b/interface/gemm.c @@ -47,22 +47,29 @@ #define SMP_THRESHOLD_MIN 65536.0 #ifdef XDOUBLE #define ERROR_NAME "QGEMM " +#define GEMV BLASFUNC(qgemv) #elif defined(DOUBLE) #define ERROR_NAME "DGEMM " +#define GEMV BLASFUNC(dgemv) #elif defined(BFLOAT16) #define ERROR_NAME "SBGEMM " +#define GEMV BLASFUNC(sbgemv) #else #define ERROR_NAME "SGEMM " +#define GEMV BLASFUNC(sgemv) #endif #else #define SMP_THRESHOLD_MIN 8192.0 #ifndef GEMM3M #ifdef XDOUBLE #define ERROR_NAME "XGEMM " +#define GEMV BLASFUNC(xgemv) #elif defined(DOUBLE) #define ERROR_NAME "ZGEMM " +#define GEMV BLASFUNC(zgemv) #else #define ERROR_NAME "CGEMM " +#define GEMV BLASFUNC(cgemv) #endif #else #ifdef XDOUBLE @@ -485,9 +492,38 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANS } #endif #endif // defined(__linux__) && defined(__x86_64__) && defined(BFLOAT16) + // fprintf(stderr,"G E M M interface m n k %d %d %d\n",args.m,args.n,args.k); if ((args.m == 0) || (args.n == 0)) return; +#if 1 +#ifndef GEMM3M + if (args.m == 1) { + char *NT=(char*)malloc(2*sizeof(char)); + if (transb&1)strcpy(NT,"T"); + else NT="N"; +// fprintf(stderr,"G E M V\n"); + GEMV(NT, &args.n ,&args.k, args.alpha, args.b, &args.ldb, args.a, &args.m, args.beta, args.c, &args.m); +//SUBROUTINE SGEMV(TRANS,M,N,ALPHA,A,LDA,X,INCX,BETA,Y,INCY) +return; + } else { + if (args.n == 1) { +#ifndef CBLAS + char *NT=(char*)malloc(2*sizeof(char)); + strcpy(NT,"N"); +#else + char *NT=(char*)malloc(2*sizeof(char)); + if (transb&1)strcpy(NT,"T"); + else strcpy(NT,"N"); +#endif +// fprintf(stderr,"G E M V ! ! ! lda=%d ldb=%d ldc=%d\n",args.lda,args.ldb,args.ldc); + GEMV(NT, &args.m ,&args.k, args.alpha, args.a, &args.lda, args.b, &args.n, args.beta, args.c, &args.n); +//SUBROUTINE SGEMV(TRANS,M,N,ALPHA,A,LDA,X,INCX,BETA,Y,INCY) + return; + } + } +#endif +#endif #if 0 fprintf(stderr, "m = %4d n = %d k = %d lda = %4d ldb = %4d ldc = %4d\n", args.m, args.n, args.k, args.lda, args.ldb, args.ldc); @@ -521,10 +557,15 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANS buffer = (XFLOAT *)blas_memory_alloc(0); -//For target LOONGSON3R5, applying an offset to the buffer is essential -//for minimizing cache conflicts and optimizing performance. -#if defined(ARCH_LOONGARCH64) && !defined(NO_AFFINITY) - sa = (XFLOAT *)((BLASLONG)buffer + (WhereAmI() & 0xf) * GEMM_OFFSET_A); +//For Loongson servers, like the 3C5000 (featuring 16 cores), applying an +//offset to the buffer is essential for minimizing cache conflicts and optimizing performance. +#if defined(LOONGSON3R5) && !defined(NO_AFFINITY) + char model_name[128]; + get_cpu_model(model_name); + if ((strstr(model_name, "3C5000") != NULL) || (strstr(model_name, "3D5000") != NULL)) + sa = (XFLOAT *)((BLASLONG)buffer + (WhereAmI() & 0xf) * GEMM_OFFSET_A); + else + sa = (XFLOAT *)((BLASLONG)buffer + GEMM_OFFSET_A); #else sa = (XFLOAT *)((BLASLONG)buffer +GEMM_OFFSET_A); #endif From 28b5334f22dc972613493bb8cd207657c002e240 Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Tue, 23 Jul 2024 20:42:39 +0000 Subject: [PATCH 239/267] Complete implementation of GEMV forwarding --- interface/gemm.c | 89 +++++++++++++++++++++++++----------------------- 1 file changed, 47 insertions(+), 42 deletions(-) diff --git a/interface/gemm.c b/interface/gemm.c index c3118672f5..c8a20d119a 100644 --- a/interface/gemm.c +++ b/interface/gemm.c @@ -1,4 +1,5 @@ /*********************************************************************/ +/* Copyright 2024 The OpenBLAS Project */ /* Copyright 2009, 2010 The University of Texas at Austin. */ /* All rights reserved. */ /* */ @@ -63,13 +64,10 @@ #ifndef GEMM3M #ifdef XDOUBLE #define ERROR_NAME "XGEMM " -#define GEMV BLASFUNC(xgemv) #elif defined(DOUBLE) #define ERROR_NAME "ZGEMM " -#define GEMV BLASFUNC(zgemv) #else #define ERROR_NAME "CGEMM " -#define GEMV BLASFUNC(cgemv) #endif #else #ifdef XDOUBLE @@ -492,42 +490,54 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANS } #endif #endif // defined(__linux__) && defined(__x86_64__) && defined(BFLOAT16) - // fprintf(stderr,"G E M M interface m n k %d %d %d\n",args.m,args.n,args.k); if ((args.m == 0) || (args.n == 0)) return; -#if 1 -#ifndef GEMM3M - if (args.m == 1) { - char *NT=(char*)malloc(2*sizeof(char)); - if (transb&1)strcpy(NT,"T"); - else NT="N"; -// fprintf(stderr,"G E M V\n"); - GEMV(NT, &args.n ,&args.k, args.alpha, args.b, &args.ldb, args.a, &args.m, args.beta, args.c, &args.m); -//SUBROUTINE SGEMV(TRANS,M,N,ALPHA,A,LDA,X,INCX,BETA,Y,INCY) -return; - } else { - if (args.n == 1) { -#ifndef CBLAS - char *NT=(char*)malloc(2*sizeof(char)); - strcpy(NT,"N"); -#else - char *NT=(char*)malloc(2*sizeof(char)); - if (transb&1)strcpy(NT,"T"); - else strcpy(NT,"N"); -#endif -// fprintf(stderr,"G E M V ! ! ! lda=%d ldb=%d ldc=%d\n",args.lda,args.ldb,args.ldc); - GEMV(NT, &args.m ,&args.k, args.alpha, args.a, &args.lda, args.b, &args.n, args.beta, args.c, &args.n); -//SUBROUTINE SGEMV(TRANS,M,N,ALPHA,A,LDA,X,INCX,BETA,Y,INCY) - return; +#if !defined(GEMM3M) && !defined(COMPLEX) + // Check if we can convert GEMM -> GEMV + if (args.k != 0) { + if (args.n == 1) { + blasint inc_x = 1; + blasint inc_y = 1; + // These were passed in as blasint, but the struct translates them to blaslong + blasint m = args.m; + blasint n = args.k; + blasint lda = args.lda; + // Create new transpose parameters + char NT = 'N'; + if (transa & 1) { + NT = 'T'; + m = args.k; + n = args.m; + } + if (transb & 1) { + inc_x = args.ldb; + } + GEMV(&NT, &m, &n, args.alpha, args.a, &lda, args.b, &inc_x, args.beta, args.c, &inc_y); + return; + } + if (args.m == 1) { + blasint inc_x = args.lda; + blasint inc_y = args.ldc; + // These were passed in as blasint, but the struct translates them to blaslong + blasint m = args.k; + blasint n = args.n; + blasint ldb = args.ldb; + // Create new transpose parameters + char NT = 'T'; + if (transa & 1) { + inc_x = 1; + } + if (transb & 1) { + NT = 'N'; + m = args.n; + n = args.k; + } + GEMV(&NT, &m, &n, args.alpha, args.b, &ldb, args.a, &inc_x, args.beta, args.c, &inc_y); + return; } } #endif -#endif -#if 0 - fprintf(stderr, "m = %4d n = %d k = %d lda = %4d ldb = %4d ldc = %4d\n", - args.m, args.n, args.k, args.lda, args.ldb, args.ldc); -#endif IDEBUG_START; @@ -557,15 +567,10 @@ return; buffer = (XFLOAT *)blas_memory_alloc(0); -//For Loongson servers, like the 3C5000 (featuring 16 cores), applying an -//offset to the buffer is essential for minimizing cache conflicts and optimizing performance. -#if defined(LOONGSON3R5) && !defined(NO_AFFINITY) - char model_name[128]; - get_cpu_model(model_name); - if ((strstr(model_name, "3C5000") != NULL) || (strstr(model_name, "3D5000") != NULL)) - sa = (XFLOAT *)((BLASLONG)buffer + (WhereAmI() & 0xf) * GEMM_OFFSET_A); - else - sa = (XFLOAT *)((BLASLONG)buffer + GEMM_OFFSET_A); +//For target LOONGSON3R5, applying an offset to the buffer is essential +//for minimizing cache conflicts and optimizing performance. +#if defined(ARCH_LOONGARCH64) && !defined(NO_AFFINITY) + sa = (XFLOAT *)((BLASLONG)buffer + (WhereAmI() & 0xf) * GEMM_OFFSET_A); #else sa = (XFLOAT *)((BLASLONG)buffer +GEMM_OFFSET_A); #endif From 90eb863d4b3d836b5b4a42202fcca84283d9ea4a Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Tue, 23 Jul 2024 23:39:07 +0100 Subject: [PATCH 240/267] Re-add accidental removal --- interface/gemm.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/interface/gemm.c b/interface/gemm.c index c8a20d119a..7004622a8d 100644 --- a/interface/gemm.c +++ b/interface/gemm.c @@ -493,6 +493,11 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANS if ((args.m == 0) || (args.n == 0)) return; +#if 0 + fprintf(stderr, "m = %4d n = %d k = %d lda = %4d ldb = %4d ldc = %4d\n", + args.m, args.n, args.k, args.lda, args.ldb, args.ldc); +#endif + #if !defined(GEMM3M) && !defined(COMPLEX) // Check if we can convert GEMM -> GEMV if (args.k != 0) { From b26424c6a20f02c3d7269a021c9b648f1fe04be3 Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Wed, 24 Jul 2024 11:25:28 +0100 Subject: [PATCH 241/267] Allow opt into GEMM -> GEMV forwarding --- Makefile.system | 9 +++++++++ cmake/system.cmake | 7 +++++++ interface/gemm.c | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/Makefile.system b/Makefile.system index 9c42dc115b..436245c356 100644 --- a/Makefile.system +++ b/Makefile.system @@ -274,9 +274,18 @@ endif ifeq ($(ARCH), loongarch64) SMALL_MATRIX_OPT = 1 endif +ifeq ($(ARCH), arm64) +GEMM_GEMV_FORWARD = 1 +endif + ifeq ($(SMALL_MATRIX_OPT), 1) CCOMMON_OPT += -DSMALL_MATRIX_OPT endif +ifeq ($(GEMM_GEMV_FORWARD), 1) +ifneq ($(ONLY_CBLAS), 1) +CCOMMON_OPT += -DGEMM_GEMV_FORWARD +endif +endif # This operation is expensive, so execution should be once. ifndef GOTOBLAS_MAKEFILE diff --git a/cmake/system.cmake b/cmake/system.cmake index b682c3af81..efb7aef94b 100644 --- a/cmake/system.cmake +++ b/cmake/system.cmake @@ -391,6 +391,13 @@ endif () if (X86_64 OR ${CORE} STREQUAL POWER10) set(SMALL_MATRIX_OPT TRUE) endif () +if (ARM64) + set(GEMM_GEMV_FORWARD TRUE) +endif () + +if (GEMM_GEMV_FORWARD AND NOT ONLY_CBLAS) + set(CCOMMON_OPT "${CCOMMON_OPT} -DGEMM_GEMV_FORWARD") +endif () if (SMALL_MATRIX_OPT) set(CCOMMON_OPT "${CCOMMON_OPT} -DSMALL_MATRIX_OPT") endif () diff --git a/interface/gemm.c b/interface/gemm.c index 7004622a8d..ac58cf27fa 100644 --- a/interface/gemm.c +++ b/interface/gemm.c @@ -498,7 +498,7 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANS args.m, args.n, args.k, args.lda, args.ldb, args.ldc); #endif -#if !defined(GEMM3M) && !defined(COMPLEX) +#if defined(GEMM_GEMV_FORWARD) && !defined(GEMM3M) && !defined(COMPLEX) // Check if we can convert GEMM -> GEMV if (args.k != 0) { if (args.n == 1) { From ba2e989c67f9d2b3403c6fbaebc05b504e601fab Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Wed, 31 Jul 2024 13:07:35 +0100 Subject: [PATCH 242/267] Add accumulators to AArch64 GEMV Kernels This helps to reduce values going missing as we accumulate. --- kernel/arm64/KERNEL.NEOVERSEV1 | 3 ++ kernel/arm64/gemv_t.S | 67 +++++++++++++++++++--------------- kernel/arm64/gemv_t_sve.c | 40 ++++++++++++++++---- 3 files changed, 74 insertions(+), 36 deletions(-) diff --git a/kernel/arm64/KERNEL.NEOVERSEV1 b/kernel/arm64/KERNEL.NEOVERSEV1 index bc59990979..53d157a0aa 100644 --- a/kernel/arm64/KERNEL.NEOVERSEV1 +++ b/kernel/arm64/KERNEL.NEOVERSEV1 @@ -1 +1,4 @@ include $(KERNELDIR)/KERNEL.ARMV8SVE + +SGEMVTKERNEL = gemv_t_sve.c +DGEMVTKERNEL = gemv_t_sve.c diff --git a/kernel/arm64/gemv_t.S b/kernel/arm64/gemv_t.S index b04367ab37..a98eef49b0 100644 --- a/kernel/arm64/gemv_t.S +++ b/kernel/arm64/gemv_t.S @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2015, The OpenBLAS Project +Copyright (c) 2015, 2024 The OpenBLAS Project All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -170,39 +170,48 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .macro KERNEL_F32_FINALIZE #if !defined(DOUBLE) - fadd v1.4s, v1.4s, v2.4s + // F8 only has 2 accumulators + // so add into those pairs fadd v1.4s, v1.4s, v3.4s - fadd v1.4s, v1.4s, v4.4s -#else - fadd v1.2d, v1.2d, v2.2d - fadd v1.2d, v1.2d, v3.2d - fadd v1.2d, v1.2d, v4.2d + fadd v2.4s, v2.4s, v4.4s #endif .endm -.macro KERNEL_F4 +.macro KERNEL_F8 #if !defined(DOUBLE) - ld1 {v2.4s}, [A_PTR], #16 - ld1 {v3.4s}, [X_PTR], #16 - fmla v1.4s, v2.4s, v3.4s -#else - ld1 {v2.2d}, [A_PTR], #16 - ld1 {v3.2d}, [X_PTR], #16 - fmla v1.2d, v2.2d, v3.2d - - ld1 {v4.2d}, [A_PTR], #16 - ld1 {v5.2d}, [X_PTR], #16 - fmla v1.2d, v4.2d, v5.2d + ld1 {v13.4s, v14.4s}, [A_PTR], #32 + ld1 {v17.4s, v18.4s}, [X_PTR], #32 + fmla v1.4s, v13.4s, v17.4s + fmla v2.4s, v14.4s, v18.4s +#else + ld1 {v13.2d, v14.2d, v15.2d, v16.2d}, [A_PTR], #64 + ld1 {v17.2d, v18.2d, v19.2d, v20.2d}, [X_PTR], #64 + fmla v1.2d, v13.2d, v17.2d + fmla v2.2d, v14.2d, v18.2d + fmla v3.2d, v15.2d, v19.2d + fmla v4.2d, v16.2d, v20.2d #endif .endm -.macro KERNEL_F4_FINALIZE +.macro KERNEL_F8_FINALIZE #if !defined(DOUBLE) - ext v2.16b, v1.16b, v1.16b, #8 + // Take the top two elements of v1 and + // put them into the first two lanes of v3 + ext v3.16b, v1.16b, v1.16b, #8 + fadd v1.2s, v1.2s, v3.2s + ext v4.16b, v2.16b, v2.16b, #8 + fadd v2.2s, v2.2s, v4.2s + // Final pair fadd v1.2s, v1.2s, v2.2s faddp TEMP, v1.2s #else faddp TEMP, v1.2d + faddp TEMP1, v2.2d + faddp TEMP2, v3.2d + faddp TEMP3, v4.2d + fadd TEMP, TEMP, TEMP1 + fadd TEMP2, TEMP2, TEMP3 + fadd TEMP, TEMP, TEMP2 #endif .endm @@ -258,7 +267,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. asr I, M, #5 cmp I, xzr - beq .Lgemv_t_kernel_F4 + beq .Lgemv_t_kernel_F8 .Lgemv_t_kernel_F320: @@ -269,24 +278,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. KERNEL_F32_FINALIZE -.Lgemv_t_kernel_F4: +.Lgemv_t_kernel_F8: ands I, M, #31 - asr I, I, #2 + asr I, I, #3 cmp I, xzr beq .Lgemv_t_kernel_F1 -.Lgemv_t_kernel_F40: +.Lgemv_t_kernel_F80: - KERNEL_F4 + KERNEL_F8 subs I, I, #1 - bne .Lgemv_t_kernel_F40 + bne .Lgemv_t_kernel_F80 .Lgemv_t_kernel_F1: - KERNEL_F4_FINALIZE + KERNEL_F8_FINALIZE - ands I, M, #3 + ands I, M, #7 ble .Lgemv_t_kernel_F_END .Lgemv_t_kernel_F10: diff --git a/kernel/arm64/gemv_t_sve.c b/kernel/arm64/gemv_t_sve.c index ab700a3748..183d9c3d16 100644 --- a/kernel/arm64/gemv_t_sve.c +++ b/kernel/arm64/gemv_t_sve.c @@ -59,20 +59,46 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLO a_ptr = a; if (inc_x == 1) { + svbool_t pg_true = SV_TRUE(); uint64_t sve_size = SV_COUNT(); + uint64_t sve_size2 = sve_size * 2; + BLASLONG m1 = m & -sve_size; + BLASLONG m2 = m & -sve_size2; + for (j = 0; j < n; j++) { + BLASLONG i = 0; + + SV_TYPE temp_vec_v2_0 = SV_DUP(0.0); + SV_TYPE temp_vec_v2_1 = SV_DUP(0.0); + for (; i < m2; i += sve_size2) { + SV_TYPE a_vec0 = svld1(pg_true, a_ptr + i); + SV_TYPE x_vec0 = svld1(pg_true, x + i); + SV_TYPE a_vec1 = svld1(pg_true, a_ptr + i + sve_size); + SV_TYPE x_vec1 = svld1(pg_true, x + i + sve_size); + temp_vec_v2_0 = svmla_m(pg_true, temp_vec_v2_0, a_vec0, x_vec0); + temp_vec_v2_1 = svmla_m(pg_true, temp_vec_v2_1, a_vec1, x_vec1); + } + + SV_TYPE temp_vec_v1 = SV_DUP(0.0); + for (; i < m1; i += sve_size) { + SV_TYPE a_vec0 = svld1(pg_true, a_ptr + i); + SV_TYPE x_vec0 = svld1(pg_true, x + i); + temp_vec_v1 = svmla_m(pg_true, temp_vec_v1, a_vec0, x_vec0); + } + SV_TYPE temp_vec = SV_DUP(0.0); - i = 0; - svbool_t pg = SV_WHILE(i, m); - while (svptest_any(SV_TRUE(), pg)) { + for (; i < m; i += sve_size) { + svbool_t pg = SV_WHILE(i, m); SV_TYPE a_vec = svld1(pg, a_ptr + i); SV_TYPE x_vec = svld1(pg, x + i); temp_vec = svmla_m(pg, temp_vec, a_vec, x_vec); - i += sve_size; - pg = SV_WHILE(i, m); } - temp = svaddv(SV_TRUE(), temp_vec); - y[iy] += alpha * temp; + + y[iy] += alpha * ( + (svaddv(SV_TRUE(), temp_vec_v2_0) + svaddv(SV_TRUE(), temp_vec)) + + (svaddv(SV_TRUE(), temp_vec_v2_1) + svaddv(SV_TRUE(), temp_vec_v1)) + ); + iy += inc_y; a_ptr += lda; } From edbf093c98930b0b2144da638309b94a1a2bb5d0 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 31 Jul 2024 19:45:15 +0200 Subject: [PATCH 243/267] Update zarch SCAL kernels to handle INF and NAN arguments (#4829) * handle INF and NAN in input (for S/D only if DUMMY2 argument is set) --- kernel/zarch/cscal.c | 60 ++++++++++++++++++++++------- kernel/zarch/dscal.c | 46 +++++++++++++---------- kernel/zarch/sscal.c | 89 +++++++++++++++++++++++++++----------------- kernel/zarch/zscal.c | 43 +++++++++++++++------ 4 files changed, 160 insertions(+), 78 deletions(-) diff --git a/kernel/zarch/cscal.c b/kernel/zarch/cscal.c index 57bb89c0ae..e623f306b9 100644 --- a/kernel/zarch/cscal.c +++ b/kernel/zarch/cscal.c @@ -234,12 +234,23 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i, } else { while (j < n1) { - - temp0 = -da_i * x[i + 1]; - x[i + 1] = da_i * x[i]; + if (isnan(x[i]) || isinf(x[i])) + temp0 = NAN; + else + temp0 = -da_i * x[i + 1]; + if (!isinf(x[i + 1])) + x[i + 1] = da_i * x[i]; + else + x[i + 1] = NAN; x[i] = temp0; - temp1 = -da_i * x[i + 1 + inc_x]; - x[i + 1 + inc_x] = da_i * x[i + inc_x]; + if (isnan(x[i+inc_x]) || isinf(x[i+inc_x])) + temp1 = NAN; + else + temp1 = -da_i * x[i + 1 + inc_x]; + if (!isinf(x[i + 1 + inc_x])) + x[i + 1 + inc_x] = da_i * x[i + inc_x]; + else + x[i + 1 + inc_x] = NAN; x[i + inc_x] = temp1; i += 2 * inc_x; j += 2; @@ -247,9 +258,14 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i, } while (j < n) { - - temp0 = -da_i * x[i + 1]; - x[i + 1] = da_i * x[i]; + if (isnan(x[i]) || isinf(x[i])) + temp0 = NAN; + else + temp0 = -da_i * x[i + 1]; + if (isinf(x[i + 1])) + x[i + 1] = NAN; + else + x[i + 1] = da_i * x[i]; x[i] = temp0; i += inc_x; j++; @@ -332,26 +348,42 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i, j = n1; } - if (da_r == 0.0) { + if (da_r == 0.0 || isnan(da_r)) { if (da_i == 0.0) { - + float res = 0.0; + if (isnan(da_r)) res = da_r; while (j < n) { - x[i] = 0.0; - x[i + 1] = 0.0; + x[i] = res; + x[i + 1] = res; i += 2; j++; } + } else if (isinf(da_r)) { + while(j < n) + { + + x[i]= NAN; + x[i+1] = da_r; + i += 2 ; + j++; + + } } else { while (j < n) { temp0 = -da_i * x[i + 1]; - x[i + 1] = da_i * x[i]; - x[i] = temp0; + if (isinf(x[i])) temp0 = NAN; + if (!isinf(x[i + 1])) + x[i + 1] = da_i * x[i]; + else + x[i + 1] = NAN; + if (x[i] == x[i]) + x[i] = temp0; i += 2; j++; diff --git a/kernel/zarch/dscal.c b/kernel/zarch/dscal.c index a5a5e34685..14695602dc 100644 --- a/kernel/zarch/dscal.c +++ b/kernel/zarch/dscal.c @@ -96,20 +96,28 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, if (inc_x == 1) { if (da == 0.0) { - - BLASLONG n1 = n & -16; - if (n1 > 0) { - - dscal_kernel_16_zero(n1, x); - j = n1; + + if (dummy2 == 0) { + BLASLONG n1 = n & -16; + if (n1 > 0) { + dscal_kernel_16_zero(n1, x); + j = n1; + } + + while (j < n) { + x[j] = 0.0; + j++; + } + } else { + while (j < n) { + if (isfinite(x[j])) + x[j] = 0.0; + else + x[j] = NAN; + j++; + } } - - while (j < n) { - - x[j] = 0.0; - j++; - } - + } else { BLASLONG n1 = n & -16; @@ -127,11 +135,9 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, } else { if (da == 0.0) { - + if (dummy2 == 0) { BLASLONG n1 = n & -4; - while (j < n1) { - x[i] = 0.0; x[i + inc_x] = 0.0; x[i + 2 * inc_x] = 0.0; @@ -139,11 +145,13 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, i += inc_x * 4; j += 4; - } + } while (j < n) { - - x[i] = 0.0; + if (dummy2==0 || isfinite(x[i])) + x[i] = 0.0; + else + x[i] = NAN; i += inc_x; j++; } diff --git a/kernel/zarch/sscal.c b/kernel/zarch/sscal.c index da2f49eaf5..6777275154 100644 --- a/kernel/zarch/sscal.c +++ b/kernel/zarch/sscal.c @@ -95,21 +95,31 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, if (inc_x == 1) { - if (da == 0.0) { - - BLASLONG n1 = n & -32; - if (n1 > 0) { - - sscal_kernel_32_zero(n1, x); - j = n1; - } - - while (j < n) { - - x[j] = 0.0; - j++; + if (da == 0.0 || !isfinite(da)) { + if (dummy2 == 0) { + BLASLONG n1 = n & -32; + if (n1 > 0) { + + sscal_kernel_32_zero(n1, x); + j = n1; + } + + while (j < n) { + + x[j] = 0.0; + j++; + } + } else { + float res = 0.0; + if (!isfinite(da)) res = NAN; + while (j < n) { + if (isfinite(x[i])) + x[j] = res; + else + x[j] = NAN; + j++; + } } - } else { BLASLONG n1 = n & -32; @@ -126,26 +136,37 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, } else { - if (da == 0.0) { - - BLASLONG n1 = n & -2; - - while (j < n1) { - - x[i] = 0.0; - x[i + inc_x] = 0.0; - - i += inc_x * 2; - j += 2; - - } - while (j < n) { - - x[i] = 0.0; - i += inc_x; - j++; - } - + if (da == 0.0 || !isfinite(da)) { + if (dummy2 == 0) { + BLASLONG n1 = n & -2; + + while (j < n1) { + + x[i] = 0.0; + x[i + inc_x] = 0.0; + + i += inc_x * 2; + j += 2; + + } + while (j < n) { + + x[i] = 0.0; + i += inc_x; + j++; + } + } else { + while (j < n) { + float res = 0.0; + if (!isfinite(da)) res = NAN; + if (isfinite(x[i])) + x[i] = res; + else + x[i] = NAN; + i += inc_x; + j++; + } + } } else { BLASLONG n1 = n & -2; diff --git a/kernel/zarch/zscal.c b/kernel/zarch/zscal.c index 4160a1a768..36466a6e0a 100644 --- a/kernel/zarch/zscal.c +++ b/kernel/zarch/zscal.c @@ -237,13 +237,19 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i, temp0 = NAN; else temp0 = -da_i * x[i + 1]; - x[i + 1] = da_i * x[i]; + if (!isinf(x[i + 1])) + x[i + 1] = da_i * x[i]; + else + x[i + 1] = NAN; x[i] = temp0; if (isnan(x[i + inc_x]) || isinf(x[i + inc_x])) temp1 = NAN; else temp1 = -da_i * x[i + 1 + inc_x]; - x[i + 1 + inc_x] = da_i * x[i + inc_x]; + if (!isinf(x[i + 1 + inc_x])) + x[i + 1 + inc_x] = da_i * x[i + inc_x]; + else + x[i + 1 + inc_x] = NAN; x[i + inc_x] = temp1; i += 2 * inc_x; j += 2; @@ -256,7 +262,10 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i, temp0 = NAN; else temp0 = -da_i * x[i + 1]; - x[i + 1] = da_i * x[i]; + if (!isinf(x[i +1])) + x[i + 1] = da_i * x[i]; + else + x[i + 1] = NAN; x[i] = temp0; i += inc_x; j++; @@ -330,7 +339,7 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i, zscal_kernel_8_zero(n1, x); else zscal_kernel_8(n1, da_r, da_i, x); - else if (da_i == 0) + else if (da_i == 0 && da_r == da_r) zscal_kernel_8_zero_i(n1, alpha, x); else zscal_kernel_8(n1, da_r, da_i, x); @@ -339,29 +348,41 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i, j = n1; } - if (da_r == 0.0) { + if (da_r == 0.0 || isnan(da_r)) { if (da_i == 0.0) { - + double res= 0.0; + if (isnan(da_r)) res = da_r; while (j < n) { - x[i] = 0.0; - x[i + 1] = 0.0; + x[i] = res; + x[i + 1] = res; i += 2; j++; } + } else if (isinf(da_r)) { + while (j < n) { + x[i] = NAN; + x[i + 1] = da_r; + i += 2; + j++; + } } else { while (j < n) { - if (isnan(x[i]) || isinf(x[i])) + if (isinf(x[i])) temp0 = NAN; else temp0 = -da_i * x[i + 1]; - x[i + 1] = da_i * x[i]; - x[i] = temp0; + if (!isinf(x[i + 1])) + x[i + 1] = da_i * x[i]; + else + x[i + 1] = NAN; + if (x[i]==x[i]) + x[i] = temp0; i += 2; j++; From fcb88b9d529b8735f4328262d19b902c719fcc44 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 31 Jul 2024 23:21:35 +0200 Subject: [PATCH 244/267] enable GEMM/GEMV forwarding for riscv and ppc --- Makefile.system | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile.system b/Makefile.system index 436245c356..12cc57f25e 100644 --- a/Makefile.system +++ b/Makefile.system @@ -277,6 +277,12 @@ endif ifeq ($(ARCH), arm64) GEMM_GEMV_FORWARD = 1 endif +ifeq($(ARCH), riscv) +GEMM_GEMV_FORWARD = 1 +endif +ifeq ($(ARCH), power) +GEMM_GEMV_FORWARD = 1 +endif ifeq ($(SMALL_MATRIX_OPT), 1) CCOMMON_OPT += -DSMALL_MATRIX_OPT From 9eecd0d33b3470d4ce9d58290ed6aa31973b9051 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 31 Jul 2024 23:29:12 +0200 Subject: [PATCH 245/267] enable GEMM/GEMV forwarding for riscv and ppc --- cmake/system.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/system.cmake b/cmake/system.cmake index efb7aef94b..a3f7fd1734 100644 --- a/cmake/system.cmake +++ b/cmake/system.cmake @@ -391,7 +391,7 @@ endif () if (X86_64 OR ${CORE} STREQUAL POWER10) set(SMALL_MATRIX_OPT TRUE) endif () -if (ARM64) +if (ARM64 OR RISCV64 OR POWER) set(GEMM_GEMV_FORWARD TRUE) endif () From 42d8865234ffcf05c7b9e6dfd9ee0d435162145f Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 1 Aug 2024 12:24:45 +0200 Subject: [PATCH 246/267] fix typo --- Makefile.system | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.system b/Makefile.system index 12cc57f25e..b065f9a981 100644 --- a/Makefile.system +++ b/Makefile.system @@ -277,7 +277,7 @@ endif ifeq ($(ARCH), arm64) GEMM_GEMV_FORWARD = 1 endif -ifeq($(ARCH), riscv) +ifeq ($(ARCH), riscv) GEMM_GEMV_FORWARD = 1 endif ifeq ($(ARCH), power) From abff4baa4d59060c482dfc726fef438e71472315 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 2 Aug 2024 16:37:01 +0200 Subject: [PATCH 247/267] re-enable queue struct members related to locking --- common_thread.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common_thread.h b/common_thread.h index d37fcb1895..4a8db682bf 100644 --- a/common_thread.h +++ b/common_thread.h @@ -111,8 +111,8 @@ typedef struct blas_queue { struct blas_queue *next; #if defined( __WIN32__) || defined(__CYGWIN32__) || defined(_WIN32) || defined(__CYGWIN__) - // CRITICAL_SECTION lock; - // HANDLE finish; + CRITICAL_SECTION lock; + HANDLE finish; volatile int finished; #else pthread_mutex_t lock; From 6468dc1142ca284f97c821d84c6d331da812cf89 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 2 Aug 2024 16:39:47 +0200 Subject: [PATCH 248/267] restore the coarse locking of the pre-4359 version --- driver/others/blas_server_win32.c | 1253 ++++++++++++++--------------- 1 file changed, 589 insertions(+), 664 deletions(-) diff --git a/driver/others/blas_server_win32.c b/driver/others/blas_server_win32.c index 3b5f510c7c..5d792eaaed 100644 --- a/driver/others/blas_server_win32.c +++ b/driver/others/blas_server_win32.c @@ -1,664 +1,589 @@ - -/*********************************************************************/ -/* Copyright 2009, 2010 The University of Texas at Austin. */ -/* All rights reserved. */ -/* */ -/* Redistribution and use in source and binary forms, with or */ -/* without modification, are permitted provided that the following */ -/* conditions are met: */ -/* */ -/* 1. Redistributions of source code must retain the above */ -/* copyright notice, this list of conditions and the following */ -/* disclaimer. */ -/* */ -/* 2. Redistributions in binary form must reproduce the above */ -/* copyright notice, this list of conditions and the following */ -/* disclaimer in the documentation and/or other materials */ -/* provided with the distribution. */ -/* */ -/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ -/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ -/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ -/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ -/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ -/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ -/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ -/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ -/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ -/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ -/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ -/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ -/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ -/* POSSIBILITY OF SUCH DAMAGE. */ -/* */ -/* The views and conclusions contained in the software and */ -/* documentation are those of the authors and should not be */ -/* interpreted as representing official policies, either expressed */ -/* or implied, of The University of Texas at Austin. */ -/*********************************************************************/ - -#include -#include -#include "common.h" - -#if !defined(unlikely) -#ifdef __GNUC__ -#define unlikely(x) __builtin_expect(!!(x), 0) -#else -#define unlikely(x) (x) -#endif -#endif - -#ifdef SMP_DEBUG -# define MT_TRACE(...) fprintf(stderr, __VA_ARGS__) -#else -# define MT_TRACE(...) -#endif - -/* This is a thread implementation for Win32 lazy implementation */ - -/* Thread server common information */ - -static blas_queue_t *work_queue = NULL; -static HANDLE kickoff_event = NULL; -static CRITICAL_SECTION queue_lock; - -/* We need this global for checking if initialization is finished. */ -int blas_server_avail = 0; - -int blas_omp_threads_local = 1; - -static void * blas_thread_buffer[MAX_CPU_NUMBER]; - -/* Local Variables */ -static BLASULONG server_lock = 0; - -static HANDLE blas_threads [MAX_CPU_NUMBER]; -static DWORD blas_threads_id[MAX_CPU_NUMBER]; -static volatile int thread_target; // target num of live threads, volatile for cross-thread reads - -//Prototypes -static void exec_threads(int , blas_queue_t *, int); -static void adjust_thread_buffers(); - -// -// Legacy code path -// -static void legacy_exec(void *func, int mode, blas_arg_t *args, void *sb) { - - if (!(mode & BLAS_COMPLEX)) { -#ifdef EXPRECISION - if ((mode & BLAS_PREC) == BLAS_XDOUBLE){ - /* REAL / Extended Double */ - void (*afunc)(BLASLONG, BLASLONG, BLASLONG, xdouble, - xdouble *, BLASLONG, xdouble *, BLASLONG, - xdouble *, BLASLONG, void *) = func; - - afunc(args -> m, args -> n, args -> k, - ((xdouble *)args -> alpha)[0], - args -> a, args -> lda, - args -> b, args -> ldb, - args -> c, args -> ldc, sb); - } else -#endif - if ((mode & BLAS_PREC) == BLAS_DOUBLE) { - /* REAL / Double */ - void (*afunc)(BLASLONG, BLASLONG, BLASLONG, double, - double *, BLASLONG, double *, BLASLONG, - double *, BLASLONG, void *) = func; - - afunc(args -> m, args -> n, args -> k, - ((double *)args -> alpha)[0], - args -> a, args -> lda, - args -> b, args -> ldb, - args -> c, args -> ldc, sb); - } else if ((mode & BLAS_PREC) == BLAS_SINGLE) { - /* REAL / Single */ - void (*afunc)(BLASLONG, BLASLONG, BLASLONG, float, - float *, BLASLONG, float *, BLASLONG, - float *, BLASLONG, void *) = func; - - afunc(args -> m, args -> n, args -> k, - ((float *)args -> alpha)[0], - args -> a, args -> lda, - args -> b, args -> ldb, - args -> c, args -> ldc, sb); -#ifdef BUILD_BFLOAT16 - } else if ((mode & BLAS_PREC) == BLAS_BFLOAT16) { - /* REAL / BFLOAT16 */ - void (*afunc)(BLASLONG, BLASLONG, BLASLONG, bfloat16, - bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, - bfloat16 *, BLASLONG, void *) = func; - - afunc(args -> m, args -> n, args -> k, - ((bfloat16 *)args -> alpha)[0], - args -> a, args -> lda, - args -> b, args -> ldb, - args -> c, args -> ldc, sb); - } else if ((mode & BLAS_PREC) == BLAS_STOBF16) { - /* REAL / BLAS_STOBF16 */ - void (*afunc)(BLASLONG, BLASLONG, BLASLONG, float, - float *, BLASLONG, bfloat16 *, BLASLONG, - float *, BLASLONG, void *) = func; - - afunc(args -> m, args -> n, args -> k, - ((float *)args -> alpha)[0], - args -> a, args -> lda, - args -> b, args -> ldb, - args -> c, args -> ldc, sb); - } else if ((mode & BLAS_PREC) == BLAS_DTOBF16) { - /* REAL / BLAS_DTOBF16 */ - void (*afunc)(BLASLONG, BLASLONG, BLASLONG, double, - double *, BLASLONG, bfloat16 *, BLASLONG, - double *, BLASLONG, void *) = func; - - afunc(args -> m, args -> n, args -> k, - ((double *)args -> alpha)[0], - args -> a, args -> lda, - args -> b, args -> ldb, - args -> c, args -> ldc, sb); -#endif - } else { - /* REAL / Other types in future */ - } - } else { -#ifdef EXPRECISION - if ((mode & BLAS_PREC) == BLAS_XDOUBLE) { - /* COMPLEX / Extended Double */ - void (*afunc)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, - xdouble *, BLASLONG, xdouble *, BLASLONG, - xdouble *, BLASLONG, void *) = func; - - afunc(args -> m, args -> n, args -> k, - ((xdouble *)args -> alpha)[0], - ((xdouble *)args -> alpha)[1], - args -> a, args -> lda, - args -> b, args -> ldb, - args -> c, args -> ldc, sb); - } else -#endif - if ((mode & BLAS_PREC) == BLAS_DOUBLE) { - /* COMPLEX / Double */ - void (*afunc)(BLASLONG, BLASLONG, BLASLONG, double, double, - double *, BLASLONG, double *, BLASLONG, - double *, BLASLONG, void *) = func; - - afunc(args -> m, args -> n, args -> k, - ((double *)args -> alpha)[0], - ((double *)args -> alpha)[1], - args -> a, args -> lda, - args -> b, args -> ldb, - args -> c, args -> ldc, sb); - } else if ((mode & BLAS_PREC) == BLAS_SINGLE) { - /* COMPLEX / Single */ - void (*afunc)(BLASLONG, BLASLONG, BLASLONG, float, float, - float *, BLASLONG, float *, BLASLONG, - float *, BLASLONG, void *) = func; - - afunc(args -> m, args -> n, args -> k, - ((float *)args -> alpha)[0], - ((float *)args -> alpha)[1], - args -> a, args -> lda, - args -> b, args -> ldb, - args -> c, args -> ldc, sb); - } else { - /* COMPLEX / Other types in future */ - } - } -} - -// -// This is a main routine of threads. Each thread waits until job is queued. -// -static DWORD WINAPI blas_thread_server(void *arg) { - - /* Thread identifier */ - BLASLONG cpu = (BLASLONG)arg; - - blas_queue_t *queue; - - MT_TRACE("Server[%2ld] Thread is started!\n", cpu); - - while (1) { - - /* Waiting for Queue */ - - MT_TRACE("Server[%2ld] Waiting for Queue.\n", cpu); - - // event raised when work is added to the queue - WaitForSingleObject(kickoff_event, INFINITE); - - if (cpu > thread_target - 2) { - //MT_TRACE("thread [%d] exiting.\n", cpu); - break; // excess thread, so worker thread exits - } - - MT_TRACE("Server[%2ld] Got it.\n", cpu); - - EnterCriticalSection(&queue_lock); - - queue = work_queue; - if (queue) - work_queue = work_queue->next; - - LeaveCriticalSection(&queue_lock); - - if(queue) { - - exec_threads(cpu, queue, 0); - } else { - - continue; //if queue == NULL - } - - MT_TRACE("Server[%2ld] Finished!\n", cpu); - - queue->finished = 1; - } - - /* Shutdown procedure */ - - MT_TRACE("Server[%2ld] Shutdown!\n", cpu); - - return 0; -} - -// -// Initializing routine -// -int blas_thread_init(void) { - BLASLONG i; - - if (blas_server_avail || (blas_cpu_number <= 1)) return 0; - - LOCK_COMMAND(&server_lock); - - adjust_thread_buffers(); - - MT_TRACE("Initializing Thread(Num. threads = %d)\n", blas_cpu_number); - - if (!blas_server_avail) { - // create the kickoff Event - kickoff_event = CreateEvent(NULL, TRUE, FALSE, NULL); - - thread_target = blas_cpu_number; - - InitializeCriticalSection(&queue_lock); - - for(i = 0; i < blas_cpu_number - 1; i++) { - //MT_TRACE("thread_init: creating thread [%d]\n", i); - - blas_threads[i] = CreateThread(NULL, 0, - blas_thread_server, (void *)i, - 0, &blas_threads_id[i]); - } - - blas_server_avail = 1; - } - - UNLOCK_COMMAND(&server_lock); - - return 0; -} - -// -// User can call one of two routines. -// exec_blas_async ... immediately returns after jobs are queued. -// exec_blas ... returns after jobs are finished. -// -int exec_blas_async(BLASLONG pos, blas_queue_t *queue) { - -#if defined(SMP_SERVER) - // Handle lazy re-init of the thread-pool after a POSIX fork - // on Cygwin or as delayed init when a static library is used - if (unlikely(blas_server_avail == 0)) blas_thread_init(); -#endif - - blas_queue_t *current; - - current = queue; - - while (current) { - current -> position = pos; - -#ifdef CONSISTENT_FPCSR - __asm__ __volatile__ ("fnstcw %0" : "=m" (current -> x87_mode)); - __asm__ __volatile__ ("stmxcsr %0" : "=m" (current -> sse_mode)); -#endif - - current->finished = 0; - current = current -> next; - pos ++; - } - - EnterCriticalSection(&queue_lock); - - if (!work_queue) - { - work_queue = queue; - } - else - { - blas_queue_t *queue_item = work_queue; - - // find the end of the work queue - while (queue_item->next) - queue_item = queue_item->next; - - // add new work to the end - queue_item->next = queue; - } - - LeaveCriticalSection(&queue_lock); - - SetEvent(kickoff_event); - - return 0; -} - -// -// Join. Wait for all queued tasks to complete -// -int exec_blas_async_wait(BLASLONG num, blas_queue_t *queue) { - - MT_TRACE("Synchronization Waiting.\n"); - - while (num) { - MT_TRACE("Waiting Queue ..\n"); - - while (!queue->finished) - YIELDING; - - queue = queue->next; - num--; - } - - MT_TRACE("Completely Done.\n\n"); - - // if work was added to the queue after this batch we can't sleep the worker threads - // by resetting the event - EnterCriticalSection(&queue_lock); - - if (work_queue == NULL) - ResetEvent(kickoff_event); - - LeaveCriticalSection(&queue_lock); - - return 0; -} - -// -// Execute Threads -// -int exec_blas(BLASLONG num, blas_queue_t *queue) { - -#if defined(SMP_SERVER) && defined(OS_CYGWIN_NT) - // Handle lazy re-init of the thread-pool after a POSIX fork - if (unlikely(blas_server_avail == 0)) blas_thread_init(); -#endif - -#ifndef ALL_THREADED - int (*routine)(blas_arg_t *, void *, void *, double *, double *, BLASLONG); -#endif - - if ((num <= 0) || (queue == NULL)) return 0; - - //Redirect to caller's callback routine - if (openblas_threads_callback_) { - int buf_index = 0, i = 0; -#ifndef USE_SIMPLE_THREADED_LEVEL3 - for (i = 0; i < num; i ++) - queue[i].position = i; -#endif - openblas_threads_callback_(1, (openblas_dojob_callback) exec_threads, num, sizeof(blas_queue_t), (void*) queue, buf_index); - return 0; - } - - if ((num > 1) && queue -> next) - exec_blas_async(1, queue -> next); - - routine = queue -> routine; - - if (queue -> mode & BLAS_LEGACY) { - legacy_exec(routine, queue -> mode, queue -> args, queue -> sb); - } else { - if (queue -> mode & BLAS_PTHREAD) { - void (*pthreadcompat)(void *) = queue -> routine; - (pthreadcompat)(queue -> args); - } else - (routine)(queue -> args, queue -> range_m, queue -> range_n, - queue -> sa, queue -> sb, 0); - } - - if ((num > 1) && queue -> next) - exec_blas_async_wait(num - 1, queue -> next); - - return 0; -} - -// -// Shutdown procedure, but user don't have to call this routine. The -// kernel automatically kill threads. -// -int BLASFUNC(blas_thread_shutdown)(void) { - - int i; - - if (!blas_server_avail) return 0; - - LOCK_COMMAND(&server_lock); - - //Free buffers allocated for threads - for(i=0; i MAX_CPU_NUMBER) num_threads = MAX_CPU_NUMBER; - - if (blas_server_avail && num_threads < blas_num_threads) { - LOCK_COMMAND(&server_lock); - - thread_target = num_threads; - - SetEvent(kickoff_event); - - for (i = num_threads - 1; i < blas_num_threads - 1; i++) { - //MT_TRACE("set_num_threads: waiting on thread [%d] to quit.\n", i); - - WaitForSingleObject(blas_threads[i], INFINITE); - - //MT_TRACE("set_num_threads: thread [%d] has quit.\n", i); - - CloseHandle(blas_threads[i]); - } - - blas_num_threads = num_threads; - - ResetEvent(kickoff_event); - - UNLOCK_COMMAND(&server_lock); - } - - if (num_threads > blas_num_threads) { - - LOCK_COMMAND(&server_lock); - - thread_target = num_threads; - - //increased_threads = 1; - if (!blas_server_avail) { - // create the kickoff Event - kickoff_event = CreateEvent(NULL, TRUE, FALSE, NULL); - - InitializeCriticalSection(&queue_lock); - - blas_server_avail = 1; - } - - for (i = (blas_num_threads > 0) ? blas_num_threads - 1 : 0; i < num_threads - 1; i++) { - //MT_TRACE("set_num_threads: creating thread [%d]\n", i); - - blas_threads[i] = CreateThread(NULL, 0, - blas_thread_server, (void *)i, - 0, &blas_threads_id[i]); - } - - blas_num_threads = num_threads; - - UNLOCK_COMMAND(&server_lock); - } - - blas_cpu_number = num_threads; -} - -// -// Openblas function to set thread count -// -void openblas_set_num_threads(int num) -{ - goto_set_num_threads(num); -} - -static void adjust_thread_buffers() { - - int i=0; - - //adjust buffer for each thread - for(i=0; i < blas_cpu_number; i++){ - if(blas_thread_buffer[i] == NULL){ - blas_thread_buffer[i] = blas_memory_alloc(2); - } - } - for(; i < MAX_CPU_NUMBER; i++){ - if(blas_thread_buffer[i] != NULL){ - blas_memory_free(blas_thread_buffer[i]); - blas_thread_buffer[i] = NULL; - } - } -} - -//Indivitual threads work executor, Helps in setting by synchronization environment and calling inner_threads routine -static void exec_threads(int cpu, blas_queue_t *queue, int buf_index) { - - void *buffer, *sa, *sb; - - buffer = blas_thread_buffer[cpu]; - sa = queue -> sa; - sb = queue -> sb; - - int (*routine)(blas_arg_t *, void *, void *, void *, void *, BLASLONG) = queue -> routine; - - #ifdef CONSISTENT_FPCSR - __asm__ __volatile__ ("ldmxcsr %0" : : "m" (queue -> sse_mode)); - __asm__ __volatile__ ("fldcw %0" : : "m" (queue -> x87_mode)); - #endif - - MT_TRACE("Server[%2ld] Started. Mode = 0x%03x M = %3ld N=%3ld K=%3ld\n", - cpu, queue->mode, queue-> args ->m, queue->args->n, queue->args->k); - - // fprintf(stderr, "queue start[%ld]!!!\n", cpu); - - #ifdef MONITOR - main_status[cpu] = MAIN_RUNNING1; - #endif - - if (sa == NULL) - sa = (void *)((BLASLONG)buffer + GEMM_OFFSET_A); - - if (sb == NULL) { - if (!(queue -> mode & BLAS_COMPLEX)) { -#ifdef EXPRECISION -if ((queue -> mode & BLAS_PREC) == BLAS_XDOUBLE) { - sb = (void *)(((BLASLONG)sa + ((XGEMM_P * XGEMM_Q * sizeof(xdouble) - + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); -} else -#endif - if ((queue -> mode & BLAS_PREC) == BLAS_DOUBLE) { -#ifdef BUILD_DOUBLE - sb = (void *)(((BLASLONG)sa + ((DGEMM_P * DGEMM_Q * sizeof(double) - + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); -#endif - } else if ((queue -> mode & BLAS_PREC) == BLAS_SINGLE) { -#ifdef BUILD_SINGLE - sb = (void *)(((BLASLONG)sa + ((SGEMM_P * SGEMM_Q * sizeof(float) - + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); -#endif - } else { - /* Other types in future */ - } -} else { -#ifdef EXPRECISION -if ((queue -> mode & BLAS_PREC) == BLAS_XDOUBLE){ - sb = (void *)(((BLASLONG)sa + ((XGEMM_P * XGEMM_Q * 2 * sizeof(xdouble) - + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); -} else -#endif - if ((queue -> mode & BLAS_PREC) == BLAS_DOUBLE){ -#ifdef BUILD_COMPLEX16 - sb = (void *)(((BLASLONG)sa + ((ZGEMM_P * ZGEMM_Q * 2 * sizeof(double) - + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); -#endif - } else if ((queue -> mode & BLAS_PREC) == BLAS_SINGLE) { -#ifdef BUILD_COMPLEX - sb = (void *)(((BLASLONG)sa + ((CGEMM_P * CGEMM_Q * 2 * sizeof(float) - + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); -#endif - } else { - /* Other types in future */ - } -} - queue->sb=sb; - } - - #ifdef MONITOR - main_status[cpu] = MAIN_RUNNING2; - #endif - - if (!(queue -> mode & BLAS_LEGACY)) { - (routine)(queue -> args, queue -> range_m, queue -> range_n, sa, sb, queue -> position); - } else { - legacy_exec(routine, queue -> mode, queue -> args, sb); - } - -} \ No newline at end of file +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#include +#include +#include "common.h" + +#if !defined(unlikely) +#ifdef __GNUC__ +#define unlikely(x) __builtin_expect(!!(x), 0) +#else +#define unlikely(x) (x) +#endif +#endif + +/* This is a thread implementation for Win32 lazy implementation */ + +/* Thread server common information */ +typedef struct{ + CRITICAL_SECTION lock; + HANDLE filled; + HANDLE killed; + + blas_queue_t *queue; /* Parameter Pointer */ + int shutdown; /* server shutdown flag */ + +} blas_pool_t; + +/* We need this global for checking if initialization is finished. */ +int blas_server_avail = 0; +int blas_omp_threads_local = 1; +/* Local Variables */ +static BLASULONG server_lock = 0; + +static blas_pool_t pool; +static HANDLE blas_threads [MAX_CPU_NUMBER]; +static DWORD blas_threads_id[MAX_CPU_NUMBER]; + + + +static void legacy_exec(void *func, int mode, blas_arg_t *args, void *sb){ + + if (!(mode & BLAS_COMPLEX)){ +#ifdef EXPRECISION + if ((mode & BLAS_PREC) == BLAS_XDOUBLE){ + /* REAL / Extended Double */ + void (*afunc)(BLASLONG, BLASLONG, BLASLONG, xdouble, + xdouble *, BLASLONG, xdouble *, BLASLONG, + xdouble *, BLASLONG, void *) = func; + + afunc(args -> m, args -> n, args -> k, + ((xdouble *)args -> alpha)[0], + args -> a, args -> lda, + args -> b, args -> ldb, + args -> c, args -> ldc, sb); + } else +#endif + if ((mode & BLAS_PREC) == BLAS_DOUBLE){ + /* REAL / Double */ + void (*afunc)(BLASLONG, BLASLONG, BLASLONG, double, + double *, BLASLONG, double *, BLASLONG, + double *, BLASLONG, void *) = func; + + afunc(args -> m, args -> n, args -> k, + ((double *)args -> alpha)[0], + args -> a, args -> lda, + args -> b, args -> ldb, + args -> c, args -> ldc, sb); + } else if ((mode & BLAS_PREC) == BLAS_SINGLE){ + /* REAL / Single */ + void (*afunc)(BLASLONG, BLASLONG, BLASLONG, float, + float *, BLASLONG, float *, BLASLONG, + float *, BLASLONG, void *) = func; + + afunc(args -> m, args -> n, args -> k, + ((float *)args -> alpha)[0], + args -> a, args -> lda, + args -> b, args -> ldb, + args -> c, args -> ldc, sb); +#ifdef BUILD_BFLOAT16 + } else if ((mode & BLAS_PREC) == BLAS_BFLOAT16){ + /* REAL / BFLOAT16 */ + void (*afunc)(BLASLONG, BLASLONG, BLASLONG, bfloat16, + bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, + bfloat16 *, BLASLONG, void *) = func; + + afunc(args -> m, args -> n, args -> k, + ((bfloat16 *)args -> alpha)[0], + args -> a, args -> lda, + args -> b, args -> ldb, + args -> c, args -> ldc, sb); + } else if ((mode & BLAS_PREC) == BLAS_STOBF16){ + /* REAL / BLAS_STOBF16 */ + void (*afunc)(BLASLONG, BLASLONG, BLASLONG, float, + float *, BLASLONG, bfloat16 *, BLASLONG, + float *, BLASLONG, void *) = func; + + afunc(args -> m, args -> n, args -> k, + ((float *)args -> alpha)[0], + args -> a, args -> lda, + args -> b, args -> ldb, + args -> c, args -> ldc, sb); + } else if ((mode & BLAS_PREC) == BLAS_DTOBF16){ + /* REAL / BLAS_DTOBF16 */ + void (*afunc)(BLASLONG, BLASLONG, BLASLONG, double, + double *, BLASLONG, bfloat16 *, BLASLONG, + double *, BLASLONG, void *) = func; + + afunc(args -> m, args -> n, args -> k, + ((double *)args -> alpha)[0], + args -> a, args -> lda, + args -> b, args -> ldb, + args -> c, args -> ldc, sb); +#endif + } else { + /* REAL / Other types in future */ + } + } else { +#ifdef EXPRECISION + if ((mode & BLAS_PREC) == BLAS_XDOUBLE){ + /* COMPLEX / Extended Double */ + void (*afunc)(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble, + xdouble *, BLASLONG, xdouble *, BLASLONG, + xdouble *, BLASLONG, void *) = func; + + afunc(args -> m, args -> n, args -> k, + ((xdouble *)args -> alpha)[0], + ((xdouble *)args -> alpha)[1], + args -> a, args -> lda, + args -> b, args -> ldb, + args -> c, args -> ldc, sb); + } else +#endif + if ((mode & BLAS_PREC) == BLAS_DOUBLE){ + /* COMPLEX / Double */ + void (*afunc)(BLASLONG, BLASLONG, BLASLONG, double, double, + double *, BLASLONG, double *, BLASLONG, + double *, BLASLONG, void *) = func; + + afunc(args -> m, args -> n, args -> k, + ((double *)args -> alpha)[0], + ((double *)args -> alpha)[1], + args -> a, args -> lda, + args -> b, args -> ldb, + args -> c, args -> ldc, sb); + } else if ((mode & BLAS_PREC) == BLAS_SINGLE) { + /* COMPLEX / Single */ + void (*afunc)(BLASLONG, BLASLONG, BLASLONG, float, float, + float *, BLASLONG, float *, BLASLONG, + float *, BLASLONG, void *) = func; + + afunc(args -> m, args -> n, args -> k, + ((float *)args -> alpha)[0], + ((float *)args -> alpha)[1], + args -> a, args -> lda, + args -> b, args -> ldb, + args -> c, args -> ldc, sb); + } else { + /* COMPLEX / Other types in future */ + } + } +} + +/* This is a main routine of threads. Each thread waits until job is */ +/* queued. */ + +static DWORD WINAPI blas_thread_server(void *arg){ + + /* Thread identifier */ +#ifdef SMP_DEBUG + BLASLONG cpu = (BLASLONG)arg; +#endif + + void *buffer, *sa, *sb; + blas_queue_t *queue; + DWORD action; + HANDLE handles[] = {pool.filled, pool.killed}; + + /* Each server needs each buffer */ + buffer = blas_memory_alloc(2); + +#ifdef SMP_DEBUG + fprintf(STDERR, "Server[%2ld] Thread is started!\n", cpu); +#endif + + while (1){ + + /* Waiting for Queue */ + +#ifdef SMP_DEBUG + fprintf(STDERR, "Server[%2ld] Waiting for Queue.\n", cpu); +#endif + + do { + action = WaitForMultipleObjects(2, handles, FALSE, INFINITE); + } while ((action != WAIT_OBJECT_0) && (action != WAIT_OBJECT_0 + 1)); + + if (action == WAIT_OBJECT_0 + 1) break; + +#ifdef SMP_DEBUG + fprintf(STDERR, "Server[%2ld] Got it.\n", cpu); +#endif + + EnterCriticalSection(&pool.lock); + + queue = pool.queue; + if (queue) pool.queue = queue->next; + + LeaveCriticalSection(&pool.lock); + + if (queue) { + int (*routine)(blas_arg_t *, void *, void *, void *, void *, BLASLONG) = queue -> routine; + + if (pool.queue) SetEvent(pool.filled); + + sa = queue -> sa; + sb = queue -> sb; + +#ifdef CONSISTENT_FPCSR + __asm__ __volatile__ ("ldmxcsr %0" : : "m" (queue -> sse_mode)); + __asm__ __volatile__ ("fldcw %0" : : "m" (queue -> x87_mode)); +#endif + +#ifdef SMP_DEBUG + fprintf(STDERR, "Server[%2ld] Started. Mode = 0x%03x M = %3ld N=%3ld K=%3ld\n", + cpu, queue->mode, queue-> args ->m, queue->args->n, queue->args->k); +#endif + + // fprintf(stderr, "queue start[%ld]!!!\n", cpu); + +#ifdef MONITOR + main_status[cpu] = MAIN_RUNNING1; +#endif + + if (sa == NULL) sa = (void *)((BLASLONG)buffer + GEMM_OFFSET_A); + + if (sb == NULL) { + if (!(queue -> mode & BLAS_COMPLEX)){ +#ifdef EXPRECISION + if ((queue -> mode & BLAS_PREC) == BLAS_XDOUBLE){ + sb = (void *)(((BLASLONG)sa + ((XGEMM_P * XGEMM_Q * sizeof(xdouble) + + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); + } else +#endif + if ((queue -> mode & BLAS_PREC) == BLAS_DOUBLE){ +#ifdef BUILD_DOUBLE + sb = (void *)(((BLASLONG)sa + ((DGEMM_P * DGEMM_Q * sizeof(double) + + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); +#endif + } else if ((queue -> mode & BLAS_PREC) == BLAS_SINGLE) { +#ifdef BUILD_SINGLE + sb = (void *)(((BLASLONG)sa + ((SGEMM_P * SGEMM_Q * sizeof(float) + + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); +#endif + } else { + /* Other types in future */ + } + } else { +#ifdef EXPRECISION + if ((queue -> mode & BLAS_PREC) == BLAS_XDOUBLE){ + sb = (void *)(((BLASLONG)sa + ((XGEMM_P * XGEMM_Q * 2 * sizeof(xdouble) + + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); + } else +#endif + if ((queue -> mode & BLAS_PREC) == BLAS_DOUBLE){ +#ifdef BUILD_COMPLEX16 + sb = (void *)(((BLASLONG)sa + ((ZGEMM_P * ZGEMM_Q * 2 * sizeof(double) + + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); +#endif + } else if ((queue -> mode & BLAS_PREC) == BLAS_SINGLE) { +#ifdef BUILD_COMPLEX + sb = (void *)(((BLASLONG)sa + ((CGEMM_P * CGEMM_Q * 2 * sizeof(float) + + GEMM_ALIGN) & ~GEMM_ALIGN)) + GEMM_OFFSET_B); +#endif + } else { + /* Other types in future */ + } + } + queue->sb=sb; + } + +#ifdef MONITOR + main_status[cpu] = MAIN_RUNNING2; +#endif + + if (!(queue -> mode & BLAS_LEGACY)) { + + (routine)(queue -> args, queue -> range_m, queue -> range_n, sa, sb, queue -> position); + } else { + legacy_exec(routine, queue -> mode, queue -> args, sb); + } + }else{ + continue; //if queue == NULL + } + +#ifdef SMP_DEBUG + fprintf(STDERR, "Server[%2ld] Finished!\n", cpu); +#endif + + EnterCriticalSection(&queue->lock); + + queue -> status = BLAS_STATUS_FINISHED; + + LeaveCriticalSection(&queue->lock); + + SetEvent(queue->finish); + } + + /* Shutdown procedure */ + +#ifdef SMP_DEBUG + fprintf(STDERR, "Server[%2ld] Shutdown!\n", cpu); +#endif + + blas_memory_free(buffer); + + return 0; + } + +/* Initializing routine */ +int blas_thread_init(void){ + BLASLONG i; + + if (blas_server_avail || (blas_cpu_number <= 1)) return 0; + + LOCK_COMMAND(&server_lock); + +#ifdef SMP_DEBUG + fprintf(STDERR, "Initializing Thread(Num. threads = %d)\n", + blas_cpu_number); +#endif + + if (!blas_server_avail){ + + InitializeCriticalSection(&pool.lock); + pool.filled = CreateEvent(NULL, FALSE, FALSE, NULL); + pool.killed = CreateEvent(NULL, TRUE, FALSE, NULL); + + pool.shutdown = 0; + pool.queue = NULL; + + for(i = 0; i < blas_cpu_number - 1; i++){ + blas_threads[i] = CreateThread(NULL, 0, + blas_thread_server, (void *)i, + 0, &blas_threads_id[i]); + } + + blas_server_avail = 1; + } + + UNLOCK_COMMAND(&server_lock); + + return 0; +} + +/* + User can call one of two routines. + + exec_blas_async ... immediately returns after jobs are queued. + + exec_blas ... returns after jobs are finished. +*/ + +int exec_blas_async(BLASLONG pos, blas_queue_t *queue){ + +#if defined(SMP_SERVER) + // Handle lazy re-init of the thread-pool after a POSIX fork + // on Cygwin or as delayed init when a static library is used + if (unlikely(blas_server_avail == 0)) blas_thread_init(); +#endif + + blas_queue_t *current; + + current = queue; + + while (current) { + InitializeCriticalSection(¤t -> lock); + current -> finish = CreateEvent(NULL, FALSE, FALSE, NULL); + current -> position = pos; + +#ifdef CONSISTENT_FPCSR + __asm__ __volatile__ ("fnstcw %0" : "=m" (current -> x87_mode)); + __asm__ __volatile__ ("stmxcsr %0" : "=m" (current -> sse_mode)); +#endif + + current = current -> next; + pos ++; + } + + EnterCriticalSection(&pool.lock); + + if (pool.queue) { + current = pool.queue; + while (current -> next) current = current -> next; + current -> next = queue; + } else { + pool.queue = queue; + } + + LeaveCriticalSection(&pool.lock); + + SetEvent(pool.filled); + + return 0; +} + +int exec_blas_async_wait(BLASLONG num, blas_queue_t *queue){ + +#ifdef SMP_DEBUG + fprintf(STDERR, "Synchronization Waiting.\n"); +#endif + + while (num){ +#ifdef SMP_DEBUG + fprintf(STDERR, "Waiting Queue ..\n"); +#endif + + WaitForSingleObject(queue->finish, INFINITE); + + CloseHandle(queue->finish); + DeleteCriticalSection(&queue -> lock); + + queue = queue -> next; + num --; + } + +#ifdef SMP_DEBUG + fprintf(STDERR, "Completely Done.\n\n"); +#endif + + return 0; +} + +/* Execute Threads */ +int exec_blas(BLASLONG num, blas_queue_t *queue){ + +#if defined(SMP_SERVER) && defined(OS_CYGWIN_NT) + // Handle lazy re-init of the thread-pool after a POSIX fork + if (unlikely(blas_server_avail == 0)) blas_thread_init(); +#endif + +#ifndef ALL_THREADED + int (*routine)(blas_arg_t *, void *, void *, double *, double *, BLASLONG); +#endif + + if ((num <= 0) || (queue == NULL)) return 0; + + if ((num > 1) && queue -> next) exec_blas_async(1, queue -> next); + + routine = queue -> routine; + + if (queue -> mode & BLAS_LEGACY) { + legacy_exec(routine, queue -> mode, queue -> args, queue -> sb); + } else + if (queue -> mode & BLAS_PTHREAD) { + void (*pthreadcompat)(void *) = queue -> routine; + (pthreadcompat)(queue -> args); + } else + (routine)(queue -> args, queue -> range_m, queue -> range_n, + queue -> sa, queue -> sb, 0); + + if ((num > 1) && queue -> next) exec_blas_async_wait(num - 1, queue -> next); + + return 0; +} + +/* Shutdown procedure, but user don't have to call this routine. The */ +/* kernel automatically kill threads. */ + +int BLASFUNC(blas_thread_shutdown)(void){ + + int i; + + if (!blas_server_avail) return 0; + + LOCK_COMMAND(&server_lock); + + if (blas_server_avail){ + + SetEvent(pool.killed); + + for(i = 0; i < blas_num_threads - 1; i++){ + // Could also just use WaitForMultipleObjects + DWORD wait_thread_value = WaitForSingleObject(blas_threads[i], 50); + +#ifndef OS_WINDOWSSTORE + // TerminateThread is only available with WINAPI_DESKTOP and WINAPI_SYSTEM not WINAPI_APP in UWP + if (WAIT_OBJECT_0 != wait_thread_value) { + TerminateThread(blas_threads[i],0); + } +#endif + + CloseHandle(blas_threads[i]); + } + + CloseHandle(pool.filled); + CloseHandle(pool.killed); + + blas_server_avail = 0; + } + + UNLOCK_COMMAND(&server_lock); + + return 0; +} + +void goto_set_num_threads(int num_threads) +{ + long i; + +#if defined(SMP_SERVER) && defined(OS_CYGWIN_NT) + // Handle lazy re-init of the thread-pool after a POSIX fork + if (unlikely(blas_server_avail == 0)) blas_thread_init(); +#endif + + if (num_threads < 1) num_threads = blas_cpu_number; + + if (num_threads > MAX_CPU_NUMBER) num_threads = MAX_CPU_NUMBER; + + if (num_threads > blas_num_threads) { + + LOCK_COMMAND(&server_lock); + + //increased_threads = 1; + if (!blas_server_avail){ + + InitializeCriticalSection(&pool.lock); + pool.filled = CreateEvent(NULL, FALSE, FALSE, NULL); + pool.killed = CreateEvent(NULL, TRUE, FALSE, NULL); + + pool.shutdown = 0; + pool.queue = NULL; + blas_server_avail = 1; + } + + for(i = (blas_num_threads > 0) ? blas_num_threads - 1 : 0; i < num_threads - 1; i++){ + + blas_threads[i] = CreateThread(NULL, 0, + blas_thread_server, (void *)i, + 0, &blas_threads_id[i]); + } + + blas_num_threads = num_threads; + + UNLOCK_COMMAND(&server_lock); + } + + blas_cpu_number = num_threads; +} + +void openblas_set_num_threads(int num) +{ + goto_set_num_threads(num); +} From 7af3c552d3a70cf32a4545109c51bfc2a9ca329b Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 3 Aug 2024 23:51:57 +0200 Subject: [PATCH 249/267] use TARGET rather than CORE from Makefile.conf_last to fill in pkgconfig --- Makefile.install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.install b/Makefile.install index 00266b26e3..6892efa510 100644 --- a/Makefile.install +++ b/Makefile.install @@ -178,7 +178,7 @@ endif @echo 'libnamesuffix='$(LIBNAMESUFFIX) >> "$(PKGFILE)" @echo 'libsuffix='$(SYMBOLSUFFIX) >> "$(PKGFILE)" @echo 'includedir='$(OPENBLAS_INCLUDE_DIR) >> "$(PKGFILE)" - @echo 'openblas_config= USE_64BITINT='$(INTERFACE64) 'DYNAMIC_ARCH='$(DYNAMIC_ARCH) 'DYNAMIC_OLDER='$(DYNAMIC_OLDER) 'NO_CBLAS='$(NO_CBLAS) 'NO_LAPACK='$(NO_LAPACK) 'NO_LAPACKE='$(NO_LAPACKE) 'NO_AFFINITY='$(NO_AFFINITY) 'USE_OPENMP='$(USE_OPENMP) $(CORE) 'MAX_THREADS='$(NUM_THREADS)>> "$(PKGFILE)" + @echo 'openblas_config= USE_64BITINT='$(INTERFACE64) 'DYNAMIC_ARCH='$(DYNAMIC_ARCH) 'DYNAMIC_OLDER='$(DYNAMIC_OLDER) 'NO_CBLAS='$(NO_CBLAS) 'NO_LAPACK='$(NO_LAPACK) 'NO_LAPACKE='$(NO_LAPACKE) 'NO_AFFINITY='$(NO_AFFINITY) 'USE_OPENMP='$(USE_OPENMP) $(TARGET) 'MAX_THREADS='$(NUM_THREADS)>> "$(PKGFILE)" @echo 'version='$(VERSION) >> "$(PKGFILE)" @echo 'extralib='$(PKG_EXTRALIB) >> "$(PKGFILE)" @cat openblas.pc.in >> "$(PKGFILE)" From f40819476c1b3de0a5d53b26cec8e444d1b417fe Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 3 Aug 2024 23:54:35 +0200 Subject: [PATCH 250/267] mention RISCV64 as a permitted architecture for DYNAMIC_ARCH --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 07a3457cc2..75e6b27f36 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,7 +30,7 @@ option(C_LAPACK "Build LAPACK from C sources instead of the original Fortran" OF option(BUILD_WITHOUT_CBLAS "Do not build the C interface (CBLAS) to the BLAS functions" OFF) -option(DYNAMIC_ARCH "Include support for multiple CPU targets, with automatic selection at runtime (x86/x86_64, aarch64 or ppc only)" OFF) +option(DYNAMIC_ARCH "Include support for multiple CPU targets, with automatic selection at runtime (x86/x86_64, aarch64, ppc or RISCV64-RVV1.0 only)" OFF) option(DYNAMIC_OLDER "Include specific support for older x86 cpu models (Penryn,Dunnington,Atom,Nano,Opteron) with DYNAMIC_ARCH" OFF) From e8bd97ab4bf4c1faff0bd55404b22dad4a766e36 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 3 Aug 2024 23:56:59 +0200 Subject: [PATCH 251/267] add RISCV64 entries for DYNAMIC_ARCH --- cmake/arch.cmake | 6 +++++- cmake/system.cmake | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cmake/arch.cmake b/cmake/arch.cmake index 27c5650abd..5f3703ae0b 100644 --- a/cmake/arch.cmake +++ b/cmake/arch.cmake @@ -57,7 +57,11 @@ if (DYNAMIC_ARCH) set(DYNAMIC_CORE POWER6 POWER8 POWER9 POWER10) set(CCOMMON_OPT "${CCOMMON_OPT} -DHAVE_P10_SUPPORT") endif () - + + if (RISCV64) + set(DYNAMIC_CORE RISCV64_GENERIC RISCV64_ZVL128B RISCV64_ZVL256B) + endif () + if (X86) set(DYNAMIC_CORE KATMAI COPPERMINE NORTHWOOD PRESCOTT BANIAS CORE2 PENRYN DUNNINGTON NEHALEM ATHLON OPTERON OPTERON_SSE3 BARCELONA BOBCAT ATOM NANO) endif () diff --git a/cmake/system.cmake b/cmake/system.cmake index a3f7fd1734..031e876abe 100644 --- a/cmake/system.cmake +++ b/cmake/system.cmake @@ -391,7 +391,7 @@ endif () if (X86_64 OR ${CORE} STREQUAL POWER10) set(SMALL_MATRIX_OPT TRUE) endif () -if (ARM64 OR RISCV64 OR POWER) +if (ARM64) set(GEMM_GEMV_FORWARD TRUE) endif () @@ -403,7 +403,7 @@ if (SMALL_MATRIX_OPT) endif () if (DYNAMIC_ARCH) - if (X86 OR X86_64 OR ARM64 OR POWER) + if (X86 OR X86_64 OR ARM64 OR POWER OR RISCV64) set(CCOMMON_OPT "${CCOMMON_OPT} -DDYNAMIC_ARCH") if (DYNAMIC_OLDER) set(CCOMMON_OPT "${CCOMMON_OPT} -DDYNAMIC_OLDER") From 2aed90171ab0db59f7d3ee4fa675136aca09e6e3 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 3 Aug 2024 23:58:10 +0200 Subject: [PATCH 252/267] Add riscv sources for DYNAMIC_ARCH --- driver/others/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/driver/others/CMakeLists.txt b/driver/others/CMakeLists.txt index 192bf2d8fc..659449fbc2 100644 --- a/driver/others/CMakeLists.txt +++ b/driver/others/CMakeLists.txt @@ -52,6 +52,8 @@ if (DYNAMIC_ARCH) list(APPEND COMMON_SOURCES dynamic_arm64.c) elseif (POWER) list(APPEND COMMON_SOURCES dynamic_power.c) + elseif (RISCV64) + list(APPEND COMMON_SOURCES dynamic_riscv64.c detect_riscv64.c) else () list(APPEND COMMON_SOURCES dynamic.c) endif () From 5257f807a9fb069bfb05ad3784a680b35cdab35d Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sun, 4 Aug 2024 00:03:17 +0200 Subject: [PATCH 253/267] fix invalid ifdef syntax in HUGETLB handling --- driver/others/memory.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver/others/memory.c b/driver/others/memory.c index 53e49b452d..6343a3785e 100644 --- a/driver/others/memory.c +++ b/driver/others/memory.c @@ -2769,7 +2769,7 @@ void *blas_memory_alloc(int procpos){ #ifdef ALLOC_DEVICEDRIVER alloc_devicedirver, #endif -#ifdef ALLOC_SHM && !defined(ALLOC_HUGETLB) +#if defined(ALLOC_SHM) && !defined(ALLOC_HUGETLB) alloc_shm, #endif #if ((defined ALLOC_HUGETLB) && (defined OS_LINUX || defined OS_AIX || defined __sun__ || defined OS_WINDOWS)) From 60abcc3991f67e871f1ffd75449b90a1bcbf2953 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sun, 4 Aug 2024 00:13:31 +0200 Subject: [PATCH 254/267] add proper return statement --- kernel/x86_64/sscal.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/x86_64/sscal.c b/kernel/x86_64/sscal.c index 6e54f8893c..38b48eff41 100644 --- a/kernel/x86_64/sscal.c +++ b/kernel/x86_64/sscal.c @@ -200,4 +200,5 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS } } } + return(0); } From f1c9803f9a788fdbc3f1f7711845b2bc74ab5578 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sun, 4 Aug 2024 00:14:31 +0200 Subject: [PATCH 255/267] add proper return statement --- kernel/x86_64/dscal.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/x86_64/dscal.c b/kernel/x86_64/dscal.c index 641f86f903..e039d9015b 100644 --- a/kernel/x86_64/dscal.c +++ b/kernel/x86_64/dscal.c @@ -242,4 +242,5 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS } } } + return(0); } From a4845fa12dba53245fb6299169c0ad083fd0a251 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sun, 4 Aug 2024 23:38:22 +0200 Subject: [PATCH 256/267] set MACOSX_RPATH to true on Apple --- CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 75e6b27f36..67d16e5303 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -256,6 +256,10 @@ if (${CMAKE_SYSTEM_NAME} MATCHES "AIX|Android|Linux|FreeBSD|OpenBSD|NetBSD|Drago endif() endif() +if (APPLE AND BUILD_SHARED_LIBS) +set(CMAKE_MACOSX_RPATH ON) +endif() + # Seems that this hack doesn't required since macOS 11 Big Sur if (APPLE AND BUILD_SHARED_LIBS AND CMAKE_HOST_SYSTEM_VERSION VERSION_LESS 20) set (CMAKE_C_USE_RESPONSE_FILE_FOR_OBJECTS 1) From c8b4ceca85ce9d8a92b5576098be2b1040b552d2 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Mon, 5 Aug 2024 16:45:05 +0200 Subject: [PATCH 257/267] prevent compilers from using FMA (Reference-LAPACK PR 1033) --- lapack-netlib/SRC/dlanv2.f | 12 ++++++++---- lapack-netlib/SRC/slanv2.f | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/lapack-netlib/SRC/dlanv2.f b/lapack-netlib/SRC/dlanv2.f index 204cddd02f..9008fdde08 100644 --- a/lapack-netlib/SRC/dlanv2.f +++ b/lapack-netlib/SRC/dlanv2.f @@ -109,7 +109,7 @@ *> \author Univ. of Colorado Denver *> \author NAG Ltd. * -*> \ingroup doubleOTHERauxiliary +*> \ingroup lanv2 * *> \par Further Details: * ===================== @@ -144,7 +144,7 @@ SUBROUTINE DLANV2( A, B, C, D, RT1R, RT1I, RT2R, RT2I, CS, SN ) * .. * .. Local Scalars .. DOUBLE PRECISION AA, BB, BCMAX, BCMIS, CC, CS1, DD, EPS, P, SAB, - $ SAC, SCALE, SIGMA, SN1, TAU, TEMP, Z, SAFMIN, + $ SAC, SCALE, SIGMA, SN1, TAU, TEMP, Z, SAFMIN, $ SAFMN2, SAFMX2 INTEGER COUNT * .. @@ -248,10 +248,14 @@ SUBROUTINE DLANV2( A, B, C, D, RT1R, RT1I, RT2R, RT2I, CS, SN ) * * Compute [ A B ] = [ CS SN ] [ AA BB ] * [ C D ] [-SN CS ] [ CC DD ] +* +* Note: Some of the multiplications are wrapped in parentheses to +* prevent compilers from using FMA instructions. See +* https://github.com/Reference-LAPACK/lapack/issues/1031. * A = AA*CS + CC*SN - B = BB*CS + DD*SN - C = -AA*SN + CC*CS + B = ( BB*CS ) + ( DD*SN ) + C = -( AA*SN ) + ( CC*CS ) D = -BB*SN + DD*CS * TEMP = HALF*( A+D ) diff --git a/lapack-netlib/SRC/slanv2.f b/lapack-netlib/SRC/slanv2.f index 6753012e75..40470867e7 100644 --- a/lapack-netlib/SRC/slanv2.f +++ b/lapack-netlib/SRC/slanv2.f @@ -109,7 +109,7 @@ *> \author Univ. of Colorado Denver *> \author NAG Ltd. * -*> \ingroup realOTHERauxiliary +*> \ingroup lanv2 * *> \par Further Details: * ===================== @@ -144,7 +144,7 @@ SUBROUTINE SLANV2( A, B, C, D, RT1R, RT1I, RT2R, RT2I, CS, SN ) * .. * .. Local Scalars .. REAL AA, BB, BCMAX, BCMIS, CC, CS1, DD, EPS, P, SAB, - $ SAC, SCALE, SIGMA, SN1, TAU, TEMP, Z, SAFMIN, + $ SAC, SCALE, SIGMA, SN1, TAU, TEMP, Z, SAFMIN, $ SAFMN2, SAFMX2 INTEGER COUNT * .. @@ -248,10 +248,14 @@ SUBROUTINE SLANV2( A, B, C, D, RT1R, RT1I, RT2R, RT2I, CS, SN ) * * Compute [ A B ] = [ CS SN ] [ AA BB ] * [ C D ] [-SN CS ] [ CC DD ] +* +* Note: Some of the multiplications are wrapped in parentheses to +* prevent compilers from using FMA instructions. See +* https://github.com/Reference-LAPACK/lapack/issues/1031. * A = AA*CS + CC*SN - B = BB*CS + DD*SN - C = -AA*SN + CC*CS + B = ( BB*CS ) + ( DD*SN ) + C = -( AA*SN ) + ( CC*CS ) D = -BB*SN + DD*CS * TEMP = HALF*( A+D ) From bce48d4a1342546f13ab7efb4b2f2d80edc930c2 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Mon, 5 Aug 2024 17:37:07 +0200 Subject: [PATCH 258/267] Fix typos and sytrd boundary workspace (Reference-LAPACK PR 1030) --- lapack-netlib/SRC/cgehrd.f | 8 +++++--- lapack-netlib/SRC/chetrd.f | 14 ++++++++------ lapack-netlib/SRC/sgelqt.f | 4 ++-- lapack-netlib/SRC/sgemlq.f | 2 +- lapack-netlib/SRC/sgemlqt.f | 4 ++-- lapack-netlib/SRC/ssytrd.f | 8 +++++--- lapack-netlib/SRC/zhetrd.f | 7 ++++--- 7 files changed, 27 insertions(+), 20 deletions(-) diff --git a/lapack-netlib/SRC/cgehrd.f b/lapack-netlib/SRC/cgehrd.f index 7ba87cc01b..35bd2e668c 100644 --- a/lapack-netlib/SRC/cgehrd.f +++ b/lapack-netlib/SRC/cgehrd.f @@ -163,7 +163,8 @@ *> \endverbatim *> * ===================================================================== - SUBROUTINE CGEHRD( N, ILO, IHI, A, LDA, TAU, WORK, LWORK, INFO ) + SUBROUTINE CGEHRD( N, ILO, IHI, A, LDA, TAU, WORK, LWORK, + $ INFO ) * * -- LAPACK computational routine -- * -- LAPACK is a software package provided by Univ. of Tennessee, -- @@ -193,7 +194,8 @@ SUBROUTINE CGEHRD( N, ILO, IHI, A, LDA, TAU, WORK, LWORK, INFO ) COMPLEX EI * .. * .. External Subroutines .. - EXTERNAL CAXPY, CGEHD2, CGEMM, CLAHR2, CLARFB, CTRMM, + EXTERNAL CAXPY, CGEHD2, CGEMM, CLAHR2, CLARFB, + $ CTRMM, $ XERBLA * .. * .. Intrinsic Functions .. @@ -230,7 +232,7 @@ SUBROUTINE CGEHRD( N, ILO, IHI, A, LDA, TAU, WORK, LWORK, INFO ) IF( NH.LE.1 ) THEN LWKOPT = 1 ELSE - NB = MIN( NBMAX, ILAENV( 1, 'DGEHRD', ' ', N, ILO, IHI, + NB = MIN( NBMAX, ILAENV( 1, 'CGEHRD', ' ', N, ILO, IHI, $ -1 ) ) LWKOPT = N*NB + TSIZE END IF diff --git a/lapack-netlib/SRC/chetrd.f b/lapack-netlib/SRC/chetrd.f index 52b9a895a9..34c74959dc 100644 --- a/lapack-netlib/SRC/chetrd.f +++ b/lapack-netlib/SRC/chetrd.f @@ -139,7 +139,7 @@ *> \author Univ. of Colorado Denver *> \author NAG Ltd. * -*> \ingroup complexHEcomputational +*> \ingroup hetrd * *> \par Further Details: * ===================== @@ -188,7 +188,8 @@ *> \endverbatim *> * ===================================================================== - SUBROUTINE CHETRD( UPLO, N, A, LDA, D, E, TAU, WORK, LWORK, INFO ) + SUBROUTINE CHETRD( UPLO, N, A, LDA, D, E, TAU, WORK, LWORK, + $ INFO ) * * -- LAPACK computational routine -- * -- LAPACK is a software package provided by Univ. of Tennessee, -- @@ -225,7 +226,8 @@ SUBROUTINE CHETRD( UPLO, N, A, LDA, D, E, TAU, WORK, LWORK, INFO ) * .. External Functions .. LOGICAL LSAME INTEGER ILAENV - EXTERNAL LSAME, ILAENV + REAL SROUNDUP_LWORK + EXTERNAL LSAME, ILAENV, SROUNDUP_LWORK * .. * .. Executable Statements .. * @@ -249,8 +251,8 @@ SUBROUTINE CHETRD( UPLO, N, A, LDA, D, E, TAU, WORK, LWORK, INFO ) * Determine the block size. * NB = ILAENV( 1, 'CHETRD', UPLO, N, -1, -1, -1 ) - LWKOPT = N*NB - WORK( 1 ) = LWKOPT + LWKOPT = MAX( 1, N*NB ) + WORK( 1 ) = SROUNDUP_LWORK(LWKOPT) END IF * IF( INFO.NE.0 ) THEN @@ -367,7 +369,7 @@ SUBROUTINE CHETRD( UPLO, N, A, LDA, D, E, TAU, WORK, LWORK, INFO ) $ TAU( I ), IINFO ) END IF * - WORK( 1 ) = LWKOPT + WORK( 1 ) = SROUNDUP_LWORK(LWKOPT) RETURN * * End of CHETRD diff --git a/lapack-netlib/SRC/sgelqt.f b/lapack-netlib/SRC/sgelqt.f index b5a4c21b30..343da9f49a 100644 --- a/lapack-netlib/SRC/sgelqt.f +++ b/lapack-netlib/SRC/sgelqt.f @@ -18,7 +18,7 @@ *> *> \verbatim *> -*> DGELQT computes a blocked LQ factorization of a real M-by-N matrix A +*> SGELQT computes a blocked LQ factorization of a real M-by-N matrix A *> using the compact WY representation of Q. *> \endverbatim * @@ -93,7 +93,7 @@ *> \author Univ. of Colorado Denver *> \author NAG Ltd. * -*> \ingroup doubleGEcomputational +*> \ingroup gelqt * *> \par Further Details: * ===================== diff --git a/lapack-netlib/SRC/sgemlq.f b/lapack-netlib/SRC/sgemlq.f index 7e4d9bf656..5793e6ea71 100644 --- a/lapack-netlib/SRC/sgemlq.f +++ b/lapack-netlib/SRC/sgemlq.f @@ -74,7 +74,7 @@ *> A is REAL array, dimension *> (LDA,M) if SIDE = 'L', *> (LDA,N) if SIDE = 'R' -*> Part of the data structure to represent Q as returned by DGELQ. +*> Part of the data structure to represent Q as returned by SGELQ. *> \endverbatim *> *> \param[in] LDA diff --git a/lapack-netlib/SRC/sgemlqt.f b/lapack-netlib/SRC/sgemlqt.f index bb128c824d..1cb882cb4e 100644 --- a/lapack-netlib/SRC/sgemlqt.f +++ b/lapack-netlib/SRC/sgemlqt.f @@ -20,7 +20,7 @@ *> *> \verbatim *> -*> DGEMLQT overwrites the general real M-by-N matrix C with +*> SGEMLQT overwrites the general real M-by-N matrix C with *> *> SIDE = 'L' SIDE = 'R' *> TRANS = 'N': Q C C Q @@ -145,7 +145,7 @@ *> \author Univ. of Colorado Denver *> \author NAG Ltd. * -*> \ingroup doubleGEcomputational +*> \ingroup gemlqt * * ===================================================================== SUBROUTINE SGEMLQT( SIDE, TRANS, M, N, K, MB, V, LDV, T, LDT, diff --git a/lapack-netlib/SRC/ssytrd.f b/lapack-netlib/SRC/ssytrd.f index f4fbecdc94..fda10d3720 100644 --- a/lapack-netlib/SRC/ssytrd.f +++ b/lapack-netlib/SRC/ssytrd.f @@ -188,7 +188,8 @@ *> \endverbatim *> * ===================================================================== - SUBROUTINE SSYTRD( UPLO, N, A, LDA, D, E, TAU, WORK, LWORK, INFO ) + SUBROUTINE SSYTRD( UPLO, N, A, LDA, D, E, TAU, WORK, LWORK, + $ INFO ) * * -- LAPACK computational routine -- * -- LAPACK is a software package provided by Univ. of Tennessee, -- @@ -248,7 +249,7 @@ SUBROUTINE SSYTRD( UPLO, N, A, LDA, D, E, TAU, WORK, LWORK, INFO ) * Determine the block size. * NB = ILAENV( 1, 'SSYTRD', UPLO, N, -1, -1, -1 ) - LWKOPT = N*NB + LWKOPT = MAX( 1, N*NB ) WORK( 1 ) = SROUNDUP_LWORK(LWKOPT) END IF * @@ -316,7 +317,8 @@ SUBROUTINE SSYTRD( UPLO, N, A, LDA, D, E, TAU, WORK, LWORK, INFO ) * Update the unreduced submatrix A(1:i-1,1:i-1), using an * update of the form: A := A - V*W**T - W*V**T * - CALL SSYR2K( UPLO, 'No transpose', I-1, NB, -ONE, A( 1, I ), + CALL SSYR2K( UPLO, 'No transpose', I-1, NB, -ONE, A( 1, + $ I ), $ LDA, WORK, LDWORK, ONE, A, LDA ) * * Copy superdiagonal elements back into A, and diagonal diff --git a/lapack-netlib/SRC/zhetrd.f b/lapack-netlib/SRC/zhetrd.f index 5b7d6546cc..7a64e228fc 100644 --- a/lapack-netlib/SRC/zhetrd.f +++ b/lapack-netlib/SRC/zhetrd.f @@ -139,7 +139,7 @@ *> \author Univ. of Colorado Denver *> \author NAG Ltd. * -*> \ingroup complex16HEcomputational +*> \ingroup hetrd * *> \par Further Details: * ===================== @@ -188,7 +188,8 @@ *> \endverbatim *> * ===================================================================== - SUBROUTINE ZHETRD( UPLO, N, A, LDA, D, E, TAU, WORK, LWORK, INFO ) + SUBROUTINE ZHETRD( UPLO, N, A, LDA, D, E, TAU, WORK, LWORK, + $ INFO ) * * -- LAPACK computational routine -- * -- LAPACK is a software package provided by Univ. of Tennessee, -- @@ -249,7 +250,7 @@ SUBROUTINE ZHETRD( UPLO, N, A, LDA, D, E, TAU, WORK, LWORK, INFO ) * Determine the block size. * NB = ILAENV( 1, 'ZHETRD', UPLO, N, -1, -1, -1 ) - LWKOPT = N*NB + LWKOPT = MAX( 1, N*NB ) WORK( 1 ) = LWKOPT END IF * From 7e8118d94e97d825def59f6b402d590ef9dc4867 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 6 Aug 2024 17:30:17 +0200 Subject: [PATCH 259/267] Support new build option LAPACK_STRLEN --- CMakeLists.txt | 2 ++ Makefile | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 67d16e5303..314e8d9d81 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,8 @@ option(BUILD_WITHOUT_LAPACK "Do not build LAPACK and LAPACKE (Only BLAS or CBLAS option(BUILD_LAPACK_DEPRECATED "When building LAPACK, include also some older, deprecated routines" ON) +set(LAPACK_STRLEN "" CACHE STRING "When building LAPACK, use this type (e.g. \"int\") for character lengths (defaults to size_t)") + option(BUILD_TESTING "Build LAPACK testsuite when building LAPACK" ON) option(BUILD_BENCHMARKS "Build the collection of BLAS/LAPACK benchmarks" OFF) diff --git a/Makefile b/Makefile index fa210ad722..78f82dea59 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,10 @@ else LAPACK_NOOPT := $(filter-out -O0 -O1 -O2 -O3 -Ofast -O -Og -Os,$(LAPACK_FFLAGS)) endif +ifdef LAPACK_STRLEN +LAPACK_FFLAGS += -DLAPACK_STRLEN=$(LAPACK_STRLEN) +endif + SUBDIRS_ALL = $(SUBDIRS) test ctest utest exports benchmark ../laswp ../bench cpp_thread_test .PHONY : all libs netlib $(RELA) test ctest shared install From cc36db643e2935874bc715af454e294034f77dd4 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 6 Aug 2024 17:31:03 +0200 Subject: [PATCH 260/267] Support new LAPACK build option LAPACK_STRLEN --- cmake/system.cmake | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmake/system.cmake b/cmake/system.cmake index 031e876abe..683c3181db 100644 --- a/cmake/system.cmake +++ b/cmake/system.cmake @@ -621,7 +621,10 @@ set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${FCOMMON_OPT}") set(FPFLAGS "${FPFLAGS} ${FCOMMON_OPT} ${COMMON_PROF}") #For LAPACK Fortran codes. -set(LAPACK_FFLAGS "${LAPACK_FFLAGS} ${CMAKE_Fortran_FLAGS}") +set(LAPACK_FFLAGS "${LAPACK_FFLAGS} ${CMAKE_Fortran_FLAGS}" ) +if (LAPACK_STRLEN) + set (LAPACK_FFLAGS "${LAPACK_FFLAGS} -DLAPACK_STRLEN=${LAPACK_STRLEN}") +endif() set(LAPACK_FPFLAGS "${LAPACK_FPFLAGS} ${FPFLAGS}") #Disable -fopenmp for LAPACK Fortran codes on Windows. From 923b79de479d7af4a460c99dfcafb462b728f79d Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 6 Aug 2024 17:55:14 +0200 Subject: [PATCH 261/267] make the type of the hidden arguments configurable via LAPACK_STRLEN (Reference-LAPACK PR 1025) --- lapack-netlib/LAPACKE/include/lapack.h | 2066 ++++++++++++------------ 1 file changed, 1035 insertions(+), 1031 deletions(-) diff --git a/lapack-netlib/LAPACKE/include/lapack.h b/lapack-netlib/LAPACKE/include/lapack.h index 28f8ad6551..532aae574e 100644 --- a/lapack-netlib/LAPACKE/include/lapack.h +++ b/lapack-netlib/LAPACKE/include/lapack.h @@ -21,6 +21,10 @@ #define LAPACK_FORTRAN_STRLEN_END #endif +#ifndef FORTRAN_STRLEN + #define FORTRAN_STRLEN size_t +#endif + /* Complex types are structures equivalent to the * Fortran complex types COMPLEX(4) and COMPLEX(8). * @@ -138,7 +142,7 @@ lapack_logical LAPACK_lsame_base( const char* ca, const char* cb , lapack_int lca, lapack_int lcb #endif #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -172,7 +176,7 @@ void LAPACK_cbbcsd_base( float* rwork, lapack_int const* lrwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -202,7 +206,7 @@ void LAPACK_dbbcsd_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -232,7 +236,7 @@ void LAPACK_sbbcsd_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -262,7 +266,7 @@ void LAPACK_zbbcsd_base( double* rwork, lapack_int const* lrwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -284,7 +288,7 @@ void LAPACK_dbdsdc_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -306,7 +310,7 @@ void LAPACK_sbdsdc_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -327,7 +331,7 @@ void LAPACK_cbdsqr_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -348,7 +352,7 @@ void LAPACK_dbdsqr_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -369,7 +373,7 @@ void LAPACK_sbdsqr_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -390,7 +394,7 @@ void LAPACK_zbdsqr_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -413,7 +417,7 @@ void LAPACK_dbdsvdx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -436,7 +440,7 @@ void LAPACK_sbdsvdx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -453,7 +457,7 @@ void LAPACK_ddisna_base( double* SEP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -470,7 +474,7 @@ void LAPACK_sdisna_base( float* SEP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -493,7 +497,7 @@ void LAPACK_cgbbrd_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -515,7 +519,7 @@ void LAPACK_dgbbrd_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -537,7 +541,7 @@ void LAPACK_sgbbrd_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -560,7 +564,7 @@ void LAPACK_zgbbrd_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -580,7 +584,7 @@ void LAPACK_cgbcon_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -600,7 +604,7 @@ void LAPACK_dgbcon_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -620,7 +624,7 @@ void LAPACK_sgbcon_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -640,7 +644,7 @@ void LAPACK_zgbcon_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -751,7 +755,7 @@ void LAPACK_cgbrfs_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -774,7 +778,7 @@ void LAPACK_dgbrfs_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -797,7 +801,7 @@ void LAPACK_sgbrfs_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -820,7 +824,7 @@ void LAPACK_zgbrfs_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -848,7 +852,7 @@ void LAPACK_cgbrfsx_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -876,7 +880,7 @@ void LAPACK_dgbrfsx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -904,7 +908,7 @@ void LAPACK_sgbrfsx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -932,7 +936,7 @@ void LAPACK_zgbrfsx_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -987,7 +991,7 @@ void LAPACK_cgbsvx_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1014,7 +1018,7 @@ void LAPACK_dgbsvx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1041,7 +1045,7 @@ void LAPACK_sgbsvx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1068,7 +1072,7 @@ void LAPACK_zgbsvx_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1098,7 +1102,7 @@ void LAPACK_cgbsvxx_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1128,7 +1132,7 @@ void LAPACK_dgbsvxx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1158,7 +1162,7 @@ void LAPACK_sgbsvxx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1188,7 +1192,7 @@ void LAPACK_zgbsvxx_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1229,7 +1233,7 @@ void LAPACK_cgbtrs_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1246,7 +1250,7 @@ void LAPACK_dgbtrs_base( double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1263,7 +1267,7 @@ void LAPACK_sgbtrs_base( float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1280,7 +1284,7 @@ void LAPACK_zgbtrs_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1297,7 +1301,7 @@ void LAPACK_cgebak_base( lapack_complex_float* V, lapack_int const* ldv, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1314,7 +1318,7 @@ void LAPACK_dgebak_base( double* V, lapack_int const* ldv, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1331,7 +1335,7 @@ void LAPACK_sgebak_base( float* V, lapack_int const* ldv, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1348,7 +1352,7 @@ void LAPACK_zgebak_base( lapack_complex_double* V, lapack_int const* ldv, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1365,7 +1369,7 @@ void LAPACK_cgebal_base( float* scale, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1382,7 +1386,7 @@ void LAPACK_dgebal_base( double* scale, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1399,7 +1403,7 @@ void LAPACK_sgebal_base( float* scale, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1416,7 +1420,7 @@ void LAPACK_zgebal_base( double* scale, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1480,7 +1484,7 @@ void LAPACK_cgecon_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1500,7 +1504,7 @@ void LAPACK_dgecon_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1520,7 +1524,7 @@ void LAPACK_sgecon_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1540,7 +1544,7 @@ void LAPACK_zgecon_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1648,7 +1652,7 @@ void LAPACK_cgees_base( float* rwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1668,7 +1672,7 @@ void LAPACK_dgees_base( double* work, lapack_int const* lwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1688,7 +1692,7 @@ void LAPACK_sgees_base( float* work, lapack_int const* lwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1708,7 +1712,7 @@ void LAPACK_zgees_base( double* rwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1730,7 +1734,7 @@ void LAPACK_cgeesx_base( float* rwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1753,7 +1757,7 @@ void LAPACK_dgeesx_base( lapack_int* iwork, lapack_int const* liwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1776,7 +1780,7 @@ void LAPACK_sgeesx_base( lapack_int* iwork, lapack_int const* liwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1798,7 +1802,7 @@ void LAPACK_zgeesx_base( double* rwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1819,7 +1823,7 @@ void LAPACK_cgeev_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1840,7 +1844,7 @@ void LAPACK_dgeev_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1861,7 +1865,7 @@ void LAPACK_sgeev_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1882,7 +1886,7 @@ void LAPACK_zgeev_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1907,7 +1911,7 @@ void LAPACK_cgeevx_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1933,7 +1937,7 @@ void LAPACK_dgeevx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1959,7 +1963,7 @@ void LAPACK_sgeevx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -1984,7 +1988,7 @@ void LAPACK_zgeevx_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -2038,7 +2042,7 @@ void LAPACK_cgejsv_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -2059,7 +2063,7 @@ void LAPACK_dgejsv_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -2080,7 +2084,7 @@ void LAPACK_sgejsv_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -2102,7 +2106,7 @@ void LAPACK_zgejsv_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -2216,7 +2220,7 @@ void LAPACK_cgels_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -2234,7 +2238,7 @@ void LAPACK_dgels_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -2252,7 +2256,7 @@ void LAPACK_sgels_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -2270,7 +2274,7 @@ void LAPACK_zgels_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -2415,7 +2419,7 @@ void LAPACK_cgemlq_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -2434,7 +2438,7 @@ void LAPACK_dgemlq_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -2453,7 +2457,7 @@ void LAPACK_sgemlq_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -2472,7 +2476,7 @@ void LAPACK_zgemlq_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -2491,7 +2495,7 @@ void LAPACK_cgemqr_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -2510,7 +2514,7 @@ void LAPACK_dgemqr_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -2529,7 +2533,7 @@ void LAPACK_sgemqr_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -2548,7 +2552,7 @@ void LAPACK_zgemqr_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -2567,7 +2571,7 @@ void LAPACK_cgemqrt_base( lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -2586,7 +2590,7 @@ void LAPACK_dgemqrt_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -2605,7 +2609,7 @@ void LAPACK_sgemqrt_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -2624,7 +2628,7 @@ void LAPACK_zgemqrt_base( lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -2983,7 +2987,7 @@ void LAPACK_cgerfs_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3006,7 +3010,7 @@ void LAPACK_dgerfs_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3029,7 +3033,7 @@ void LAPACK_sgerfs_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3052,7 +3056,7 @@ void LAPACK_zgerfs_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3080,7 +3084,7 @@ void LAPACK_cgerfsx_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3108,7 +3112,7 @@ void LAPACK_dgerfsx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3136,7 +3140,7 @@ void LAPACK_sgerfsx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3164,7 +3168,7 @@ void LAPACK_zgerfsx_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3250,7 +3254,7 @@ void LAPACK_cgesdd_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3271,7 +3275,7 @@ void LAPACK_dgesdd_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3292,7 +3296,7 @@ void LAPACK_sgesdd_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3314,7 +3318,7 @@ void LAPACK_zgesdd_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3339,7 +3343,7 @@ void LAPACK_cgedmd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3364,7 +3368,7 @@ void LAPACK_dgedmd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3388,7 +3392,7 @@ void LAPACK_sgedmd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3413,7 +3417,7 @@ void LAPACK_zgedmd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3441,7 +3445,7 @@ void LAPACK_cgedmdq_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3468,7 +3472,7 @@ void LAPACK_dgedmdq_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3495,7 +3499,7 @@ void LAPACK_sgedmdq_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3524,7 +3528,7 @@ void LAPACK_zgedmdq_base( lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3594,7 +3598,7 @@ void LAPACK_cgesvd_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3614,7 +3618,7 @@ void LAPACK_dgesvd_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3634,7 +3638,7 @@ void LAPACK_sgesvd_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3655,7 +3659,7 @@ void LAPACK_zgesvd_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3677,7 +3681,7 @@ void LAPACK_cgesvdq_base( float* rwork, lapack_int const* lrwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3699,7 +3703,7 @@ void LAPACK_dgesvdq_base( double* rwork, lapack_int const* lrwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3721,7 +3725,7 @@ void LAPACK_sgesvdq_base( float* rwork, lapack_int const* lrwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3743,7 +3747,7 @@ void LAPACK_zgesvdq_base( double* rwork, lapack_int const* lrwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3767,7 +3771,7 @@ void LAPACK_cgesvdx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3791,7 +3795,7 @@ void LAPACK_dgesvdx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3814,7 +3818,7 @@ void LAPACK_sgesvdx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3838,7 +3842,7 @@ void LAPACK_zgesvdx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3858,7 +3862,7 @@ void LAPACK_cgesvj_base( float* rwork, lapack_int const* lrwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3877,7 +3881,7 @@ void LAPACK_dgesvj_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3896,7 +3900,7 @@ void LAPACK_sgesvj_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3916,7 +3920,7 @@ void LAPACK_zgesvj_base( double* rwork, lapack_int const* lrwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3943,7 +3947,7 @@ void LAPACK_cgesvx_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3970,7 +3974,7 @@ void LAPACK_dgesvx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -3997,7 +4001,7 @@ void LAPACK_sgesvx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4024,7 +4028,7 @@ void LAPACK_zgesvx_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4054,7 +4058,7 @@ void LAPACK_cgesvxx_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4084,7 +4088,7 @@ void LAPACK_dgesvxx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4114,7 +4118,7 @@ void LAPACK_sgesvxx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4144,7 +4148,7 @@ void LAPACK_zgesvxx_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4261,7 +4265,7 @@ lapack_int LAPACK_cgetrs_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4278,7 +4282,7 @@ lapack_int LAPACK_dgetrs_base( double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4295,7 +4299,7 @@ lapack_int LAPACK_sgetrs_base( float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4312,7 +4316,7 @@ lapack_int LAPACK_zgetrs_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4330,7 +4334,7 @@ void LAPACK_cgetsls_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4348,7 +4352,7 @@ void LAPACK_dgetsls_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4366,7 +4370,7 @@ void LAPACK_sgetsls_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4384,7 +4388,7 @@ void LAPACK_zgetsls_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4438,7 +4442,7 @@ void LAPACK_cggbak_base( lapack_complex_float* V, lapack_int const* ldv, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4456,7 +4460,7 @@ void LAPACK_dggbak_base( double* V, lapack_int const* ldv, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4474,7 +4478,7 @@ void LAPACK_sggbak_base( float* V, lapack_int const* ldv, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4492,7 +4496,7 @@ void LAPACK_zggbak_base( lapack_complex_double* V, lapack_int const* ldv, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4512,7 +4516,7 @@ void LAPACK_cggbal_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4532,7 +4536,7 @@ void LAPACK_dggbal_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4552,7 +4556,7 @@ void LAPACK_sggbal_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4572,7 +4576,7 @@ void LAPACK_zggbal_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4595,7 +4599,7 @@ void LAPACK_cgges_base( float* rwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4618,7 +4622,7 @@ void LAPACK_dgges_base( double* work, lapack_int const* lwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4641,7 +4645,7 @@ void LAPACK_sgges_base( float* work, lapack_int const* lwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4664,7 +4668,7 @@ void LAPACK_zgges_base( double* rwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4687,7 +4691,7 @@ void LAPACK_cgges3_base( float* rwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4710,7 +4714,7 @@ void LAPACK_dgges3_base( double* work, lapack_int const* lwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4733,7 +4737,7 @@ void LAPACK_sgges3_base( float* work, lapack_int const* lwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4756,7 +4760,7 @@ void LAPACK_zgges3_base( double* rwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4782,7 +4786,7 @@ void LAPACK_cggesx_base( lapack_int* iwork, lapack_int const* liwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4808,7 +4812,7 @@ void LAPACK_dggesx_base( lapack_int* iwork, lapack_int const* liwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4834,7 +4838,7 @@ void LAPACK_sggesx_base( lapack_int* iwork, lapack_int const* liwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4860,7 +4864,7 @@ void LAPACK_zggesx_base( lapack_int* iwork, lapack_int const* liwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4883,7 +4887,7 @@ void LAPACK_cggev_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4906,7 +4910,7 @@ void LAPACK_dggev_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4929,7 +4933,7 @@ void LAPACK_sggev_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4952,7 +4956,7 @@ void LAPACK_zggev_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4975,7 +4979,7 @@ void LAPACK_cggev3_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -4998,7 +5002,7 @@ void LAPACK_dggev3_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5021,7 +5025,7 @@ void LAPACK_sggev3_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5044,7 +5048,7 @@ void LAPACK_zggev3_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5074,7 +5078,7 @@ void LAPACK_cggevx_base( lapack_int* iwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5104,7 +5108,7 @@ void LAPACK_dggevx_base( lapack_int* iwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5134,7 +5138,7 @@ void LAPACK_sggevx_base( lapack_int* iwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5164,7 +5168,7 @@ void LAPACK_zggevx_base( lapack_int* iwork, lapack_logical* BWORK, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5228,7 +5232,7 @@ void LAPACK_cgghd3_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5248,7 +5252,7 @@ void LAPACK_dgghd3_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5268,7 +5272,7 @@ void LAPACK_sgghd3_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5288,7 +5292,7 @@ void LAPACK_zgghd3_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5307,7 +5311,7 @@ void LAPACK_cgghrd_base( lapack_complex_float* Z, lapack_int const* ldz, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5326,7 +5330,7 @@ void LAPACK_dgghrd_base( double* Z, lapack_int const* ldz, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5345,7 +5349,7 @@ void LAPACK_sgghrd_base( float* Z, lapack_int const* ldz, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5364,7 +5368,7 @@ void LAPACK_zgghrd_base( lapack_complex_double* Z, lapack_int const* ldz, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5511,7 +5515,7 @@ void LAPACK_cggsvd_base( lapack_complex_float* work, float* rwork, lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5533,7 +5537,7 @@ void LAPACK_sggsvd_base( float* q, lapack_int const* ldq, float* work, lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5555,7 +5559,7 @@ void LAPACK_dggsvd_base( double* q, lapack_int const* ldq, double* work, lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5578,7 +5582,7 @@ void LAPACK_zggsvd_base( lapack_complex_double* work, double* rwork, lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5603,7 +5607,7 @@ void LAPACK_cggsvd3_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5627,7 +5631,7 @@ void LAPACK_dggsvd3_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5651,7 +5655,7 @@ void LAPACK_sggsvd3_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5676,7 +5680,7 @@ void LAPACK_zggsvd3_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5699,7 +5703,7 @@ void LAPACK_sggsvp_base( lapack_int* iwork, float* tau, float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5722,7 +5726,7 @@ void LAPACK_dggsvp_base( lapack_int* iwork, double* tau, double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5744,7 +5748,7 @@ void LAPACK_cggsvp_base( lapack_int* iwork, float* rwork, lapack_complex_float* tau, lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5766,7 +5770,7 @@ void LAPACK_zggsvp_base( lapack_int* iwork, double* rwork, lapack_complex_double* tau, lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5792,7 +5796,7 @@ void LAPACK_cggsvp3_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5817,7 +5821,7 @@ void LAPACK_dggsvp3_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5842,7 +5846,7 @@ void LAPACK_sggsvp3_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5868,7 +5872,7 @@ void LAPACK_zggsvp3_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5890,7 +5894,7 @@ void LAPACK_cgtcon_base( lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5913,7 +5917,7 @@ void LAPACK_dgtcon_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5936,7 +5940,7 @@ void LAPACK_sgtcon_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5958,7 +5962,7 @@ void LAPACK_zgtcon_base( lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -5986,7 +5990,7 @@ void LAPACK_cgtrfs_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6014,7 +6018,7 @@ void LAPACK_dgtrfs_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6042,7 +6046,7 @@ void LAPACK_sgtrfs_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6070,7 +6074,7 @@ void LAPACK_zgtrfs_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6135,7 +6139,7 @@ void LAPACK_cgtsvx_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6164,7 +6168,7 @@ void LAPACK_dgtsvx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6193,7 +6197,7 @@ void LAPACK_sgtsvx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6222,7 +6226,7 @@ void LAPACK_zgtsvx_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6278,7 +6282,7 @@ void LAPACK_cgttrs_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6298,7 +6302,7 @@ void LAPACK_dgttrs_base( double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6318,7 +6322,7 @@ void LAPACK_sgttrs_base( float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6338,7 +6342,7 @@ void LAPACK_zgttrs_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6358,7 +6362,7 @@ void LAPACK_chbev_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6378,7 +6382,7 @@ void LAPACK_zhbev_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6398,7 +6402,7 @@ void LAPACK_chbev_2stage_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6418,7 +6422,7 @@ void LAPACK_zhbev_2stage_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6439,7 +6443,7 @@ void LAPACK_chbevd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6460,7 +6464,7 @@ void LAPACK_zhbevd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6481,7 +6485,7 @@ void LAPACK_chbevd_2stage_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6502,7 +6506,7 @@ void LAPACK_zhbevd_2stage_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6527,7 +6531,7 @@ void LAPACK_chbevx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6552,7 +6556,7 @@ void LAPACK_zhbevx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6577,7 +6581,7 @@ void LAPACK_chbevx_2stage_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6602,7 +6606,7 @@ void LAPACK_zhbevx_2stage_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6622,7 +6626,7 @@ void LAPACK_chbgst_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6642,7 +6646,7 @@ void LAPACK_zhbgst_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6663,7 +6667,7 @@ void LAPACK_chbgv_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6684,7 +6688,7 @@ void LAPACK_zhbgv_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6706,7 +6710,7 @@ void LAPACK_chbgvd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6728,7 +6732,7 @@ void LAPACK_zhbgvd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6754,7 +6758,7 @@ void LAPACK_chbgvx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6780,7 +6784,7 @@ void LAPACK_zhbgvx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6800,7 +6804,7 @@ void LAPACK_chbtrd_base( lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6820,7 +6824,7 @@ void LAPACK_zhbtrd_base( lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6839,7 +6843,7 @@ void LAPACK_checon_base( lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6858,7 +6862,7 @@ void LAPACK_zhecon_base( lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6878,7 +6882,7 @@ void LAPACK_checon_3_base( lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6898,7 +6902,7 @@ void LAPACK_zhecon_3_base( lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6918,7 +6922,7 @@ void LAPACK_cheequb_base( lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6938,7 +6942,7 @@ void LAPACK_zheequb_base( lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6957,7 +6961,7 @@ void LAPACK_cheev_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6976,7 +6980,7 @@ void LAPACK_zheev_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -6995,7 +6999,7 @@ void LAPACK_cheev_2stage_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7014,7 +7018,7 @@ void LAPACK_zheev_2stage_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7034,7 +7038,7 @@ void LAPACK_cheevd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7054,7 +7058,7 @@ void LAPACK_zheevd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7074,7 +7078,7 @@ void LAPACK_cheevd_2stage_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7094,7 +7098,7 @@ void LAPACK_zheevd_2stage_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7118,7 +7122,7 @@ void LAPACK_cheevr_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7142,7 +7146,7 @@ void LAPACK_zheevr_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7166,7 +7170,7 @@ void LAPACK_cheevr_2stage_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7190,7 +7194,7 @@ void LAPACK_zheevr_2stage_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7214,7 +7218,7 @@ void LAPACK_cheevx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7238,7 +7242,7 @@ void LAPACK_zheevx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7262,7 +7266,7 @@ void LAPACK_cheevx_2stage_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7286,7 +7290,7 @@ void LAPACK_zheevx_2stage_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7303,7 +7307,7 @@ void LAPACK_chegst_base( const lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7320,7 +7324,7 @@ void LAPACK_zhegst_base( const lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7340,7 +7344,7 @@ void LAPACK_chegv_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7360,7 +7364,7 @@ void LAPACK_zhegv_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7380,7 +7384,7 @@ void LAPACK_chegv_2stage_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7400,7 +7404,7 @@ void LAPACK_zhegv_2stage_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7421,7 +7425,7 @@ void LAPACK_chegvd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7442,7 +7446,7 @@ void LAPACK_zhegvd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7467,7 +7471,7 @@ void LAPACK_chegvx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7492,7 +7496,7 @@ void LAPACK_zhegvx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7515,7 +7519,7 @@ void LAPACK_cherfs_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7538,7 +7542,7 @@ void LAPACK_zherfs_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7565,7 +7569,7 @@ void LAPACK_cherfsx_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7592,7 +7596,7 @@ void LAPACK_zherfsx_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7610,7 +7614,7 @@ void LAPACK_chesv_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7628,7 +7632,7 @@ void LAPACK_zhesv_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7646,7 +7650,7 @@ void LAPACK_chesv_aa_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7664,7 +7668,7 @@ void LAPACK_zhesv_aa_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7683,7 +7687,7 @@ void LAPACK_chesv_aa_2stage_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7702,7 +7706,7 @@ void LAPACK_zhesv_aa_2stage_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7721,7 +7725,7 @@ void LAPACK_chesv_rk_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7740,7 +7744,7 @@ void LAPACK_zhesv_rk_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7758,7 +7762,7 @@ void LAPACK_chesv_rook_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7776,7 +7780,7 @@ void LAPACK_zhesv_rook_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7800,7 +7804,7 @@ void LAPACK_chesvx_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7824,7 +7828,7 @@ void LAPACK_zhesvx_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7853,7 +7857,7 @@ void LAPACK_chesvxx_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7882,7 +7886,7 @@ void LAPACK_zhesvxx_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7897,7 +7901,7 @@ void LAPACK_cheswapr_base( lapack_int const* n, lapack_complex_float* A, lapack_int const* lda, lapack_int const* i1, lapack_int const* i2 #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7912,7 +7916,7 @@ void LAPACK_zheswapr_base( lapack_int const* n, lapack_complex_double* A, lapack_int const* lda, lapack_int const* i1, lapack_int const* i2 #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7932,7 +7936,7 @@ void LAPACK_chetrd_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7952,7 +7956,7 @@ void LAPACK_zhetrd_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7973,7 +7977,7 @@ void LAPACK_chetrd_2stage_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -7994,7 +7998,7 @@ void LAPACK_zhetrd_2stage_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8011,7 +8015,7 @@ void LAPACK_chetrf_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8028,7 +8032,7 @@ void LAPACK_zhetrf_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8045,7 +8049,7 @@ void LAPACK_chetrf_aa_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8062,7 +8066,7 @@ void LAPACK_zhetrf_aa_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8080,7 +8084,7 @@ void LAPACK_chetrf_aa_2stage_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8098,7 +8102,7 @@ void LAPACK_zhetrf_aa_2stage_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8116,7 +8120,7 @@ void LAPACK_chetrf_rk_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8134,7 +8138,7 @@ void LAPACK_zhetrf_rk_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8151,7 +8155,7 @@ void LAPACK_chetrf_rook_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8168,7 +8172,7 @@ void LAPACK_zhetrf_rook_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8185,7 +8189,7 @@ void LAPACK_chetri_base( lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8202,7 +8206,7 @@ void LAPACK_zhetri_base( lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8219,7 +8223,7 @@ void LAPACK_chetri2_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8236,7 +8240,7 @@ void LAPACK_zhetri2_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8253,7 +8257,7 @@ void LAPACK_chetri2x_base( lapack_complex_float* work, lapack_int const* nb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8270,7 +8274,7 @@ void LAPACK_zhetri2x_base( lapack_complex_double* work, lapack_int const* nb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8288,7 +8292,7 @@ void LAPACK_chetri_3_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8306,7 +8310,7 @@ void LAPACK_zhetri_3_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8323,7 +8327,7 @@ void LAPACK_chetrs_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8340,7 +8344,7 @@ void LAPACK_zhetrs_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8358,7 +8362,7 @@ void LAPACK_chetrs2_base( lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8376,7 +8380,7 @@ void LAPACK_zhetrs2_base( lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8394,7 +8398,7 @@ void LAPACK_chetrs_3_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8412,7 +8416,7 @@ void LAPACK_zhetrs_3_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8430,7 +8434,7 @@ void LAPACK_chetrs_aa_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8448,7 +8452,7 @@ void LAPACK_zhetrs_aa_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8466,7 +8470,7 @@ void LAPACK_chetrs_aa_2stage_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8484,7 +8488,7 @@ void LAPACK_zhetrs_aa_2stage_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8501,7 +8505,7 @@ void LAPACK_chetrs_rook_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8518,7 +8522,7 @@ void LAPACK_zhetrs_rook_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8536,7 +8540,7 @@ void LAPACK_chfrk_base( float const* beta, lapack_complex_float* C #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8554,7 +8558,7 @@ void LAPACK_zhfrk_base( double const* beta, lapack_complex_double* C #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8577,7 +8581,7 @@ void LAPACK_chgeqz_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8600,7 +8604,7 @@ void LAPACK_dhgeqz_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8623,7 +8627,7 @@ void LAPACK_shgeqz_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8646,7 +8650,7 @@ void LAPACK_zhgeqz_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8665,7 +8669,7 @@ void LAPACK_chpcon_base( lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8684,7 +8688,7 @@ void LAPACK_zhpcon_base( lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8704,7 +8708,7 @@ void LAPACK_chpev_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8724,7 +8728,7 @@ void LAPACK_zhpev_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8745,7 +8749,7 @@ void LAPACK_chpevd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8766,7 +8770,7 @@ void LAPACK_zhpevd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8790,7 +8794,7 @@ void LAPACK_chpevx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8814,7 +8818,7 @@ void LAPACK_zhpevx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8831,7 +8835,7 @@ void LAPACK_chpgst_base( lapack_complex_float const* BP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8848,7 +8852,7 @@ void LAPACK_zhpgst_base( lapack_complex_double const* BP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8869,7 +8873,7 @@ void LAPACK_chpgv_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8890,7 +8894,7 @@ void LAPACK_zhpgv_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8912,7 +8916,7 @@ void LAPACK_chpgvd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8934,7 +8938,7 @@ void LAPACK_zhpgvd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8959,7 +8963,7 @@ void LAPACK_chpgvx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -8984,7 +8988,7 @@ void LAPACK_zhpgvx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9007,7 +9011,7 @@ void LAPACK_chprfs_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9030,7 +9034,7 @@ void LAPACK_zhprfs_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9047,7 +9051,7 @@ void LAPACK_chpsv_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9064,7 +9068,7 @@ void LAPACK_zhpsv_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9088,7 +9092,7 @@ void LAPACK_chpsvx_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9112,7 +9116,7 @@ void LAPACK_zhpsvx_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9131,7 +9135,7 @@ void LAPACK_chptrd_base( lapack_complex_float* tau, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9150,7 +9154,7 @@ void LAPACK_zhptrd_base( lapack_complex_double* tau, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9166,7 +9170,7 @@ void LAPACK_chptrf_base( lapack_complex_float* AP, lapack_int* ipiv, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9182,7 +9186,7 @@ void LAPACK_zhptrf_base( lapack_complex_double* AP, lapack_int* ipiv, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9199,7 +9203,7 @@ void LAPACK_chptri_base( lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9216,7 +9220,7 @@ void LAPACK_zhptri_base( lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9233,7 +9237,7 @@ void LAPACK_chptrs_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9250,7 +9254,7 @@ void LAPACK_zhptrs_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9272,7 +9276,7 @@ void LAPACK_chsein_base( float* rwork, lapack_int* IFAILL, lapack_int* IFAILR, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9294,7 +9298,7 @@ void LAPACK_dhsein_base( double* work, lapack_int* IFAILL, lapack_int* IFAILR, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9316,7 +9320,7 @@ void LAPACK_shsein_base( float* work, lapack_int* IFAILL, lapack_int* IFAILR, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9338,7 +9342,7 @@ void LAPACK_zhsein_base( double* rwork, lapack_int* IFAILL, lapack_int* IFAILR, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9357,7 +9361,7 @@ void LAPACK_chseqr_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9377,7 +9381,7 @@ void LAPACK_dhseqr_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9397,7 +9401,7 @@ void LAPACK_shseqr_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9416,7 +9420,7 @@ void LAPACK_zhseqr_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9470,7 +9474,7 @@ void LAPACK_clacp2_base( float const* A, lapack_int const* lda, lapack_complex_float* B, lapack_int const* ldb #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9486,7 +9490,7 @@ void LAPACK_zlacp2_base( double const* A, lapack_int const* lda, lapack_complex_double* B, lapack_int const* ldb #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9502,7 +9506,7 @@ void LAPACK_clacpy_base( lapack_complex_float const* A, lapack_int const* lda, lapack_complex_float* B, lapack_int const* ldb #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9518,7 +9522,7 @@ void LAPACK_dlacpy_base( double const* A, lapack_int const* lda, double* B, lapack_int const* ldb #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9534,7 +9538,7 @@ void LAPACK_slacpy_base( float const* A, lapack_int const* lda, float* B, lapack_int const* ldb #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9550,7 +9554,7 @@ void LAPACK_zlacpy_base( lapack_complex_double const* A, lapack_int const* lda, lapack_complex_double* B, lapack_int const* ldb #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9687,7 +9691,7 @@ void LAPACK_zlagsy( double LAPACK_dlamch_base( char const* cmach #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9700,7 +9704,7 @@ double LAPACK_dlamch_base( lapack_float_return LAPACK_slamch_base( char const* cmach #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9716,7 +9720,7 @@ lapack_float_return LAPACK_clangb_base( lapack_complex_float const* AB, lapack_int const* ldab, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9732,7 +9736,7 @@ double LAPACK_dlangb_base( double const* AB, lapack_int const* ldab, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9748,7 +9752,7 @@ lapack_float_return LAPACK_slangb_base( float const* AB, lapack_int const* ldab, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9764,7 +9768,7 @@ double LAPACK_zlangb_base( lapack_complex_double const* AB, lapack_int const* ldab, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9780,7 +9784,7 @@ lapack_float_return LAPACK_clange_base( lapack_complex_float const* A, lapack_int const* lda, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9796,7 +9800,7 @@ double LAPACK_dlange_base( double const* A, lapack_int const* lda, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9812,7 +9816,7 @@ lapack_float_return LAPACK_slange_base( float const* A, lapack_int const* lda, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9828,7 +9832,7 @@ double LAPACK_zlange_base( lapack_complex_double const* A, lapack_int const* lda, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9845,7 +9849,7 @@ lapack_float_return LAPACK_clangt_base( lapack_complex_float const* D, lapack_complex_float const* DU #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9862,7 +9866,7 @@ double LAPACK_dlangt_base( double const* D, double const* DU #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9879,7 +9883,7 @@ lapack_float_return LAPACK_slangt_base( float const* D, float const* DU #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9896,7 +9900,7 @@ double LAPACK_zlangt_base( lapack_complex_double const* D, lapack_complex_double const* DU #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9912,7 +9916,7 @@ lapack_float_return LAPACK_clanhb_base( lapack_complex_float const* AB, lapack_int const* ldab, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9928,7 +9932,7 @@ double LAPACK_zlanhb_base( lapack_complex_double const* AB, lapack_int const* ldab, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9944,7 +9948,7 @@ lapack_float_return LAPACK_clanhe_base( lapack_complex_float const* A, lapack_int const* lda, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9960,7 +9964,7 @@ double LAPACK_zlanhe_base( lapack_complex_double const* A, lapack_int const* lda, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9976,7 +9980,7 @@ lapack_float_return LAPACK_clanhp_base( lapack_complex_float const* AP, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -9992,7 +9996,7 @@ double LAPACK_zlanhp_base( lapack_complex_double const* AP, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10008,7 +10012,7 @@ lapack_float_return LAPACK_clanhs_base( lapack_complex_float const* A, lapack_int const* lda, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10024,7 +10028,7 @@ double LAPACK_dlanhs_base( double const* A, lapack_int const* lda, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10040,7 +10044,7 @@ lapack_float_return LAPACK_slanhs_base( float const* A, lapack_int const* lda, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10056,7 +10060,7 @@ double LAPACK_zlanhs_base( lapack_complex_double const* A, lapack_int const* lda, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10072,7 +10076,7 @@ lapack_float_return LAPACK_clanht_base( float const* D, lapack_complex_float const* E #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10088,7 +10092,7 @@ double LAPACK_zlanht_base( double const* D, lapack_complex_double const* E #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10104,7 +10108,7 @@ lapack_float_return LAPACK_clansb_base( lapack_complex_float const* AB, lapack_int const* ldab, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10120,7 +10124,7 @@ double LAPACK_dlansb_base( double const* AB, lapack_int const* ldab, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10136,7 +10140,7 @@ lapack_float_return LAPACK_slansb_base( float const* AB, lapack_int const* ldab, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10152,7 +10156,7 @@ double LAPACK_zlansb_base( lapack_complex_double const* AB, lapack_int const* ldab, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10168,7 +10172,7 @@ lapack_float_return LAPACK_clansp_base( lapack_complex_float const* AP, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10184,7 +10188,7 @@ double LAPACK_dlansp_base( double const* AP, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10200,7 +10204,7 @@ lapack_float_return LAPACK_slansp_base( float const* AP, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10216,7 +10220,7 @@ double LAPACK_zlansp_base( lapack_complex_double const* AP, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10232,7 +10236,7 @@ double LAPACK_dlanst_base( double const* D, double const* E #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10248,7 +10252,7 @@ lapack_float_return LAPACK_slanst_base( float const* D, float const* E #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10264,7 +10268,7 @@ lapack_float_return LAPACK_clansy_base( lapack_complex_float const* A, lapack_int const* lda, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10280,7 +10284,7 @@ double LAPACK_dlansy_base( double const* A, lapack_int const* lda, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10296,7 +10300,7 @@ lapack_float_return LAPACK_slansy_base( float const* A, lapack_int const* lda, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10312,7 +10316,7 @@ double LAPACK_zlansy_base( lapack_complex_double const* A, lapack_int const* lda, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10328,7 +10332,7 @@ lapack_float_return LAPACK_clantb_base( lapack_complex_float const* AB, lapack_int const* ldab, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10344,7 +10348,7 @@ double LAPACK_dlantb_base( double const* AB, lapack_int const* ldab, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10360,7 +10364,7 @@ lapack_float_return LAPACK_slantb_base( float const* AB, lapack_int const* ldab, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10376,7 +10380,7 @@ double LAPACK_zlantb_base( lapack_complex_double const* AB, lapack_int const* ldab, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10392,7 +10396,7 @@ lapack_float_return LAPACK_clantp_base( lapack_complex_float const* AP, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10408,7 +10412,7 @@ double LAPACK_dlantp_base( double const* AP, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10424,7 +10428,7 @@ lapack_float_return LAPACK_slantp_base( float const* AP, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10440,7 +10444,7 @@ double LAPACK_zlantp_base( lapack_complex_double const* AP, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10456,7 +10460,7 @@ lapack_float_return LAPACK_clantr_base( lapack_complex_float const* A, lapack_int const* lda, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10472,7 +10476,7 @@ double LAPACK_dlantr_base( double const* A, lapack_int const* lda, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10488,7 +10492,7 @@ lapack_float_return LAPACK_slantr_base( float const* A, lapack_int const* lda, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10504,7 +10508,7 @@ double LAPACK_zlantr_base( lapack_complex_double const* A, lapack_int const* lda, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10600,7 +10604,7 @@ void LAPACK_clarf_base( lapack_complex_float* C, lapack_int const* ldc, lapack_complex_float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10618,7 +10622,7 @@ void LAPACK_dlarf_base( double* C, lapack_int const* ldc, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10636,7 +10640,7 @@ void LAPACK_slarf_base( float* C, lapack_int const* ldc, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10654,7 +10658,7 @@ void LAPACK_zlarf_base( lapack_complex_double* C, lapack_int const* ldc, lapack_complex_double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10672,7 +10676,7 @@ void LAPACK_clarfb_base( lapack_complex_float* C, lapack_int const* ldc, lapack_complex_float* work, lapack_int const* ldwork #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10690,7 +10694,7 @@ void LAPACK_dlarfb_base( double* C, lapack_int const* ldc, double* work, lapack_int const* ldwork #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10708,7 +10712,7 @@ void LAPACK_slarfb_base( float* C, lapack_int const* ldc, float* work, lapack_int const* ldwork #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10726,7 +10730,7 @@ void LAPACK_zlarfb_base( lapack_complex_double* C, lapack_int const* ldc, lapack_complex_double* work, lapack_int const* ldwork #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10771,7 +10775,7 @@ void LAPACK_clarft_base( lapack_complex_float const* tau, lapack_complex_float* T, lapack_int const* ldt #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10788,7 +10792,7 @@ void LAPACK_dlarft_base( double const* tau, double* T, lapack_int const* ldt #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10805,7 +10809,7 @@ void LAPACK_slarft_base( float const* tau, float* T, lapack_int const* ldt #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10822,7 +10826,7 @@ void LAPACK_zlarft_base( lapack_complex_double const* tau, lapack_complex_double* T, lapack_int const* ldt #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10840,7 +10844,7 @@ void LAPACK_clarfx_base( lapack_complex_float* C, lapack_int const* ldc, lapack_complex_float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10858,7 +10862,7 @@ void LAPACK_dlarfx_base( double* C, lapack_int const* ldc, double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10876,7 +10880,7 @@ void LAPACK_slarfx_base( float* C, lapack_int const* ldc, float* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10894,7 +10898,7 @@ void LAPACK_zlarfx_base( lapack_complex_double* C, lapack_int const* ldc, lapack_complex_double* work #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10964,7 +10968,7 @@ void LAPACK_clascl_base( lapack_complex_float* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -10982,7 +10986,7 @@ void LAPACK_dlascl_base( double* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11000,7 +11004,7 @@ void LAPACK_slascl_base( float* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11018,7 +11022,7 @@ void LAPACK_zlascl_base( lapack_complex_double* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11035,7 +11039,7 @@ void LAPACK_claset_base( lapack_complex_float const* beta, lapack_complex_float* A, lapack_int const* lda #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11052,7 +11056,7 @@ void LAPACK_dlaset_base( double const* beta, double* A, lapack_int const* lda #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11069,7 +11073,7 @@ void LAPACK_slaset_base( float const* beta, float* A, lapack_int const* lda #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11086,7 +11090,7 @@ void LAPACK_zlaset_base( lapack_complex_double const* beta, lapack_complex_double* A, lapack_int const* lda #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11102,7 +11106,7 @@ void LAPACK_dlasrt_base( double* D, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11118,7 +11122,7 @@ void LAPACK_slasrt_base( float* D, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11188,7 +11192,7 @@ void LAPACK_clatms_base( lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11210,7 +11214,7 @@ void LAPACK_dlatms_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11232,7 +11236,7 @@ void LAPACK_slatms_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11254,7 +11258,7 @@ void LAPACK_zlatms_base( lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11270,7 +11274,7 @@ lapack_int LAPACK_clauum_base( lapack_complex_float* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11286,7 +11290,7 @@ lapack_int LAPACK_dlauum_base( double* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11302,7 +11306,7 @@ lapack_int LAPACK_slauum_base( float* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11318,7 +11322,7 @@ lapack_int LAPACK_zlauum_base( lapack_complex_double* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11341,7 +11345,7 @@ void LAPACK_dopgtr_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11360,7 +11364,7 @@ void LAPACK_sopgtr_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11379,7 +11383,7 @@ void LAPACK_dopmtr_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11398,7 +11402,7 @@ void LAPACK_sopmtr_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11424,7 +11428,7 @@ void LAPACK_dorbdb_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11450,7 +11454,7 @@ void LAPACK_sorbdb_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11476,7 +11480,7 @@ void LAPACK_dorcsd_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11502,7 +11506,7 @@ void LAPACK_sorcsd_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11525,7 +11529,7 @@ void LAPACK_dorcsd2by1_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11548,7 +11552,7 @@ void LAPACK_sorcsd2by1_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11566,7 +11570,7 @@ void LAPACK_dorgbr_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11584,7 +11588,7 @@ void LAPACK_sorgbr_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11682,7 +11686,7 @@ void LAPACK_dorgtr_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11700,7 +11704,7 @@ void LAPACK_sorgtr_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11753,7 +11757,7 @@ void LAPACK_dormbr_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11772,7 +11776,7 @@ void LAPACK_sormbr_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11791,7 +11795,7 @@ void LAPACK_dormhr_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11810,7 +11814,7 @@ void LAPACK_sormhr_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11829,7 +11833,7 @@ void LAPACK_dormlq_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11848,7 +11852,7 @@ void LAPACK_sormlq_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11867,7 +11871,7 @@ void LAPACK_dormql_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11886,7 +11890,7 @@ void LAPACK_sormql_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11905,7 +11909,7 @@ void LAPACK_dormqr_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11924,7 +11928,7 @@ void LAPACK_sormqr_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11943,7 +11947,7 @@ void LAPACK_dormrq_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11962,7 +11966,7 @@ void LAPACK_sormrq_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -11981,7 +11985,7 @@ void LAPACK_dormrz_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12000,7 +12004,7 @@ void LAPACK_sormrz_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12019,7 +12023,7 @@ void LAPACK_dormtr_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12038,7 +12042,7 @@ void LAPACK_sormtr_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12058,7 +12062,7 @@ void LAPACK_cpbcon_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12078,7 +12082,7 @@ void LAPACK_dpbcon_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12098,7 +12102,7 @@ void LAPACK_spbcon_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12118,7 +12122,7 @@ void LAPACK_zpbcon_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12137,7 +12141,7 @@ void LAPACK_cpbequ_base( float* amax, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12156,7 +12160,7 @@ void LAPACK_dpbequ_base( double* amax, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12175,7 +12179,7 @@ void LAPACK_spbequ_base( float* amax, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12194,7 +12198,7 @@ void LAPACK_zpbequ_base( double* amax, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12217,7 +12221,7 @@ void LAPACK_cpbrfs_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12240,7 +12244,7 @@ void LAPACK_dpbrfs_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12263,7 +12267,7 @@ void LAPACK_spbrfs_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12286,7 +12290,7 @@ void LAPACK_zpbrfs_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12302,7 +12306,7 @@ void LAPACK_cpbstf_base( lapack_complex_float* AB, lapack_int const* ldab, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12318,7 +12322,7 @@ void LAPACK_dpbstf_base( double* AB, lapack_int const* ldab, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12334,7 +12338,7 @@ void LAPACK_spbstf_base( float* AB, lapack_int const* ldab, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12350,7 +12354,7 @@ void LAPACK_zpbstf_base( lapack_complex_double* AB, lapack_int const* ldab, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12367,7 +12371,7 @@ void LAPACK_cpbsv_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12384,7 +12388,7 @@ void LAPACK_dpbsv_base( double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12401,7 +12405,7 @@ void LAPACK_spbsv_base( float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12418,7 +12422,7 @@ void LAPACK_zpbsv_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12444,7 +12448,7 @@ void LAPACK_cpbsvx_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12470,7 +12474,7 @@ void LAPACK_dpbsvx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12496,7 +12500,7 @@ void LAPACK_spbsvx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12522,7 +12526,7 @@ void LAPACK_zpbsvx_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12538,7 +12542,7 @@ void LAPACK_cpbtrf_base( lapack_complex_float* AB, lapack_int const* ldab, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12554,7 +12558,7 @@ void LAPACK_dpbtrf_base( double* AB, lapack_int const* ldab, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12570,7 +12574,7 @@ void LAPACK_spbtrf_base( float* AB, lapack_int const* ldab, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12586,7 +12590,7 @@ void LAPACK_zpbtrf_base( lapack_complex_double* AB, lapack_int const* ldab, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12603,7 +12607,7 @@ void LAPACK_cpbtrs_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12620,7 +12624,7 @@ void LAPACK_dpbtrs_base( double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12637,7 +12641,7 @@ void LAPACK_spbtrs_base( float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12654,7 +12658,7 @@ void LAPACK_zpbtrs_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12670,7 +12674,7 @@ void LAPACK_cpftrf_base( lapack_complex_float* A, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12686,7 +12690,7 @@ void LAPACK_dpftrf_base( double* A, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12702,7 +12706,7 @@ void LAPACK_spftrf_base( float* A, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12718,7 +12722,7 @@ void LAPACK_zpftrf_base( lapack_complex_double* A, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12734,7 +12738,7 @@ void LAPACK_cpftri_base( lapack_complex_float* A, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12750,7 +12754,7 @@ void LAPACK_dpftri_base( double* A, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12766,7 +12770,7 @@ void LAPACK_spftri_base( float* A, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12782,7 +12786,7 @@ void LAPACK_zpftri_base( lapack_complex_double* A, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12799,7 +12803,7 @@ void LAPACK_cpftrs_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12816,7 +12820,7 @@ void LAPACK_dpftrs_base( double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12833,7 +12837,7 @@ void LAPACK_spftrs_base( float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12850,7 +12854,7 @@ void LAPACK_zpftrs_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12870,7 +12874,7 @@ void LAPACK_cpocon_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12890,7 +12894,7 @@ void LAPACK_dpocon_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12910,7 +12914,7 @@ void LAPACK_spocon_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -12930,7 +12934,7 @@ void LAPACK_zpocon_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13025,7 +13029,7 @@ void LAPACK_cporfs_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13048,7 +13052,7 @@ void LAPACK_dporfs_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13071,7 +13075,7 @@ void LAPACK_sporfs_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13094,7 +13098,7 @@ void LAPACK_zporfs_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13121,7 +13125,7 @@ void LAPACK_cporfsx_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13148,7 +13152,7 @@ void LAPACK_dporfsx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13175,7 +13179,7 @@ void LAPACK_sporfsx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13202,7 +13206,7 @@ void LAPACK_zporfsx_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13219,7 +13223,7 @@ void LAPACK_cposv_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13236,7 +13240,7 @@ void LAPACK_dposv_base( double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13253,7 +13257,7 @@ void LAPACK_sposv_base( float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13270,7 +13274,7 @@ void LAPACK_zposv_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13290,7 +13294,7 @@ void LAPACK_dsposv_base( float* swork, lapack_int* iter, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13311,7 +13315,7 @@ void LAPACK_zcposv_base( double* rwork, lapack_int* iter, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13337,7 +13341,7 @@ void LAPACK_cposvx_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13363,7 +13367,7 @@ void LAPACK_dposvx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13389,7 +13393,7 @@ void LAPACK_sposvx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13415,7 +13419,7 @@ void LAPACK_zposvx_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13444,7 +13448,7 @@ void LAPACK_cposvxx_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13473,7 +13477,7 @@ void LAPACK_dposvxx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13502,7 +13506,7 @@ void LAPACK_sposvxx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13531,7 +13535,7 @@ void LAPACK_zposvxx_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13547,7 +13551,7 @@ void LAPACK_cpotf2_base( lapack_complex_float* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13563,7 +13567,7 @@ void LAPACK_dpotf2_base( double* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13579,7 +13583,7 @@ void LAPACK_spotf2_base( float* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13595,7 +13599,7 @@ void LAPACK_zpotf2_base( lapack_complex_double* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13611,7 +13615,7 @@ lapack_int LAPACK_cpotrf_base( lapack_complex_float* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13627,7 +13631,7 @@ lapack_int LAPACK_dpotrf_base( double* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13643,7 +13647,7 @@ lapack_int LAPACK_spotrf_base( float* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13659,7 +13663,7 @@ lapack_int LAPACK_zpotrf_base( lapack_complex_double* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13675,7 +13679,7 @@ void LAPACK_cpotrf2_base( lapack_complex_float* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13691,7 +13695,7 @@ void LAPACK_dpotrf2_base( double* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13707,7 +13711,7 @@ void LAPACK_spotrf2_base( float* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13723,7 +13727,7 @@ void LAPACK_zpotrf2_base( lapack_complex_double* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13739,7 +13743,7 @@ void LAPACK_cpotri_base( lapack_complex_float* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13755,7 +13759,7 @@ void LAPACK_dpotri_base( double* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13771,7 +13775,7 @@ void LAPACK_spotri_base( float* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13787,7 +13791,7 @@ void LAPACK_zpotri_base( lapack_complex_double* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13804,7 +13808,7 @@ void LAPACK_cpotrs_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13821,7 +13825,7 @@ void LAPACK_dpotrs_base( double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13838,7 +13842,7 @@ void LAPACK_spotrs_base( float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13855,7 +13859,7 @@ void LAPACK_zpotrs_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13875,7 +13879,7 @@ void LAPACK_cppcon_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13895,7 +13899,7 @@ void LAPACK_dppcon_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13915,7 +13919,7 @@ void LAPACK_sppcon_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13935,7 +13939,7 @@ void LAPACK_zppcon_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13954,7 +13958,7 @@ void LAPACK_cppequ_base( float* amax, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13973,7 +13977,7 @@ void LAPACK_dppequ_base( double* amax, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -13992,7 +13996,7 @@ void LAPACK_sppequ_base( float* amax, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14011,7 +14015,7 @@ void LAPACK_zppequ_base( double* amax, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14034,7 +14038,7 @@ void LAPACK_cpprfs_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14057,7 +14061,7 @@ void LAPACK_dpprfs_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14080,7 +14084,7 @@ void LAPACK_spprfs_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14103,7 +14107,7 @@ void LAPACK_zpprfs_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14120,7 +14124,7 @@ void LAPACK_cppsv_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14137,7 +14141,7 @@ void LAPACK_dppsv_base( double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14154,7 +14158,7 @@ void LAPACK_sppsv_base( float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14171,7 +14175,7 @@ void LAPACK_zppsv_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14197,7 +14201,7 @@ void LAPACK_cppsvx_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14223,7 +14227,7 @@ void LAPACK_dppsvx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14249,7 +14253,7 @@ void LAPACK_sppsvx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14275,7 +14279,7 @@ void LAPACK_zppsvx_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14291,7 +14295,7 @@ void LAPACK_cpptrf_base( lapack_complex_float* AP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14307,7 +14311,7 @@ void LAPACK_dpptrf_base( double* AP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14323,7 +14327,7 @@ void LAPACK_spptrf_base( float* AP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14339,7 +14343,7 @@ void LAPACK_zpptrf_base( lapack_complex_double* AP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14355,7 +14359,7 @@ void LAPACK_cpptri_base( lapack_complex_float* AP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14371,7 +14375,7 @@ void LAPACK_dpptri_base( double* AP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14387,7 +14391,7 @@ void LAPACK_spptri_base( float* AP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14403,7 +14407,7 @@ void LAPACK_zpptri_base( lapack_complex_double* AP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14420,7 +14424,7 @@ void LAPACK_cpptrs_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14437,7 +14441,7 @@ void LAPACK_dpptrs_base( double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14454,7 +14458,7 @@ void LAPACK_spptrs_base( float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14471,7 +14475,7 @@ void LAPACK_zpptrs_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14489,7 +14493,7 @@ void LAPACK_cpstrf_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14507,7 +14511,7 @@ void LAPACK_dpstrf_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14525,7 +14529,7 @@ void LAPACK_spstrf_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14543,7 +14547,7 @@ void LAPACK_zpstrf_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14602,7 +14606,7 @@ void LAPACK_cpteqr_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14621,7 +14625,7 @@ void LAPACK_dpteqr_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14640,7 +14644,7 @@ void LAPACK_spteqr_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14659,7 +14663,7 @@ void LAPACK_zpteqr_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14684,7 +14688,7 @@ void LAPACK_cptrfs_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14737,7 +14741,7 @@ void LAPACK_zptrfs_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14795,7 +14799,7 @@ void LAPACK_cptsvx_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14820,7 +14824,7 @@ void LAPACK_dptsvx_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14845,7 +14849,7 @@ void LAPACK_sptsvx_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14871,7 +14875,7 @@ void LAPACK_zptsvx_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14917,7 +14921,7 @@ void LAPACK_cpttrs_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14951,7 +14955,7 @@ void LAPACK_zpttrs_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14970,7 +14974,7 @@ void LAPACK_dsbev_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -14989,7 +14993,7 @@ void LAPACK_ssbev_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15008,7 +15012,7 @@ void LAPACK_dsbev_2stage_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15027,7 +15031,7 @@ void LAPACK_ssbev_2stage_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15047,7 +15051,7 @@ void LAPACK_dsbevd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15067,7 +15071,7 @@ void LAPACK_ssbevd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15087,7 +15091,7 @@ void LAPACK_dsbevd_2stage_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15107,7 +15111,7 @@ void LAPACK_ssbevd_2stage_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15131,7 +15135,7 @@ void LAPACK_dsbevx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15155,7 +15159,7 @@ void LAPACK_ssbevx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15179,7 +15183,7 @@ void LAPACK_dsbevx_2stage_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15203,7 +15207,7 @@ void LAPACK_ssbevx_2stage_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15222,7 +15226,7 @@ void LAPACK_dsbgst_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15241,7 +15245,7 @@ void LAPACK_ssbgst_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15261,7 +15265,7 @@ void LAPACK_dsbgv_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15281,7 +15285,7 @@ void LAPACK_ssbgv_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15302,7 +15306,7 @@ void LAPACK_dsbgvd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15323,7 +15327,7 @@ void LAPACK_ssbgvd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15348,7 +15352,7 @@ void LAPACK_dsbgvx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15373,7 +15377,7 @@ void LAPACK_ssbgvx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15393,7 +15397,7 @@ void LAPACK_dsbtrd_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15413,7 +15417,7 @@ void LAPACK_ssbtrd_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15431,7 +15435,7 @@ void LAPACK_dsfrk_base( double const* beta, double* C #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15449,7 +15453,7 @@ void LAPACK_ssfrk_base( float const* beta, float* C #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15468,7 +15472,7 @@ void LAPACK_cspcon_base( lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15488,7 +15492,7 @@ void LAPACK_dspcon_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15508,7 +15512,7 @@ void LAPACK_sspcon_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15527,7 +15531,7 @@ void LAPACK_zspcon_base( lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15546,7 +15550,7 @@ void LAPACK_dspev_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15565,7 +15569,7 @@ void LAPACK_sspev_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15585,7 +15589,7 @@ void LAPACK_dspevd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15605,7 +15609,7 @@ void LAPACK_sspevd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15628,7 +15632,7 @@ void LAPACK_dspevx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15651,7 +15655,7 @@ void LAPACK_sspevx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15668,7 +15672,7 @@ void LAPACK_dspgst_base( double const* BP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15685,7 +15689,7 @@ void LAPACK_sspgst_base( float const* BP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15705,7 +15709,7 @@ void LAPACK_dspgv_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15725,7 +15729,7 @@ void LAPACK_sspgv_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15746,7 +15750,7 @@ void LAPACK_dspgvd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15767,7 +15771,7 @@ void LAPACK_sspgvd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15791,7 +15795,7 @@ void LAPACK_dspgvx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15815,7 +15819,7 @@ void LAPACK_sspgvx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15838,7 +15842,7 @@ void LAPACK_csprfs_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15861,7 +15865,7 @@ void LAPACK_dsprfs_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15884,7 +15888,7 @@ void LAPACK_ssprfs_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15907,7 +15911,7 @@ void LAPACK_zsprfs_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15924,7 +15928,7 @@ void LAPACK_cspsv_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15941,7 +15945,7 @@ void LAPACK_dspsv_base( double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15958,7 +15962,7 @@ void LAPACK_sspsv_base( float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15975,7 +15979,7 @@ void LAPACK_zspsv_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -15999,7 +16003,7 @@ void LAPACK_cspsvx_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16023,7 +16027,7 @@ void LAPACK_dspsvx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16047,7 +16051,7 @@ void LAPACK_sspsvx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16071,7 +16075,7 @@ void LAPACK_zspsvx_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16090,7 +16094,7 @@ void LAPACK_dsptrd_base( double* tau, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16109,7 +16113,7 @@ void LAPACK_ssptrd_base( float* tau, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16125,7 +16129,7 @@ void LAPACK_csptrf_base( lapack_complex_float* AP, lapack_int* ipiv, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16141,7 +16145,7 @@ void LAPACK_dsptrf_base( double* AP, lapack_int* ipiv, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16157,7 +16161,7 @@ void LAPACK_ssptrf_base( float* AP, lapack_int* ipiv, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16173,7 +16177,7 @@ void LAPACK_zsptrf_base( lapack_complex_double* AP, lapack_int* ipiv, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16190,7 +16194,7 @@ void LAPACK_csptri_base( lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16207,7 +16211,7 @@ void LAPACK_dsptri_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16224,7 +16228,7 @@ void LAPACK_ssptri_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16241,7 +16245,7 @@ void LAPACK_zsptri_base( lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16258,7 +16262,7 @@ void LAPACK_csptrs_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16275,7 +16279,7 @@ void LAPACK_dsptrs_base( double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16292,7 +16296,7 @@ void LAPACK_ssptrs_base( float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16309,7 +16313,7 @@ void LAPACK_zsptrs_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16332,7 +16336,7 @@ void LAPACK_dstebz_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16355,7 +16359,7 @@ void LAPACK_sstebz_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16376,7 +16380,7 @@ void LAPACK_cstedc_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16396,7 +16400,7 @@ void LAPACK_dstedc_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16416,7 +16420,7 @@ void LAPACK_sstedc_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16437,7 +16441,7 @@ void LAPACK_zstedc_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16461,7 +16465,7 @@ void LAPACK_cstegr_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16485,7 +16489,7 @@ void LAPACK_dstegr_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16509,7 +16513,7 @@ void LAPACK_sstegr_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16533,7 +16537,7 @@ void LAPACK_zstegr_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16600,7 +16604,7 @@ void LAPACK_cstemr_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16623,7 +16627,7 @@ void LAPACK_dstemr_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16646,7 +16650,7 @@ void LAPACK_sstemr_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16669,7 +16673,7 @@ void LAPACK_zstemr_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16688,7 +16692,7 @@ void LAPACK_csteqr_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16707,7 +16711,7 @@ void LAPACK_dsteqr_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16726,7 +16730,7 @@ void LAPACK_ssteqr_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16745,7 +16749,7 @@ void LAPACK_zsteqr_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16778,7 +16782,7 @@ void LAPACK_dstev_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16797,7 +16801,7 @@ void LAPACK_sstev_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16817,7 +16821,7 @@ void LAPACK_dstevd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16837,7 +16841,7 @@ void LAPACK_sstevd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16861,7 +16865,7 @@ void LAPACK_dstevr_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16885,7 +16889,7 @@ void LAPACK_sstevr_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16909,7 +16913,7 @@ void LAPACK_dstevx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16933,7 +16937,7 @@ void LAPACK_sstevx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16952,7 +16956,7 @@ void LAPACK_csycon_base( lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16972,7 +16976,7 @@ void LAPACK_dsycon_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -16992,7 +16996,7 @@ void LAPACK_ssycon_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17011,7 +17015,7 @@ void LAPACK_zsycon_base( lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17031,7 +17035,7 @@ void LAPACK_csycon_3_base( lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17052,7 +17056,7 @@ void LAPACK_dsycon_3_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17073,7 +17077,7 @@ void LAPACK_ssycon_3_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17093,7 +17097,7 @@ void LAPACK_zsycon_3_base( lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17110,7 +17114,7 @@ void LAPACK_csyconv_base( lapack_complex_float* E, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17127,7 +17131,7 @@ void LAPACK_dsyconv_base( double* E, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17144,7 +17148,7 @@ void LAPACK_ssyconv_base( float* E, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17161,7 +17165,7 @@ void LAPACK_zsyconv_base( lapack_complex_double* E, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17181,7 +17185,7 @@ void LAPACK_csyequb_base( lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17201,7 +17205,7 @@ void LAPACK_dsyequb_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17221,7 +17225,7 @@ void LAPACK_ssyequb_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17241,7 +17245,7 @@ void LAPACK_zsyequb_base( lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17259,7 +17263,7 @@ void LAPACK_dsyev_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17277,7 +17281,7 @@ void LAPACK_ssyev_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17295,7 +17299,7 @@ void LAPACK_dsyev_2stage_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17313,7 +17317,7 @@ void LAPACK_ssyev_2stage_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17332,7 +17336,7 @@ void LAPACK_dsyevd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17351,7 +17355,7 @@ void LAPACK_ssyevd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17370,7 +17374,7 @@ void LAPACK_dsyevd_2stage_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17389,7 +17393,7 @@ void LAPACK_ssyevd_2stage_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17412,7 +17416,7 @@ void LAPACK_dsyevr_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17435,7 +17439,7 @@ void LAPACK_ssyevr_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17458,7 +17462,7 @@ void LAPACK_dsyevr_2stage_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17481,7 +17485,7 @@ void LAPACK_ssyevr_2stage_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17504,7 +17508,7 @@ void LAPACK_dsyevx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17527,7 +17531,7 @@ void LAPACK_ssyevx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17550,7 +17554,7 @@ void LAPACK_dsyevx_2stage_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17573,7 +17577,7 @@ void LAPACK_ssyevx_2stage_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17590,7 +17594,7 @@ void LAPACK_dsygst_base( double const* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17607,7 +17611,7 @@ void LAPACK_ssygst_base( float const* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17626,7 +17630,7 @@ void LAPACK_dsygv_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17645,7 +17649,7 @@ void LAPACK_ssygv_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17664,7 +17668,7 @@ void LAPACK_dsygv_2stage_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17683,7 +17687,7 @@ void LAPACK_ssygv_2stage_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17703,7 +17707,7 @@ void LAPACK_dsygvd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17723,7 +17727,7 @@ void LAPACK_ssygvd_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17747,7 +17751,7 @@ void LAPACK_dsygvx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17771,7 +17775,7 @@ void LAPACK_ssygvx_base( lapack_int* iwork, lapack_int* IFAIL, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17788,7 +17792,7 @@ void LAPACK_csyr_base( lapack_complex_float const* X, lapack_int const* incx, lapack_complex_float* A, lapack_int const* lda #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17805,7 +17809,7 @@ void LAPACK_zsyr_base( lapack_complex_double const* X, lapack_int const* incx, lapack_complex_double* A, lapack_int const* lda #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17828,7 +17832,7 @@ void LAPACK_csyrfs_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17851,7 +17855,7 @@ void LAPACK_dsyrfs_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17874,7 +17878,7 @@ void LAPACK_ssyrfs_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17897,7 +17901,7 @@ void LAPACK_zsyrfs_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17924,7 +17928,7 @@ void LAPACK_csyrfsx_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17951,7 +17955,7 @@ void LAPACK_dsyrfsx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -17978,7 +17982,7 @@ void LAPACK_ssyrfsx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18005,7 +18009,7 @@ void LAPACK_zsyrfsx_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18023,7 +18027,7 @@ void LAPACK_csysv_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18041,7 +18045,7 @@ void LAPACK_dsysv_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18059,7 +18063,7 @@ void LAPACK_ssysv_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18077,7 +18081,7 @@ void LAPACK_zsysv_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18095,7 +18099,7 @@ void LAPACK_csysv_aa_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18113,7 +18117,7 @@ void LAPACK_dsysv_aa_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18131,7 +18135,7 @@ void LAPACK_ssysv_aa_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18149,7 +18153,7 @@ void LAPACK_zsysv_aa_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18168,7 +18172,7 @@ void LAPACK_csysv_aa_2stage_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18187,7 +18191,7 @@ void LAPACK_dsysv_aa_2stage_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18206,7 +18210,7 @@ void LAPACK_ssysv_aa_2stage_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18225,7 +18229,7 @@ void LAPACK_zsysv_aa_2stage_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18244,7 +18248,7 @@ void LAPACK_csysv_rk_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18263,7 +18267,7 @@ void LAPACK_dsysv_rk_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18282,7 +18286,7 @@ void LAPACK_ssysv_rk_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18301,7 +18305,7 @@ void LAPACK_zsysv_rk_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18319,7 +18323,7 @@ void LAPACK_csysv_rook_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18337,7 +18341,7 @@ void LAPACK_dsysv_rook_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18355,7 +18359,7 @@ void LAPACK_ssysv_rook_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18373,7 +18377,7 @@ void LAPACK_zsysv_rook_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18397,7 +18401,7 @@ void LAPACK_csysvx_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18421,7 +18425,7 @@ void LAPACK_dsysvx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18445,7 +18449,7 @@ void LAPACK_ssysvx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18469,7 +18473,7 @@ void LAPACK_zsysvx_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18498,7 +18502,7 @@ void LAPACK_csysvxx_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18527,7 +18531,7 @@ void LAPACK_dsysvxx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18556,7 +18560,7 @@ void LAPACK_ssysvxx_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18585,7 +18589,7 @@ void LAPACK_zsysvxx_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18600,7 +18604,7 @@ void LAPACK_csyswapr_base( lapack_int const* n, lapack_complex_float* A, lapack_int const* lda, lapack_int const* i1, lapack_int const* i2 #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18615,7 +18619,7 @@ void LAPACK_dsyswapr_base( lapack_int const* n, double* A, lapack_int const* lda, lapack_int const* i1, lapack_int const* i2 #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18630,7 +18634,7 @@ void LAPACK_ssyswapr_base( lapack_int const* n, float* A, lapack_int const* lda, lapack_int const* i1, lapack_int const* i2 #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18645,7 +18649,7 @@ void LAPACK_zsyswapr_base( lapack_int const* n, lapack_complex_double* A, lapack_int const* lda, lapack_int const* i1, lapack_int const* i2 #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18665,7 +18669,7 @@ void LAPACK_dsytrd_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18685,7 +18689,7 @@ void LAPACK_ssytrd_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18706,7 +18710,7 @@ void LAPACK_dsytrd_2stage_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18727,7 +18731,7 @@ void LAPACK_ssytrd_2stage_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18744,7 +18748,7 @@ void LAPACK_csytrf_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18761,7 +18765,7 @@ void LAPACK_dsytrf_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18778,7 +18782,7 @@ void LAPACK_ssytrf_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18795,7 +18799,7 @@ void LAPACK_zsytrf_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18812,7 +18816,7 @@ void LAPACK_csytrf_aa_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18829,7 +18833,7 @@ void LAPACK_dsytrf_aa_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18846,7 +18850,7 @@ void LAPACK_ssytrf_aa_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18863,7 +18867,7 @@ void LAPACK_zsytrf_aa_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18881,7 +18885,7 @@ void LAPACK_csytrf_aa_2stage_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18899,7 +18903,7 @@ void LAPACK_dsytrf_aa_2stage_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18917,7 +18921,7 @@ void LAPACK_ssytrf_aa_2stage_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18935,7 +18939,7 @@ void LAPACK_zsytrf_aa_2stage_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18953,7 +18957,7 @@ void LAPACK_csytrf_rk_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18971,7 +18975,7 @@ void LAPACK_dsytrf_rk_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -18989,7 +18993,7 @@ void LAPACK_ssytrf_rk_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19007,7 +19011,7 @@ void LAPACK_zsytrf_rk_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19024,7 +19028,7 @@ void LAPACK_csytrf_rook_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19041,7 +19045,7 @@ void LAPACK_dsytrf_rook_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19058,7 +19062,7 @@ void LAPACK_ssytrf_rook_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19075,7 +19079,7 @@ void LAPACK_zsytrf_rook_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19092,7 +19096,7 @@ void LAPACK_csytri_base( lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19109,7 +19113,7 @@ void LAPACK_dsytri_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19126,7 +19130,7 @@ void LAPACK_ssytri_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19143,7 +19147,7 @@ void LAPACK_zsytri_base( lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19160,7 +19164,7 @@ void LAPACK_csytri2_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19177,7 +19181,7 @@ void LAPACK_dsytri2_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19194,7 +19198,7 @@ void LAPACK_ssytri2_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19211,7 +19215,7 @@ void LAPACK_zsytri2_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19228,7 +19232,7 @@ void LAPACK_csytri2x_base( lapack_complex_float* work, lapack_int const* nb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19245,7 +19249,7 @@ void LAPACK_dsytri2x_base( double* work, lapack_int const* nb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19262,7 +19266,7 @@ void LAPACK_ssytri2x_base( float* work, lapack_int const* nb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19279,7 +19283,7 @@ void LAPACK_zsytri2x_base( lapack_complex_double* work, lapack_int const* nb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19297,7 +19301,7 @@ void LAPACK_csytri_3_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19315,7 +19319,7 @@ void LAPACK_dsytri_3_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19333,7 +19337,7 @@ void LAPACK_ssytri_3_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19351,7 +19355,7 @@ void LAPACK_zsytri_3_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19368,7 +19372,7 @@ void LAPACK_csytrs_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19385,7 +19389,7 @@ void LAPACK_dsytrs_base( double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19402,7 +19406,7 @@ void LAPACK_ssytrs_base( float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19419,7 +19423,7 @@ void LAPACK_zsytrs_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19437,7 +19441,7 @@ void LAPACK_csytrs2_base( lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19455,7 +19459,7 @@ void LAPACK_dsytrs2_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19473,7 +19477,7 @@ void LAPACK_ssytrs2_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19491,7 +19495,7 @@ void LAPACK_zsytrs2_base( lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19509,7 +19513,7 @@ void LAPACK_csytrs_3_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19527,7 +19531,7 @@ void LAPACK_dsytrs_3_base( double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19545,7 +19549,7 @@ void LAPACK_ssytrs_3_base( float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19563,7 +19567,7 @@ void LAPACK_zsytrs_3_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19581,7 +19585,7 @@ void LAPACK_csytrs_aa_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19599,7 +19603,7 @@ void LAPACK_dsytrs_aa_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19617,7 +19621,7 @@ void LAPACK_ssytrs_aa_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19635,7 +19639,7 @@ void LAPACK_zsytrs_aa_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19653,7 +19657,7 @@ void LAPACK_csytrs_aa_2stage_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19671,7 +19675,7 @@ void LAPACK_dsytrs_aa_2stage_base( double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19689,7 +19693,7 @@ void LAPACK_ssytrs_aa_2stage_base( float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19707,7 +19711,7 @@ void LAPACK_zsytrs_aa_2stage_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19724,7 +19728,7 @@ void LAPACK_csytrs_rook_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19741,7 +19745,7 @@ void LAPACK_dsytrs_rook_base( double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19758,7 +19762,7 @@ void LAPACK_ssytrs_rook_base( float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19775,7 +19779,7 @@ void LAPACK_zsytrs_rook_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19794,7 +19798,7 @@ void LAPACK_ctbcon_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19813,7 +19817,7 @@ void LAPACK_dtbcon_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19832,7 +19836,7 @@ void LAPACK_stbcon_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19851,7 +19855,7 @@ void LAPACK_ztbcon_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19873,7 +19877,7 @@ void LAPACK_ctbrfs_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19895,7 +19899,7 @@ void LAPACK_dtbrfs_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19917,7 +19921,7 @@ void LAPACK_stbrfs_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19939,7 +19943,7 @@ void LAPACK_ztbrfs_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19956,7 +19960,7 @@ void LAPACK_ctbtrs_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19973,7 +19977,7 @@ void LAPACK_dtbtrs_base( double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -19990,7 +19994,7 @@ void LAPACK_stbtrs_base( float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20007,7 +20011,7 @@ void LAPACK_ztbtrs_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20024,7 +20028,7 @@ void LAPACK_ctfsm_base( lapack_complex_float const* A, lapack_complex_float* B, lapack_int const* ldb #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20041,7 +20045,7 @@ void LAPACK_dtfsm_base( double const* A, double* B, lapack_int const* ldb #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20058,7 +20062,7 @@ void LAPACK_stfsm_base( float const* A, float* B, lapack_int const* ldb #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20075,7 +20079,7 @@ void LAPACK_ztfsm_base( lapack_complex_double const* A, lapack_complex_double* B, lapack_int const* ldb #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20091,7 +20095,7 @@ void LAPACK_ctftri_base( lapack_complex_float* A, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20107,7 +20111,7 @@ void LAPACK_dtftri_base( double* A, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20123,7 +20127,7 @@ void LAPACK_stftri_base( float* A, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20139,7 +20143,7 @@ void LAPACK_ztftri_base( lapack_complex_double* A, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20156,7 +20160,7 @@ void LAPACK_ctfttp_base( lapack_complex_float* AP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20173,7 +20177,7 @@ void LAPACK_dtfttp_base( double* AP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20190,7 +20194,7 @@ void LAPACK_stfttp_base( float* AP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20207,7 +20211,7 @@ void LAPACK_ztfttp_base( lapack_complex_double* AP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20224,7 +20228,7 @@ void LAPACK_ctfttr_base( lapack_complex_float* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20241,7 +20245,7 @@ void LAPACK_dtfttr_base( double* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20258,7 +20262,7 @@ void LAPACK_stfttr_base( float* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20275,7 +20279,7 @@ void LAPACK_ztfttr_base( lapack_complex_double* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20297,7 +20301,7 @@ void LAPACK_ctgevc_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20318,7 +20322,7 @@ void LAPACK_dtgevc_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20339,7 +20343,7 @@ void LAPACK_stgevc_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20361,7 +20365,7 @@ void LAPACK_ztgevc_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20490,7 +20494,7 @@ void LAPACK_ctgsja_base( lapack_complex_float* work, lapack_int* ncycle, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20515,7 +20519,7 @@ void LAPACK_dtgsja_base( double* work, lapack_int* ncycle, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20540,7 +20544,7 @@ void LAPACK_stgsja_base( float* work, lapack_int* ncycle, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20565,7 +20569,7 @@ void LAPACK_ztgsja_base( lapack_complex_double* work, lapack_int* ncycle, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20589,7 +20593,7 @@ void LAPACK_ctgsna_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20613,7 +20617,7 @@ void LAPACK_dtgsna_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20637,7 +20641,7 @@ void LAPACK_stgsna_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20661,7 +20665,7 @@ void LAPACK_ztgsna_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20686,7 +20690,7 @@ void LAPACK_ctgsyl_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20711,7 +20715,7 @@ void LAPACK_dtgsyl_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20736,7 +20740,7 @@ void LAPACK_stgsyl_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20761,7 +20765,7 @@ void LAPACK_ztgsyl_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20780,7 +20784,7 @@ void LAPACK_ctpcon_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20799,7 +20803,7 @@ void LAPACK_dtpcon_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20818,7 +20822,7 @@ void LAPACK_stpcon_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20837,7 +20841,7 @@ void LAPACK_ztpcon_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20925,7 +20929,7 @@ void LAPACK_ctpmlqt_base( lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20945,7 +20949,7 @@ void LAPACK_dtpmlqt_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20965,7 +20969,7 @@ void LAPACK_stpmlqt_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -20985,7 +20989,7 @@ void LAPACK_ztpmlqt_base( lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21005,7 +21009,7 @@ void LAPACK_ctpmqrt_base( lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21025,7 +21029,7 @@ void LAPACK_dtpmqrt_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21045,7 +21049,7 @@ void LAPACK_stpmqrt_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21065,7 +21069,7 @@ void LAPACK_ztpmqrt_base( lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21152,7 +21156,7 @@ void LAPACK_ctprfb_base( lapack_complex_float* B, lapack_int const* ldb, lapack_complex_float* work, lapack_int const* ldwork #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21171,7 +21175,7 @@ void LAPACK_dtprfb_base( double* B, lapack_int const* ldb, double* work, lapack_int const* ldwork #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21190,7 +21194,7 @@ void LAPACK_stprfb_base( float* B, lapack_int const* ldb, float* work, lapack_int const* ldwork #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21209,7 +21213,7 @@ void LAPACK_ztprfb_base( lapack_complex_double* B, lapack_int const* ldb, lapack_complex_double* work, lapack_int const* ldwork #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21231,7 +21235,7 @@ void LAPACK_ctprfs_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21253,7 +21257,7 @@ void LAPACK_dtprfs_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21275,7 +21279,7 @@ void LAPACK_stprfs_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21297,7 +21301,7 @@ void LAPACK_ztprfs_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21313,7 +21317,7 @@ void LAPACK_ctptri_base( lapack_complex_float* AP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21329,7 +21333,7 @@ void LAPACK_dtptri_base( double* AP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21345,7 +21349,7 @@ void LAPACK_stptri_base( float* AP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21361,7 +21365,7 @@ void LAPACK_ztptri_base( lapack_complex_double* AP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21378,7 +21382,7 @@ void LAPACK_ctptrs_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21395,7 +21399,7 @@ void LAPACK_dtptrs_base( double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21412,7 +21416,7 @@ void LAPACK_stptrs_base( float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21429,7 +21433,7 @@ void LAPACK_ztptrs_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21446,7 +21450,7 @@ void LAPACK_ctpttf_base( lapack_complex_float* ARF, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21463,7 +21467,7 @@ void LAPACK_dtpttf_base( double* ARF, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21480,7 +21484,7 @@ void LAPACK_stpttf_base( float* ARF, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21497,7 +21501,7 @@ void LAPACK_ztpttf_base( lapack_complex_double* ARF, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21514,7 +21518,7 @@ void LAPACK_ctpttr_base( lapack_complex_float* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21531,7 +21535,7 @@ void LAPACK_dtpttr_base( double* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21548,7 +21552,7 @@ void LAPACK_stpttr_base( float* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21565,7 +21569,7 @@ void LAPACK_ztpttr_base( lapack_complex_double* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21584,7 +21588,7 @@ void LAPACK_ctrcon_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21603,7 +21607,7 @@ void LAPACK_dtrcon_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21622,7 +21626,7 @@ void LAPACK_strcon_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21641,7 +21645,7 @@ void LAPACK_ztrcon_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21662,7 +21666,7 @@ void LAPACK_ctrevc_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21682,7 +21686,7 @@ void LAPACK_dtrevc_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21702,7 +21706,7 @@ void LAPACK_strevc_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21723,7 +21727,7 @@ void LAPACK_ztrevc_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21744,7 +21748,7 @@ void LAPACK_ctrevc3_base( float* rwork, lapack_int const* lrwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21764,7 +21768,7 @@ void LAPACK_dtrevc3_base( double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21784,7 +21788,7 @@ void LAPACK_strevc3_base( float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21805,7 +21809,7 @@ void LAPACK_ztrevc3_base( double* rwork, lapack_int const* lrwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21822,7 +21826,7 @@ void LAPACK_ctrexc_base( lapack_complex_float* Q, lapack_int const* ldq, lapack_int const* ifst, lapack_int const* ilst, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21840,7 +21844,7 @@ void LAPACK_dtrexc_base( double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21858,7 +21862,7 @@ void LAPACK_strexc_base( float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21875,7 +21879,7 @@ void LAPACK_ztrexc_base( lapack_complex_double* Q, lapack_int const* ldq, lapack_int const* ifst, lapack_int const* ilst, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21897,7 +21901,7 @@ void LAPACK_ctrrfs_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21919,7 +21923,7 @@ void LAPACK_dtrrfs_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21941,7 +21945,7 @@ void LAPACK_strrfs_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21963,7 +21967,7 @@ void LAPACK_ztrrfs_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -21985,7 +21989,7 @@ void LAPACK_ctrsen_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22009,7 +22013,7 @@ void LAPACK_dtrsen_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22033,7 +22037,7 @@ void LAPACK_strsen_base( lapack_int* iwork, lapack_int const* liwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22055,7 +22059,7 @@ void LAPACK_ztrsen_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22078,7 +22082,7 @@ void LAPACK_ctrsna_base( float* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22101,7 +22105,7 @@ void LAPACK_dtrsna_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22124,7 +22128,7 @@ void LAPACK_strsna_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22147,7 +22151,7 @@ void LAPACK_ztrsna_base( double* rwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22166,7 +22170,7 @@ void LAPACK_ctrsyl_base( float* scale, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22185,7 +22189,7 @@ void LAPACK_dtrsyl_base( double* scale, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22204,7 +22208,7 @@ void LAPACK_strsyl_base( float* scale, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22223,7 +22227,7 @@ void LAPACK_ztrsyl_base( double* scale, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22242,7 +22246,7 @@ void LAPACK_ctrsyl3_base( float* swork, lapack_int const *ldswork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22262,7 +22266,7 @@ void LAPACK_dtrsyl3_base( double* swork, lapack_int const *ldswork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22282,7 +22286,7 @@ void LAPACK_strsyl3_base( float* swork, lapack_int const *ldswork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22301,7 +22305,7 @@ void LAPACK_ztrsyl3_base( double* swork, lapack_int const *ldswork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22317,7 +22321,7 @@ lapack_int LAPACK_ctrtri_base( lapack_complex_float* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22333,7 +22337,7 @@ lapack_int LAPACK_dtrtri_base( double* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22349,7 +22353,7 @@ lapack_int LAPACK_strtri_base( float* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22365,7 +22369,7 @@ lapack_int LAPACK_ztrtri_base( lapack_complex_double* A, lapack_int const* lda, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22382,7 +22386,7 @@ lapack_int LAPACK_ctrtrs_base( lapack_complex_float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22399,7 +22403,7 @@ lapack_int LAPACK_dtrtrs_base( double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22416,7 +22420,7 @@ lapack_int LAPACK_strtrs_base( float* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22433,7 +22437,7 @@ lapack_int LAPACK_ztrtrs_base( lapack_complex_double* B, lapack_int const* ldb, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22450,7 +22454,7 @@ void LAPACK_ctrttf_base( lapack_complex_float* ARF, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22467,7 +22471,7 @@ void LAPACK_dtrttf_base( double* ARF, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22484,7 +22488,7 @@ void LAPACK_strttf_base( float* ARF, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22501,7 +22505,7 @@ void LAPACK_ztrttf_base( lapack_complex_double* ARF, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22518,7 +22522,7 @@ void LAPACK_ctrttp_base( lapack_complex_float* AP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22535,7 +22539,7 @@ void LAPACK_dtrttp_base( double* AP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22552,7 +22556,7 @@ void LAPACK_strttp_base( float* AP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22569,7 +22573,7 @@ void LAPACK_ztrttp_base( lapack_complex_double* AP, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22627,7 +22631,7 @@ void LAPACK_cunbdb_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22653,7 +22657,7 @@ void LAPACK_zunbdb_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22680,7 +22684,7 @@ void LAPACK_cuncsd_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22707,7 +22711,7 @@ void LAPACK_zuncsd_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t, size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22731,7 +22735,7 @@ void LAPACK_cuncsd2by1_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22755,7 +22759,7 @@ void LAPACK_zuncsd2by1_base( lapack_int* iwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22773,7 +22777,7 @@ void LAPACK_cungbr_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22791,7 +22795,7 @@ void LAPACK_zungbr_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22889,7 +22893,7 @@ void LAPACK_cungtr_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22907,7 +22911,7 @@ void LAPACK_zungtr_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22960,7 +22964,7 @@ void LAPACK_cunmbr_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22979,7 +22983,7 @@ void LAPACK_zunmbr_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -22998,7 +23002,7 @@ void LAPACK_cunmhr_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -23017,7 +23021,7 @@ void LAPACK_zunmhr_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -23036,7 +23040,7 @@ void LAPACK_cunmlq_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -23055,7 +23059,7 @@ void LAPACK_zunmlq_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -23074,7 +23078,7 @@ void LAPACK_cunmql_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -23093,7 +23097,7 @@ void LAPACK_zunmql_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -23112,7 +23116,7 @@ void LAPACK_cunmqr_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -23131,7 +23135,7 @@ void LAPACK_zunmqr_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -23150,7 +23154,7 @@ void LAPACK_cunmrq_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -23169,7 +23173,7 @@ void LAPACK_zunmrq_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -23188,7 +23192,7 @@ void LAPACK_cunmrz_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -23207,7 +23211,7 @@ void LAPACK_zunmrz_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -23226,7 +23230,7 @@ void LAPACK_cunmtr_base( lapack_complex_float* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -23245,7 +23249,7 @@ void LAPACK_zunmtr_base( lapack_complex_double* work, lapack_int const* lwork, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -23264,7 +23268,7 @@ void LAPACK_cupgtr_base( lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -23283,7 +23287,7 @@ void LAPACK_zupgtr_base( lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t + , FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -23302,7 +23306,7 @@ void LAPACK_cupmtr_base( lapack_complex_float* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END @@ -23321,7 +23325,7 @@ void LAPACK_zupmtr_base( lapack_complex_double* work, lapack_int* info #ifdef LAPACK_FORTRAN_STRLEN_END - , size_t, size_t, size_t + , FORTRAN_STRLEN, FORTRAN_STRLEN, FORTRAN_STRLEN #endif ); #ifdef LAPACK_FORTRAN_STRLEN_END From 797ae08dbec065862a247481ddd364ed250f3320 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 6 Aug 2024 21:38:00 +0200 Subject: [PATCH 262/267] Add explanation of LAPACK_STRLEN --- Makefile.rule | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile.rule b/Makefile.rule index d25299775d..33209e9cd7 100644 --- a/Makefile.rule +++ b/Makefile.rule @@ -134,6 +134,12 @@ VERSION = 0.3.27.dev # Build LAPACK Deprecated functions since LAPACK 3.6.0 BUILD_LAPACK_DEPRECATED = 1 +# The variable type assumed for the length of character arguments when passing +# data between Fortran LAPACK and C BLAS (defaults to "size_t", but older GCC +# versions used "int"). Mismatches will not cause runtime failures but may result +# in build warnings or errors when building with link-time optimization (LTO) +# LAPACK_STRLEN=int + # Build RecursiveLAPACK on top of LAPACK # BUILD_RELAPACK = 1 # Have RecursiveLAPACK actually replace standard LAPACK routines instead of From ccc23338d74eec980fcd194794502f461e7e4bc7 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 7 Aug 2024 19:39:02 +0200 Subject: [PATCH 263/267] have the dummy GEMM3M kernel at least forward to regular GEMM --- kernel/generic/zgemm3mkernel_dump.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kernel/generic/zgemm3mkernel_dump.c b/kernel/generic/zgemm3mkernel_dump.c index a59bb08ce3..c4a614f0ae 100644 --- a/kernel/generic/zgemm3mkernel_dump.c +++ b/kernel/generic/zgemm3mkernel_dump.c @@ -25,10 +25,16 @@ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *****************************************************************************/ +#if 1 +#include "zgemmkernel_2x2.c" + + +#else #include "common.h" int CNAME(BLASLONG bm, BLASLONG bn, BLASLONG bk, FLOAT alphar, FLOAT alphai, FLOAT * ba, FLOAT * bb, FLOAT * C, BLASLONG ldc) { return 0; } +#endif From 46e331a91722da3bfcfe7bfbc17ce0d49d68430f Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 7 Aug 2024 19:41:10 +0200 Subject: [PATCH 264/267] remove the unworkable GEMM3M restriction from GENERIC again --- kernel/Makefile.L3 | 9 --------- 1 file changed, 9 deletions(-) diff --git a/kernel/Makefile.L3 b/kernel/Makefile.L3 index 87fd5ca100..ed1c74ecff 100644 --- a/kernel/Makefile.L3 +++ b/kernel/Makefile.L3 @@ -17,15 +17,6 @@ ifeq ($(ARCH), ia64) USE_GEMM3M = 1 endif -ifneq ($(DYNAMIC_ARCH), 1) -ifeq ($(TARGET), GENERIC) -USE_GEMM3M = 0 -endif -else -ifeq ($(CORE), GENERIC) -USE_GEMM3M = 0 -endif -endif ifeq ($(ARCH), arm) USE_TRMM = 1 From 76db713e797e237cda856294b19f2fa911bea7da Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 7 Aug 2024 21:37:20 +0200 Subject: [PATCH 265/267] fix invocation of GEMM3M tests --- test/Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/Makefile b/test/Makefile index 27afce8d25..cfb2d41f54 100644 --- a/test/Makefile +++ b/test/Makefile @@ -189,8 +189,11 @@ endif endif +ifeq ($(SUPPORT_GEMM3M),1) +level3: $(B3) $(S3) $(D3) $(C3) $(Z3) level3_3m +else level3: $(B3) $(S3) $(D3) $(C3) $(Z3) - +endif ifneq ($(CROSS), 1) rm -f ?BLAT3.SUMM @@ -263,7 +266,7 @@ endif endif -level3_3m : zblat3_3m cblat3_3m +level3_3m: zblat3_3m cblat3_3m ifneq ($(CROSS), 1) rm -f ?BLAT3_3M.SUMM OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./cblat3_3m < ./cblat3_3m.dat From 7878976236e314829de2093b9864f635fa8354c2 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 8 Aug 2024 18:03:38 +0200 Subject: [PATCH 266/267] disable forwarding from SBGEMM to SBGEMV for now --- interface/gemm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface/gemm.c b/interface/gemm.c index ac58cf27fa..64b8b620cf 100644 --- a/interface/gemm.c +++ b/interface/gemm.c @@ -498,7 +498,7 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANS args.m, args.n, args.k, args.lda, args.ldb, args.ldc); #endif -#if defined(GEMM_GEMV_FORWARD) && !defined(GEMM3M) && !defined(COMPLEX) +#if defined(GEMM_GEMV_FORWARD) && !defined(GEMM3M) && !defined(COMPLEX) && !defined(BFLOAT16) // Check if we can convert GEMM -> GEMV if (args.k != 0) { if (args.n == 1) { From 1df95bb23adb6d9ea669c789afef58a0706a96bf Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 8 Aug 2024 18:51:25 +0200 Subject: [PATCH 267/267] Update Changelog.txt for 0.3.28 --- Changelog.txt | 123 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/Changelog.txt b/Changelog.txt index 03c3cfbd97..7f89a2eab7 100644 --- a/Changelog.txt +++ b/Changelog.txt @@ -1,4 +1,127 @@ OpenBLAS ChangeLog +==================================================================== +Version 0.3.28 + 8-Aug-2024 + +general: +- Reworked the unfinished implementation of HUGETLB from GotoBLAS + for allocating huge memory pages as buffers on suitable systems +- Changed the unfinished implementation of GEMM3M for the generic + target on all architectures to at least forward to regular GEMM +- Improved multithreaded GEMM performance for large non-skinny matrices +- Improved BLAS3 performance on larger multicore systems through improved + parallelism +- Improved performance of the initial memory allocation by reducing + locking overhead +- Improved performance of GBMV at small problem sizes by introducing + a size barrier for the switch to multithreading +- Added an implementation of the CBLAS_GEMM_BATCH extension +- Fixed miscompilation of CAXPYC and ZAXPYC on all architectures in + CMAKE builds (error introduced in 0.3.27) +- Fixed corner cases involving the handling of NAN and INFINITY + arguments in ?SCAL on all architectures +- Added support for cross-compiling to WEBM with CMAKE (in addition + to the already present makefile support) +- Fixed NAN handling and potential accuracy issues in compilations with + Intel ICX by supplying a suitable fp-model option by default +- The contents of the github project wiki have been converted into + a new set of documentation included with the source code. +- It is now possible to register a callback function that replaces + the built-in support for multithreading with an external backend + like TBB (openblas_set_threads_callback_function) +- Fixed potential duplication of suffixes in shared library naming +- Improved C compiler detection by the build system to tolerate more + naming variants for gcc builds +- Fixed an unnecessary dependency of the utest on CBLAS +- Fixed spurious error reports from the BLAS extensions utest +- Fixed unwanted invocation of the GEMM3M tests in cross-compilation +- Fixed a flaw in the makefile build that could lead to the pkgconfig + file containing an entry of UNKNOWN for the target cpu after installing +- Integrated fixes from the Reference-LAPACK project: + - Fixed uninitialized variables in the LAPACK tests for ?QP3RK (PR 961) + - Fixed potential bounds error in ?UNHR_COL/?ORHR_COL (PR 1018) + - Fixed potential infinite loop in the LAPACK testsuite (PR 1024) + - Make the variable type used for hidden length arguments configurable (PR 1025) + - Fixed SYTRD workspace computation and various typos (PR 1030) + - Prevent compiler use of FMA that could increase numerical error in ?GEEVX (PR 1033) + +x86-64: +- reverted thread management under Windows to its state before 0.3.26 + due to signs of race conditions in some circumstances now under study +- fixed accidental selection of the unoptimized generic SBGEMM kernel + in CMAKE builds for CooperLake and SapphireRapids targets +- fixed a potential thread buffer overrun in SBSTOBF16 on small systems +- fixed an accuracy issue in ZSCAL introduced in 0.3.26 +- fixed compilation with CMAKE and recent releases of LLVM +- added support for Intel Emerald Rapids and Meteor Lake cpus +- added autodetection support for the Zhaoxin KX-7000 cpu +- fixed autodetection of Intel Prescott (probably broken since 0.3.19) +- fixed compilation for older targets with the Yocto SDK +- fixed compilation of the converter-generated C versions + of the LAPACK sources with gcc-14 +- improved compiler options when building with CMAKE and LLVM for + AVX512-capable targets +- added support for supplying the L2 cache size via an environment + variable (OPENBLAS_L2_SIZE) in case it is not correctly reported + (as in some VM configurations) +- improved the error message shown when thread creation fails on startup +- fixed setting the rpath entry of the dylib in CMAKE builds on MacOS + +arm: +- fixed building for baremetal targets with make + +arm64: +- Added a fast path forwarding SGEMM and DGEMM calls with a 1xN or Mx1 + matrix to the corresponding GEMV kernel +- added optimized SGEMV and DGEMV kernels for A64FX +- added optimized SVE kernels for small-matrix GEMM +- added A64FX to the cpu list for DYNAMIC_ARCH +- fixed building with support for cpu affinity +- worked around accuracy problems with C/ZNRM2 on NeoverseN1 and + Apple M targets +- improved GEMM performance on Neoverse V1 +- fixed compilation for NEOVERSEN2 with older compilers +- fixed potential miscompilation of the SVE SDOT and DDOT kernels +- fixed potential miscompilation of the non-SVE CDOT and ZDOT kernels +- fixed a potential overflow when using very large user-defined BUFFERSIZE +- fixed setting the rpath entry of the dylib in CMAKE builds on MacOS + +power: +- Added a fast path forwarding SGEMM and DGEMM calls with a 1xN or Mx1 + matrix to the corresponding GEMV kernel +- significantly improved performance of SBGEMM on POWER10 +- fixed compilation with OpenMP and the XLF compiler +- fixed building of the BLAS extension utests under AIX +- fixed building of parts of the LAPACK testsuite with XLF +- fixed CSWAP/ZSWAP on big-endian POWER10 targets +- fixed a performance regression in SAXPY on POWER10 with OpenXL +- fixed accuracy issues in CSCAL/ZSCAL when compiled with LLVM +- fixed building for POWER9 under FreeBSD +- fixed a potential overflow when using very large user-defined BUFFERSIZE +- fixed an accuracy issue in the POWER6 kernels for GEMM and GEMV + +riscv64: +- Added a fast path forwarding SGEMM and DGEMM calls with a 1xN or Mx1 + matrix to the corresponding GEMV kernel +- fixed building for RISCV64_GENERIC with OpenMP enabled +- added DYNAMIC_ARCH support (comprising GENERIC_RISCV64 and the two + RVV 1.0 targets with vector length of 128 and 256) +- worked around the ZVL128B kernels for AXPBY mishandling the special + case of zero Y increment + +loongarch64: +- improved GEMM performance on servers of the 3C5000 generation +- improved performance and stability of DGEMM +- improved GEMV and TRSM kernels for LSX and LASX vector ABIs +- fixed CMAKE compilation with the INTERFACE64 option set +- fixed compilation with CMAKE +- worked around spurious errors flagged by the BLAS3 tests +- worked around a miscompilation of the POTRS utest by gcc 14.1 + +mips64: +- fixed ASUM and SUM kernels to accept negative step sizes in X +- fixed complex GEMV kernels for MSA + ==================================================================== Version 0.3.27 4-Apr-2024