From f5b9c398ad547b99ec38920898cf866b453613be Mon Sep 17 00:00:00 2001 From: nprasadm Date: Tue, 15 Feb 2022 12:50:22 +0530 Subject: [PATCH 1/7] Scalapack: Scalapack_test.sh script file minor correction and file permission modification Change-Id: Ica16f901ee17c49d8a29b5e87cf2546a0843c8e3 --- scalapack_test.sh | 149 +++++++++++++++++++++++----------------------- 1 file changed, 75 insertions(+), 74 deletions(-) mode change 100644 => 100755 scalapack_test.sh diff --git a/scalapack_test.sh b/scalapack_test.sh old mode 100644 new mode 100755 index 86460456..8b29076e --- a/scalapack_test.sh +++ b/scalapack_test.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash echo " " echo " Scalapack Testing Started " @@ -8,445 +9,445 @@ echo " " >> ../ScalaPack_TestResults.txt echo " xcbrd " >>../ScalaPack_TestResults.txt -mpirun xcbrd >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xcbrd >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xcdblu " >> ../ScalaPack_TestResults.txt -mpirun xcdblu >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xcdblu >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xcdtlu " >> ../ScalaPack_TestResults.txt -mpirun xcdtlu >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xcdtlu >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xcevc " >> ../ScalaPack_TestResults.txt -mpirun xcevc >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xcevc >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xcgblu " >> ../ScalaPack_TestResults.txt -mpirun xcgblu >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xcgblu >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xcgsep " >> ../ScalaPack_TestResults.txt -mpirun xcgsep >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xcgsep >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xcheevr " >> ../ScalaPack_TestResults.txt -mpirun xcheevr >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xcheevr >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xchrd " >> ../ScalaPack_TestResults.txt -mpirun xchrd >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xchrd >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xcinv " >> ../ScalaPack_TestResults.txt -mpirun xcinv >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xcinv >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xcllt " >> ../ScalaPack_TestResults.txt -mpirun xcllt >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xcllt >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xcls " >> ../ScalaPack_TestResults.txt -mpirun xcls >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xcls >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xclu " >> ../ScalaPack_TestResults.txt -mpirun xclu >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xclu >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xcnep " >> ../ScalaPack_TestResults.txt -mpirun xcnep >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xcnep >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xcpbllt " >> ../ScalaPack_TestResults.txt -mpirun xcpbllt >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xcpbllt >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xcptllt " >> ../ScalaPack_TestResults.txt -mpirun xcptllt >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xcptllt >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xcqr " >> ../ScalaPack_TestResults.txt -mpirun xcqr >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xcqr >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xcsep " >> ../ScalaPack_TestResults.txt -mpirun xcsep >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xcsep >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xctrd " >> ../ScalaPack_TestResults.txt -mpirun xctrd >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xctrd >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xdbrd " >> ../ScalaPack_TestResults.txt -mpirun xdbrd >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xdbrd >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xddblu " >> ../ScalaPack_TestResults.txt -mpirun xddblu >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xddblu >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xddtlu " >> ../ScalaPack_TestResults.txt -mpirun xddtlu >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xddtlu >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xdgblu " >> ../ScalaPack_TestResults.txt -mpirun xdgblu >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xdgblu >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xdgsep " >> ../ScalaPack_TestResults.txt -mpirun xdgsep >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xdgsep >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xdhrd " >> ../ScalaPack_TestResults.txt -mpirun xdhrd >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xdhrd >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xdhseqr " >> ../ScalaPack_TestResults.txt -mpirun xdhseqr >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xdhseqr >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xdinv " >> ../ScalaPack_TestResults.txt -mpirun xdinv >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xdinv >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xdllt " >> ../ScalaPack_TestResults.txt -mpirun xdllt >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xdllt >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xdls " >> ../ScalaPack_TestResults.txt -mpirun xdls >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xdls >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xdlu " >> ../ScalaPack_TestResults.txt -mpirun xdlu >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xdlu >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xdnep " >> ../ScalaPack_TestResults.txt -mpirun xdnep >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xdnep >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xdpbllt " >> ../ScalaPack_TestResults.txt -mpirun xdpbllt >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xdpbllt >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xdptllt " >> ../ScalaPack_TestResults.txt -mpirun xdptllt >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xdptllt >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xdqr " >> ../ScalaPack_TestResults.txt -mpirun xdqr >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xdqr >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xdsep " >> ../ScalaPack_TestResults.txt -mpirun xdsep >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xdsep >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xdsvd " >> ../ScalaPack_TestResults.txt -mpirun xdsvd >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xdsvd >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xdsyevr " >> ../ScalaPack_TestResults.txt -mpirun xdsyevr >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xdsyevr >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xdtrd " >> ../ScalaPack_TestResults.txt -mpirun xdtrd >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xdtrd >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xsbrd " >> ../ScalaPack_TestResults.txt -mpirun xsbrd >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xsbrd >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xsdblu " >> ../ScalaPack_TestResults.txt -mpirun xsdblu >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xsdblu >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xsdtlu " >> ../ScalaPack_TestResults.txt -mpirun xsdtlu >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xsdtlu >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xsgblu " >> ../ScalaPack_TestResults.txt -mpirun xsgblu >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xsgblu >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xsgsep " >> ../ScalaPack_TestResults.txt -mpirun xsgsep >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xsgsep >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xshrd " >> ../ScalaPack_TestResults.txt -mpirun xshrd >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xshrd >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xshseqr " >> ../ScalaPack_TestResults.txt -mpirun xshseqr >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xshseqr >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xsinv " >> ../ScalaPack_TestResults.txt -mpirun xsinv >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xsinv >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xsllt " >> ../ScalaPack_TestResults.txt -mpirun xsllt >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xsllt >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xsls " >> ../ScalaPack_TestResults.txt -mpirun xsls >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xsls >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xslu " >> ../ScalaPack_TestResults.txt -mpirun xslu >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xslu >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xsnep " >> ../ScalaPack_TestResults.txt -mpirun xsnep >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xsnep >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xspbllt " >> ../ScalaPack_TestResults.txt -mpirun xspbllt >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xspbllt >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xsptllt " >> ../ScalaPack_TestResults.txt -mpirun xsptllt >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xsptllt >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xsqr " >> ../ScalaPack_TestResults.txt -mpirun xsqr >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xsqr >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xssep " >> ../ScalaPack_TestResults.txt -mpirun xssep >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xssep >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xssvd " >> ../ScalaPack_TestResults.txt -mpirun xssvd >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xssvd >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xssyevr " >> ../ScalaPack_TestResults.txt -mpirun xssyevr >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xssyevr >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xstrd " >> ../ScalaPack_TestResults.txt -mpirun xstrd >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xstrd >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xzbrd " >> ../ScalaPack_TestResults.txt -mpirun xzbrd >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xzbrd >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xzdblu " >> ../ScalaPack_TestResults.txt -mpirun xzdblu >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xzdblu >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xzdtlu " >> ../ScalaPack_TestResults.txt -mpirun xzdtlu >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xzdtlu >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xzevc " >> ../ScalaPack_TestResults.txt -mpirun xzevc >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xzevc >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xzgblu " >> ../ScalaPack_TestResults.txt -mpirun xzgblu >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xzgblu >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xzgsep " >> ../ScalaPack_TestResults.txt -mpirun xzgsep >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xzgsep >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xzheevr " >> ../ScalaPack_TestResults.txt -mpirun xzheevr >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xzheevr >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xzhrd " >> ../ScalaPack_TestResults.txt -mpirun xzhrd >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xzhrd >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xzinv " >> ../ScalaPack_TestResults.txt -mpirun xzinv >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xzinv >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xzllt " >> ../ScalaPack_TestResults.txt -mpirun xzllt >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xzllt >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xzls " >> ../ScalaPack_TestResults.txt -mpirun xzls >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xzls >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xzlu " >> ../ScalaPack_TestResults.txt -mpirun xzlu >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xzlu >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xznep " >> ../ScalaPack_TestResults.txt -mpirun xznep >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xznep >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xzpbllt " >> ../ScalaPack_TestResults.txt -mpirun xzpbllt >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xzpbllt >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xzptllt " >> ../ScalaPack_TestResults.txt -mpirun xzptllt >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xzptllt >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xzqr " >> ../ScalaPack_TestResults.txt -mpirun xzqr >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xzqr >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xzsep " >> ../ScalaPack_TestResults.txt -mpirun xzsep >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xzsep >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt echo " xztrd " >> ../ScalaPack_TestResults.txt -mpirun xztrd >> ../ScalaPack_TestResults.txt +mpirun --bind-to core --nooversubscribe xztrd >> ../ScalaPack_TestResults.txt sleep 3 echo "---------------------------------------------------------" >> ../ScalaPack_TestResults.txt echo " " >> ../ScalaPack_TestResults.txt From 88336842d6cad47b1086348597a316b121b4c568 Mon Sep 17 00:00:00 2001 From: nprasadm Date: Mon, 21 Mar 2022 15:41:33 +0530 Subject: [PATCH 2/7] SCALAPACK: License file updated for 2022 Change-Id: Ia0c1ee64f987f2bcdec418c83cabe30ac7585519 --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index c09a8ad5..e5aa1c83 100644 --- a/LICENSE +++ b/LICENSE @@ -5,7 +5,7 @@ Copyright (c) 2000-2011 The University of California Berkeley. All rights reserved. Copyright (c) 2006-2011 The University of Colorado Denver. All rights reserved. -Copyright (C) 2020-2021 Advanced Micro Devices, Inc. All rights +Copyright (C) 2020-2022 Advanced Micro Devices, Inc. All rights reserved. $COPYRIGHT$ From d3000133a46f2711dd3f86eebd5968a526591813 Mon Sep 17 00:00:00 2001 From: nprasadm Date: Wed, 22 Dec 2021 20:54:42 +0530 Subject: [PATCH 3/7] SCALAPACK: Decoupling option for BLACS sources from scalapack build added. To build with external BLACS library, provide the additional option '-DBLACS_LIBRARY=/libBLACS.so' in the cmake configure command. Signed-off-by: Nagendra AMD-Internal: [CPUPL-1971] Change-Id: I0001a17a190ad0a34095d525b7b472a75cdb0d5b --- CMakeLists.txt | 89 ++++++++++++++++++++++++++---------- PBLAS/TESTING/CMakeLists.txt | 28 ++++++------ 2 files changed, 78 insertions(+), 39 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a857bddb..24c03c9f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -167,13 +167,49 @@ ENDIF(ENABLE_ILP64) ADD_DEFINITIONS( "-D${CDEFS}" "${f2cflag}") +# -------------------------------------------------- +# By default static library +OPTION(BUILD_SHARED_LIBS "Build shared libraries" OFF ) +OPTION(BUILD_STATIC_LIBS "Build static libraries" ON ) + +# -------------------------------------------------- +# Subdirectories that need to be processed + +macro(append_subdir_files variable dirname) +get_directory_property(holder DIRECTORY ${dirname} DEFINITION ${variable}) +foreach(depfile ${holder}) + list(APPEND ${variable} "${dirname}/${depfile}") +endforeach() +endmacro() + + # -------------------------------------------------- # ScaLAPACK needs BLAS and LAPACK option(USE_OPTIMIZED_LAPACK_BLAS "Whether or not to search for optimized LAPACK and BLAS libraries on your machine (if not found, Reference LAPACK and BLAS will be downloaded and installed)" OFF) -message(STATUS "CHECKING BLAS AND LAPACK LIBRARIES") +IF(BLACS_LIBRARY) + message(STATUS "CHECKING & VERIFYING BLACS LIBRARY") + include(CheckFunctionExists) + set(CMAKE_REQUIRED_LIBRARIES ${BLACS_LIBRARY}) + message(STATUS "--> BLACS supplied by user is ${BLACS_LIBRARY}.") + CHECK_FUNCTION_EXISTS("Cblacs_get" BLACS_FOUND) + unset( CMAKE_REQUIRED_LIBRARIES ) + message(STATUS "--> BLACS routine blacs_pinfo is found: ${BLACS_FOUND}.") +ENDIF() +if(BLACS_FOUND) + message(STATUS "--> BLACS supplied by user is WORKING, will use ${BLACS_LIBRARY}.") +else( BLACS_FOUND ) +# +# BLACS +# + add_subdirectory(BLACS) + append_subdir_files(blacs "BLACS/SRC") + message(STATUS "--> BLACS supplied by user is NOT WORKING, will use BLACS source code for building aocl-scalapack") +endif( BLACS_FOUND ) + +message(STATUS "CHECKING BLAS AND LAPACK LIBRARIES") IF(LAPACK_LIBRARIES) include(CheckFortranFunctionExists) message(STATUS "--> LAPACK supplied by user is ${LAPACK_LIBRARIES}.") @@ -218,32 +254,11 @@ message(STATUS "LAPACK library: ${LAPACK_LIBRARIES}") MESSAGE(STATUS "=========") -# -------------------------------------------------- -# By default static library -OPTION(BUILD_SHARED_LIBS "Build shared libraries" OFF ) -OPTION(BUILD_STATIC_LIBS "Build static libraries" ON ) - -# -------------------------------------------------- -# Subdirectories that need to be processed - -macro(append_subdir_files variable dirname) -get_directory_property(holder DIRECTORY ${dirname} DEFINITION ${variable}) -foreach(depfile ${holder}) - list(APPEND ${variable} "${dirname}/${depfile}") -endforeach() -endmacro() - # # AOCL_DTL # #add_subdirectory(AOCL_DTL) -# -# BLACS -# -add_subdirectory(BLACS) -append_subdir_files(blacs "BLACS/SRC") - # # TOOLS # @@ -285,19 +300,42 @@ if(WIN32 AND BUILD_SHARED_LIBS) endif () if (UNIX) - add_library(scalapack ${blacs} ${tools} ${tools-C} ${extra_lapack} ${pblas} ${pblas-F} ${ptzblas} ${ptools} ${pbblas} ${redist} ${src} ${src-C}) - target_link_libraries( scalapack ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) - scalapack_install_library(scalapack) + if(BLACS_FOUND) + add_library(scalapack ${tools} ${tools-C} ${extra_lapack} ${pblas} ${pblas-F} ${ptzblas} ${ptools} ${pbblas} ${redist} ${src} ${src-C}) + target_link_libraries( scalapack ${BLACS_LIBRARY} ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) + scalapack_install_library(scalapack) + else(BLACS_FOUND) + add_library(scalapack ${blacs} ${tools} ${tools-C} ${extra_lapack} ${pblas} ${pblas-F} ${ptzblas} ${ptools} ${pbblas} ${redist} ${src} ${src-C}) + target_link_libraries( scalapack ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) + scalapack_install_library(scalapack) + endif(BLACS_FOUND) else (UNIX) # Need to separate Fortran and C Code if (CMAKE_C_COMPILER_ID MATCHES MSVC) # create Fortran objects and add to scalapack library first + if(BLACS_FOUND) + add_library(scalapack-F OBJECT ${pblas-F} ${pbblas} ${ptzblas} ${tools} ${src} ${extra_lapack} ) + add_library(scalapack $ ${tools-C} ${pblas} ${ptools} ${redist} ${src-C}) + target_link_libraries( scalapack ${BLACS_LIBRARY} ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} ${MPI_LIBRARY}) + target_link_directories( scalapack PUBLIC ${CMAKE_IFORT_LIBDEPS_DIR}) + scalapack_install_library(scalapack) + else(BLACS_FOUND) add_library(scalapack-F OBJECT ${pblas-F} ${pbblas} ${ptzblas} ${tools} ${src} ${extra_lapack} ) add_library(scalapack $ ${blacs} ${tools-C} ${pblas} ${ptools} ${redist} ${src-C}) target_link_libraries( scalapack ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} ${MPI_LIBRARY}) target_link_directories( scalapack PUBLIC ${CMAKE_IFORT_LIBDEPS_DIR}) scalapack_install_library(scalapack) + endif(BLACS_FOUND) else (CMAKE_C_COMPILER_ID MATCHES Clang) # create C objects and add to scalapack library first + if(BLACS_FOUND) + add_library(scalapack-C OBJECT ${tools-C} ${pblas} ${ptools} ${redist} ${src-C}) + target_link_libraries( scalapack-C ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} ${MPI_LIBRARY}) + target_link_directories( scalapack-C PUBLIC ${CMAKE_IFORT_LIBDEPS_DIR}) + add_library(scalapack $ ${pblas-F} ${pbblas} ${ptzblas} ${tools} ${src} ${extra_lapack} ) + target_link_libraries( scalapack ${BLACS_LIBRARY} ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} ${MPI_LIBRARY}) + target_link_directories( scalapack PUBLIC ${CMAKE_IFORT_LIBDEPS_DIR}) + scalapack_install_library(scalapack) + else(BLACS_FOUND) add_library(scalapack-C OBJECT ${blacs} ${tools-C} ${pblas} ${ptools} ${redist} ${src-C}) target_link_libraries( scalapack-C ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} ${MPI_LIBRARY}) target_link_directories( scalapack-C PUBLIC ${CMAKE_IFORT_LIBDEPS_DIR}) @@ -305,6 +343,7 @@ else (UNIX) # Need to separate Fortran and C Code target_link_libraries( scalapack ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} ${MPI_LIBRARY}) target_link_directories( scalapack PUBLIC ${CMAKE_IFORT_LIBDEPS_DIR}) scalapack_install_library(scalapack) + endif(BLACS_FOUND) endif () endif (UNIX) add_subdirectory(TESTING) diff --git a/PBLAS/TESTING/CMakeLists.txt b/PBLAS/TESTING/CMakeLists.txt index caf2867f..a592b0d2 100644 --- a/PBLAS/TESTING/CMakeLists.txt +++ b/PBLAS/TESTING/CMakeLists.txt @@ -60,20 +60,20 @@ add_executable(dpb3tst pdblas3tst.f ${dpbtcom}) add_executable(cpb3tst pcblas3tst.f ${cpbtcom}) add_executable(zpb3tst pzblas3tst.f ${zpbtcom}) -target_link_libraries(spb1tst scalapack ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) -target_link_libraries(dpb1tst scalapack ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) -target_link_libraries(cpb1tst scalapack ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) -target_link_libraries(zpb1tst scalapack ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) - -target_link_libraries(spb2tst scalapack ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) -target_link_libraries(dpb2tst scalapack ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) -target_link_libraries(cpb2tst scalapack ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) -target_link_libraries(zpb2tst scalapack ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) - -target_link_libraries(spb3tst scalapack ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) -target_link_libraries(dpb3tst scalapack ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) -target_link_libraries(cpb3tst scalapack ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) -target_link_libraries(zpb3tst scalapack ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) +target_link_libraries(spb1tst scalapack ${BLACS_LIBRARIES} ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) +target_link_libraries(dpb1tst scalapack ${BLACS_LIBRARIES} ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) +target_link_libraries(cpb1tst scalapack ${BLACS_LIBRARIES} ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) +target_link_libraries(zpb1tst scalapack ${BLACS_LIBRARIES} ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) + +target_link_libraries(spb2tst scalapack ${BLACS_LIBRARIES} ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) +target_link_libraries(dpb2tst scalapack ${BLACS_LIBRARIES} ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) +target_link_libraries(cpb2tst scalapack ${BLACS_LIBRARIES} ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) +target_link_libraries(zpb2tst scalapack ${BLACS_LIBRARIES} ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) + +target_link_libraries(spb3tst scalapack ${BLACS_LIBRARIES} ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) +target_link_libraries(dpb3tst scalapack ${BLACS_LIBRARIES} ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) +target_link_libraries(cpb3tst scalapack ${BLACS_LIBRARIES} ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) +target_link_libraries(zpb3tst scalapack ${BLACS_LIBRARIES} ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES}) add_test(spb1tst ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ./spb1tst) add_test(dpb1tst ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ./dpb1tst) From ba85f208de16e7cf09be2de7d9aab1192ee48953 Mon Sep 17 00:00:00 2001 From: "nprasadm@amd.com" Date: Tue, 19 Apr 2022 12:39:41 +0530 Subject: [PATCH 4/7] Scalapack version API updated Change-Id: I2cae621ff9a12ef015aba82f6b9cad1fd4b29f8c --- SRC/get_aocl_scalapack_version.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SRC/get_aocl_scalapack_version.c b/SRC/get_aocl_scalapack_version.c index 639dacc5..5252cb69 100644 --- a/SRC/get_aocl_scalapack_version.c +++ b/SRC/get_aocl_scalapack_version.c @@ -3,7 +3,7 @@ /* --------------------------------------------------------------------- * * -- AOCL ScaLAPACK routine -- -* Copyright (c) 2020-2021 Advanced Micro Devices, Inc.  All rights reserved. +* Copyright (c) 2020-2022 Advanced Micro Devices, Inc.  All rights reserved. * * --------------------------------------------------------------------- */ @@ -22,9 +22,9 @@ void get_aocl_scalapack_version_( version ) #endif { /* -* version = "AOCL 3.1 ScaLAPACK. Supports Netlib ScaLAPACK 2.1.0" +* version = "AOCL-ScaLAPACK 3.1.2, Supports Netlib ScaLAPACK 2.1.0" */ - strcpy(version, "AOCL ScaLAPACK 3.1, supports ScaLAPACK 2.1.0"); + strcpy(version, "AOCL-ScaLAPACK 3.1.2, supports ScaLAPACK 2.1.0"); return; } From 400e90f66358e5baede605bcbcb14760a21c72a2 Mon Sep 17 00:00:00 2001 From: nprasadm Date: Tue, 26 Apr 2022 11:14:41 +0530 Subject: [PATCH 5/7] SCALAPACK: AOCL scalapack progress implementation in PDGETRF API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit -- AOCL libraries are used for lengthy computations which can go on for hours or days, once the operation is started, the user doesn’t get any update on current state of the computation. This (AOCL progress) feature enables user to receive a periodic update from the libraries. -- User can define AOCL_scalapack_Progress in application and register a callback with the library if it is interested in receiving the periodic update. -- The library invokes this callback periodically with information about current state of the operation. -- The update frequency is statically set in the code, it can be modified as needed if the library is built from source. Sample test application showing usage of AOCL Progress feature with PDGETRF API is included. AMD-Internal: [CPUPL-2113] Change-Id: I58daea17c1318ab710cd19437af7c647d04389a1 --- CMakeLists.txt | 5 + EXAMPLE/aocl_progress_example/Readme.txt | 14 ++ .../pdgerf_example_app.c | 132 ++++++++++++++++++ SRC/CMakeLists.txt | 2 +- SRC/aocl_scalapack_progress.c | 33 +++++ SRC/aocl_scalapack_progress.h | 41 ++++++ SRC/pdgetrf0.f | 21 ++- SRC/pxsyevx.h | 4 +- 8 files changed, 248 insertions(+), 4 deletions(-) create mode 100644 EXAMPLE/aocl_progress_example/Readme.txt create mode 100644 EXAMPLE/aocl_progress_example/pdgerf_example_app.c create mode 100644 SRC/aocl_scalapack_progress.c create mode 100644 SRC/aocl_scalapack_progress.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 24c03c9f..7650ff5c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -165,6 +165,11 @@ else(ENABLE_ILP64) add_definitions("-DUInt=unsigned int ") ENDIF(ENABLE_ILP64) +if(ENABLE_AOCL_PROGRESS) + add_definitions("-DAOCL_PROGRESS") +ENDIF(ENABLE_AOCL_PROGRESS) + + ADD_DEFINITIONS( "-D${CDEFS}" "${f2cflag}") # -------------------------------------------------- diff --git a/EXAMPLE/aocl_progress_example/Readme.txt b/EXAMPLE/aocl_progress_example/Readme.txt new file mode 100644 index 00000000..51e53836 --- /dev/null +++ b/EXAMPLE/aocl_progress_example/Readme.txt @@ -0,0 +1,14 @@ +Procedure to build and run the sample application with aocl progress feature: +============================================================================= + +1) copy below AOCL libraries to the "/EXAMPLE/aocl_progress_example/" folder: + 1) libscalapack.a + 2) libflame.a + 3) blis-mt.a + +2) Run the below command to build the 'pdgetrf' test application with 'aocl-progress' feature. + mpicc -O0 -g pdgerf_example_app.c libscalapack.a libflame.a -fopenmp libblis-mt.a -lm -lgfortran -o test + +3) Run the below commands to run the application: + Ex: mpirun -np 4 ./test 32 8 2 2 + mpirun -np 8 ./test 1024 32 4 2 diff --git a/EXAMPLE/aocl_progress_example/pdgerf_example_app.c b/EXAMPLE/aocl_progress_example/pdgerf_example_app.c new file mode 100644 index 00000000..63c6db1d --- /dev/null +++ b/EXAMPLE/aocl_progress_example/pdgerf_example_app.c @@ -0,0 +1,132 @@ +#include +#include +#include +#include +#include +#include +#include "mpi.h" + +void blacs_get_(int*, int*, int*); +void blacs_pinfo_(int*, int*); +void blacs_gridinit_(int*, char*, int*, int*); +void blacs_gridinfo_(int*, int*, int*, int*, int*); +void descinit_(int*, int*, int*, int*, int*, int*, int*, int*, int*, int*); +void pdgetrf_(int*, int*, double*, int*, int*, int*, int*, int*); +void blacs_gridexit_(int*); +int numroc_(int*, int*, int*, int*, int*); + +int AOCL_progress(char* api, int *lenapi, int *progress, int *mpi_rank, int *total_mpi_processes); + +int AOCL_progress(char* api, int *lenapi, int *progress, int *mpi_rank, int *total_mpi_processes) +{ + printf( "In AOCL Progress MPI Rank: %i API: %s progress: %i MPI processes: %i\n", *mpi_rank, api, *progress,*total_mpi_processes ); + return 0; +} + + +int main(int argc, char **argv) { + int izero=0; + int ione=1; + int myrank_mpi, nprocs_mpi; + MPI_Init( &argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &myrank_mpi); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs_mpi); + + int n = 1000; // (Global) Matrix size + int nprow = 2; // Number of row procs + int npcol = 2; // Number of column procs + int nb = 256; // (Global) Block size + char uplo='L'; // Matrix is lower triangular + char layout='R'; // Block cyclic, Row major processor mapping + + printf("Usage: ./test matrix_size block_size nprocs_row nprocs_col\n"); + + if(argc > 1) { + n = atoi(argv[1]); + } + if(argc > 2) { + nb = atoi(argv[2]); + } + if(argc > 3) { + nprow = atoi(argv[3]); + } + if(argc > 4) { + npcol = atoi(argv[4]); + } + + assert(nprow * npcol == nprocs_mpi); + + // Initialize BLACS + int iam, nprocs; + int zero = 0; + int ictxt, myrow, mycol; + blacs_pinfo_(&iam, &nprocs) ; // BLACS rank and world size + blacs_get_(&zero, &zero, &ictxt ); // -> Create context + blacs_gridinit_(&ictxt, &layout, &nprow, &npcol ); // Context -> Initialize the grid + blacs_gridinfo_(&ictxt, &nprow, &npcol, &myrow, &mycol ); // Context -> Context grid info (# procs row/col, current procs row/col) + + // Compute the size of the local matrices + int mpA = numroc_( &n, &nb, &myrow, &izero, &nprow ); // My proc -> row of local A + int nqA = numroc_( &n, &nb, &mycol, &izero, &npcol ); // My proc -> col of local A + + printf(" Proc %d/%d for MPI, proc %d/%d for BLACS in position (%d,%d)/(%d,%d) with local matrix %dx%d, global matrix %d, block size %d\n",myrank_mpi,nprocs_mpi,iam,nprocs,myrow,mycol,nprow,npcol,mpA,nqA,n,nb); + + // Allocate and fill the matrices A and B + // A[I,J] = (I == J ? 5*n : I+J) + double *A; + int *IPPIV; + A = (double *)calloc(mpA*nqA,sizeof(double)) ; + if (A==NULL){ printf("Error of memory allocation A on proc %dx%d\n",myrow,mycol); exit(0); } + + IPPIV = (int *)calloc(2*n,sizeof(int)) ; + if (IPPIV==NULL){ printf("Error of memory allocation IPPIV %d\n",2*n); exit(0); } + + int k = 0; + for (int j = 0; j < nqA; j++) { // local col + int l_j = j / nb; // which block + int x_j = j % nb; // where within that block + int J = (l_j * npcol + mycol) * nb + x_j; // global col + for (int i = 0; i < mpA; i++) { // local row + int l_i = i / nb; // which block + int x_i = i % nb; // where within that block + int I = (l_i * nprow + myrow) * nb + x_i; // global row + assert(I < n); + assert(J < n); + if(I == J) { + A[k] = n*n; + } else { + A[k] = I+J; + } + //printf("%d %d -> %d %d -> %f\n", i, j, I, J, A[k]); + k++; + } + } + + // Create descriptor + int descA[9]; + int info; + int lddA = mpA > 1 ? mpA : 1; + descinit_( descA, &n, &n, &nb, &nb, &izero, &izero, &ictxt, &lddA, &info); + if(info != 0) { + printf("Error in descinit, info = %d\n", info); + } + + // Run pdgetrf and time + double MPIt1 = MPI_Wtime(); + printf("[%dx%d] Starting pdgetrf\n", myrow, mycol); + aocl_scalapack_set_progress(&AOCL_progress); + pdgetrf_( &n, &n, A, &ione, &ione, descA, IPPIV, &info ); + + if (info != 0) { + printf("Error in pdgetrf, info = %d\n", info); + } + + double MPIt2 = MPI_Wtime(); + printf("[%dx%d] Done, time %e s.\n", myrow, mycol, MPIt2 - MPIt1); + free(A); + + // Exit and finalize + blacs_gridexit_(&ictxt); + MPI_Finalize(); + return 0; +} diff --git a/SRC/CMakeLists.txt b/SRC/CMakeLists.txt index ef99a395..7f162c8a 100644 --- a/SRC/CMakeLists.txt +++ b/SRC/CMakeLists.txt @@ -3,7 +3,7 @@ set (ALLAUX pjlaenv.f pilaenvx.f piparmq.f pilaver.f pmpim2.f pmpcol.f) set (ALLAUX-C pbchkvect.c getpbbuf.c pcrot.c pslaiect.c pdlaiect.c pzrot.c - slamov.c clamov.c dlamov.c zlamov.c + slamov.c clamov.c dlamov.c zlamov.c aocl_scalapack_progress.c get_aocl_scalapack_version.c aocl_dtl_trace_entry.c aocl_dtl_trace_exit.c) set (SCLAUX diff --git a/SRC/aocl_scalapack_progress.c b/SRC/aocl_scalapack_progress.c new file mode 100644 index 00000000..96748380 --- /dev/null +++ b/SRC/aocl_scalapack_progress.c @@ -0,0 +1,33 @@ +/* --------------------------------------------------------------------- + * * + * * -- AOCL ScaLAPACK progress support routine/s -- + * * Copyright (c) 2022 Advanced Micro Devices, Inc.  All rights reserved. + * * + * * --------------------------------------------------------------------- + * */ + +#include "aocl_scalapack_progress.h" + +aocl_scalapack_progress_callback aocl_scalapack_progress_ptr_ = NULL; +#define aocl_scalapack_set_progress aocl_scalapack_set_progress_ + +#ifdef __STDC__ +void aocl_scalapack_set_progress( aocl_scalapack_progress_callback func ) +#else + void aocl_scalapack_set_progress( func ) + aocl_scalapack_progress_callback func; +#endif +{ + aocl_scalapack_progress_ptr_ = func; +} + +void aocl_scalapack_progress_(char* api, integer *lenapi, integer* progress, + integer* current_process, integer *total_processes) +{ + integer ret; + if (aocl_scalapack_progress_ptr_ != NULL ) { + ret = aocl_scalapack_progress_ptr_ ( api, lenapi, progress, current_process, total_processes); + } + + return; +} diff --git a/SRC/aocl_scalapack_progress.h b/SRC/aocl_scalapack_progress.h new file mode 100644 index 00000000..0f655945 --- /dev/null +++ b/SRC/aocl_scalapack_progress.h @@ -0,0 +1,41 @@ +/* --------------------------------------------------------------------- + * * + * * -- AOCL ScaLAPACK progress support routine/s -- + * * Copyright (c) 2022 Advanced Micro Devices, Inc.  All rights reserved. + * * + * * --------------------------------------------------------------------- + * */ +#ifndef _AOCL_SCALAPACK_PROGRESS_ +#define _AOCL_SCALAPACK_PROGRESS_ + +#include +#include +/** Declarations **/ + +#define aocl_scalapack_set_progress_ aocl_scalapack_set_progress + +#if defined(ENABLE_ILP64) +typedef int64_t integer; +typedef uint64_t uinteger; +#else +typedef int integer; +typedef unsigned long uinteger; +#endif + +typedef integer ( *aocl_scalapack_progress_callback )( +char *api, +integer *lenapi, +integer *progress, +integer *current_process, +integer *total_processes +); + + +integer aocl_scalapack_progress ( char* api, integer *lenapi, integer* progress, + integer* current_process, integer *total_processes ); + +aocl_scalapack_progress_callback aocl_scalapack_progress_ptr_; + +void aocl_scalapack_set_progress( aocl_scalapack_progress_callback func ); + +#endif // _AOCL_SCALAPACK_PROGRESS_ diff --git a/SRC/pdgetrf0.f b/SRC/pdgetrf0.f index 1fa80999..e92e4dd4 100644 --- a/SRC/pdgetrf0.f +++ b/SRC/pdgetrf0.f @@ -1,5 +1,5 @@ * -- ScaLAPACK routine -- -* Copyright (c) 2020-21 Advanced Micro Devices, Inc.  All rights reserved. +* Copyright (c) 2020-22 Advanced Micro Devices, Inc.  All rights reserved. * June 10, 2020 * * ===================================================================== @@ -147,6 +147,11 @@ SUBROUTINE PDGETRF0( M, N, A, IA, JA, DESCA, IPIV, INFO ) CHARACTER COLBTOP, COLCTOP, ROWBTOP INTEGER I, ICOFF, ICTXT, IINFO, IN, IROFF, J, JB, JN, $ MN, MYCOL, MYROW, NPCOL, NPROW +* +#ifdef AOCL_PROGRESS + INTEGER TOTAL_MPI_PROCESSES, STEP, LSTAGE, CURRENT_RANK + CHARACTER*7 API_NAME +#endif * .. * .. Local Arrays .. INTEGER IDUM1( 1 ), IDUM2( 1 ) @@ -159,6 +164,8 @@ SUBROUTINE PDGETRF0( M, N, A, IA, JA, DESCA, IPIV, INFO ) * .. External Functions .. INTEGER ICEIL EXTERNAL ICEIL +* +* * .. * .. Intrinsic Functions .. INTRINSIC MIN, MOD @@ -222,6 +229,14 @@ SUBROUTINE PDGETRF0( M, N, A, IA, JA, DESCA, IPIV, INFO ) JN = MIN( ICEIL( JA, DESCA( NB_ ) )*DESCA( NB_ ), JA+MN-1 ) JB = JN - JA + 1 * +#ifdef AOCL_PROGRESS + CURRENT_RANK = MYCOL+MYROW*NPCOL + TOTAL_MPI_PROCESSES = NPROW*NPCOL + STEP = JB + LSTAGE = 7 + API_NAME = 'PDGETRF' +#endif + * Factor diagonal and subdiagonal blocks and test for exact * singularity. * @@ -255,6 +270,10 @@ SUBROUTINE PDGETRF0( M, N, A, IA, JA, DESCA, IPIV, INFO ) DO 10 J = JN+1, JA+MN-1, DESCA( NB_ ) JB = MIN( MN-J+JA, DESCA( NB_ ) ) I = IA + J - JA +#ifdef AOCL_PROGRESS + CALL AOCL_SCALAPACK_PROGRESS ( API_NAME, LSTAGE, + $ J, CURRENT_RANK, TOTAL_MPI_PROCESSES ) +#endif * * Factor diagonal and subdiagonal blocks and test for exact * singularity. diff --git a/SRC/pxsyevx.h b/SRC/pxsyevx.h index a32ee29e..73d7c89f 100644 --- a/SRC/pxsyevx.h +++ b/SRC/pxsyevx.h @@ -63,7 +63,7 @@ #define pslaiect_ PSLAIECT #define get_aocl_scalapack_version_ GET_AOCL_SCALAPACK_VERSION - +#define aocl_scalapack_progress_ AOCL_SCALAPACK_PROGRESS #endif #if (F77_CALL_C == NOCHANGE) @@ -85,5 +85,5 @@ #define pslaiect_ pslaiect #define get_aocl_scalapack_version_ get_aocl_scalapack_version - +#define aocl_scalapack_progress_ aocl_scalapack_progress #endif From e1cfef01cb1783aa45c0401863254db6d099e7af Mon Sep 17 00:00:00 2001 From: nprasadm Date: Mon, 2 May 2022 10:49:37 +0530 Subject: [PATCH 6/7] Scalapack: Fix for ILP64 mode build error added Change-Id: Ic21b5fbd90319e31ffdc9d6d858149c81f84015f --- SRC/aocl_scalapack_progress.h | 1 + 1 file changed, 1 insertion(+) diff --git a/SRC/aocl_scalapack_progress.h b/SRC/aocl_scalapack_progress.h index 0f655945..3e4bec49 100644 --- a/SRC/aocl_scalapack_progress.h +++ b/SRC/aocl_scalapack_progress.h @@ -9,6 +9,7 @@ #define _AOCL_SCALAPACK_PROGRESS_ #include +#include #include /** Declarations **/ From b56bb661308a2d5aecd9892b5f051beddc77d028 Mon Sep 17 00:00:00 2001 From: nprasadm Date: Wed, 25 May 2022 17:07:21 +0530 Subject: [PATCH 7/7] SCALAPACK: AOCL-3.2 release version and documents updation 1. Scalapack version API updated for 3.2 2. AOCL-Progress README document updated in /EXAMPLE/aocl_progress_example/README.txt. 3. README_ScaLAPACK_AMD file added in the '/' folder. Change-Id: I199c3920418b20dd75354b15a994d0771719ca7f --- EXAMPLE/aocl_progress_example/README.txt | 60 ++++++++++++++++++++++++ EXAMPLE/aocl_progress_example/Readme.txt | 14 ------ README.md | 1 + README_ScaLAPACK_AMD | 48 +++++++++++++++++++ SRC/get_aocl_scalapack_version.c | 4 +- 5 files changed, 111 insertions(+), 16 deletions(-) create mode 100644 EXAMPLE/aocl_progress_example/README.txt delete mode 100644 EXAMPLE/aocl_progress_example/Readme.txt create mode 120000 README.md create mode 100644 README_ScaLAPACK_AMD diff --git a/EXAMPLE/aocl_progress_example/README.txt b/EXAMPLE/aocl_progress_example/README.txt new file mode 100644 index 00000000..3a3d83a4 --- /dev/null +++ b/EXAMPLE/aocl_progress_example/README.txt @@ -0,0 +1,60 @@ +Checking AOCL-ScaLAPACK Operation Progress + +AOCL libraries may be used to perform lengthy computations (for example, matrix multiplications, solver involving large matrices). These operations/computations may go on for hours. + +AOCL progress feature provides mechanism for the application to check how far the computations have progressed. Selected set of APIs of AOCL libraries periodically updates the application with progress made so far via a callback function. + +Usage: + +The Application needs to define a callback function in specific format and register this callback function with the AOCL-ScaLAPACK library. + +The callback function prototype must be as defined below. +int aocl_scalapack_progress( +char* api, +integer lenapi, +integer *progress, +integer *mpi_rank, +integer *total_mpi_processes +) + + +The table below explains various parameters +Parameters | Purpose +--------------------------------------------------------------------- +api | Name of the API which is currently running +lenapi | Length of API name character buffer +progress | Linear progress made in current thread so far +mpi_rank | Current process rank +total_mpi_processes | Total number of processes used to perform the operation + +Callback Registration: + +The callback function must be registered with library for it to report the progress. + +aocl_scalapack_set_progress(aocl_scalapack_progress); + +Example: +int aocl_scalapack_progress(char* api, int *lenapi, int *progress, int *mpi_rank, int *total_mpi_processes) +{ + printf( "In AOCL Progress MPI Rank: %i API: %s progress: %i MPI processes: %i\n", *mpi_rank, api, *progress,*total_mpi_processes ); + return 0; +} + +Limitations +- AOCL-ScALAPACK Progress feature is currently supported only on Linux + + +Procedure to build and run the sample application with aocl progress feature: +============================================================================= + +1) copy below AOCL libraries to the "/EXAMPLE/aocl_progress_example/" folder: + 1) libscalapack.a + 2) libflame.a + 3) blis-mt.a + +2) Run the below command to build the 'pdgetrf' test application with 'aocl-progress' feature. + mpicc -O0 -g pdgerf_example_app.c libscalapack.a libflame.a -fopenmp libblis-mt.a -lm -lgfortran -o test + +3) Run the below commands to run the application: + Ex: mpirun -np 4 ./test 32 8 2 2 + mpirun -np 8 ./test 1024 32 4 2 diff --git a/EXAMPLE/aocl_progress_example/Readme.txt b/EXAMPLE/aocl_progress_example/Readme.txt deleted file mode 100644 index 51e53836..00000000 --- a/EXAMPLE/aocl_progress_example/Readme.txt +++ /dev/null @@ -1,14 +0,0 @@ -Procedure to build and run the sample application with aocl progress feature: -============================================================================= - -1) copy below AOCL libraries to the "/EXAMPLE/aocl_progress_example/" folder: - 1) libscalapack.a - 2) libflame.a - 3) blis-mt.a - -2) Run the below command to build the 'pdgetrf' test application with 'aocl-progress' feature. - mpicc -O0 -g pdgerf_example_app.c libscalapack.a libflame.a -fopenmp libblis-mt.a -lm -lgfortran -o test - -3) Run the below commands to run the application: - Ex: mpirun -np 4 ./test 32 8 2 2 - mpirun -np 8 ./test 1024 32 4 2 diff --git a/README.md b/README.md new file mode 120000 index 00000000..7f0d841b --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +README_ScaLAPACK_AMD \ No newline at end of file diff --git a/README_ScaLAPACK_AMD b/README_ScaLAPACK_AMD new file mode 100644 index 00000000..5fca3ca3 --- /dev/null +++ b/README_ScaLAPACK_AMD @@ -0,0 +1,48 @@ + +AOCL-ScaLAPACK +=============================================================================== +ScaLAPACK, or Scalable LAPACK, is a library of high performance linear algebra +routines for distributed memory computers supporting MPI. + +AOCL-ScaLAPACK is the optimized version of ScaLAPACK for AMD EPYC family of +processors. + +=============================================================================== + +1. Install MPI library and set the PATH and LD_LIBRARY_PATH environment + variables to point to installed binaries. + eg. export PATH=/bin:$PATH + eg. export LD_LIBRARY_PATH=/lib:$LD_LIBRARY_PATH + +2. Download AMD optimized versions of BLIS and libFLAME from following link + https://developer.amd.com/amd-aocl/ + +3. Install latest CMAKE tool. + +4. Install AOCL-BLIS and AOCL-libFLAME libraries either using pre-built binaries or build + from source. + To build AOCL-BLIS and AOCL-libFLAME from source, clone from following github links + BLIS: https://github.com/amd/blis + libFLAME: https://github.com/amd/libflame + +5. Steps to build the AOCL-ScaLAPACK library and the test suite: + + a. Create a new directory. For example, build: + $ mkdir build + $ cd build + + b. Set PATH and LD_LIBRARY_PATH appropriately to the MPI installation. + + c. To Build the AOCL-ScaLAPACK library and the test suite, Run the below commands: + $ cmake .. -DBUILD_SHARED_LIBS=OFF -DBLAS_LIBRARIES="-fopenmp /libblis-mt.a" + -DLAPACK_LIBRARIES="/libflame.a" + -DCMAKE_C_COMPILER=mpicc -DCMAKE_Fortran_COMPILER=mpif90 + -DUSE_OPTIMIZED_LAPACK_BLAS=OFF [-D DENABLE_ILP64=ON] + $ make -j + + This command generates the AOCL-ScaLAPACK library in the 'build/lib' folder and test applications in the 'build/TESTING' folder. + +4. To Run the AOCL-ScaLAPACK test suite, Run the below script in the 'build/' folder: + cp -f ../scalapack_test.sh . + ./scalapack_test.sh + Test logs will be generated in 'ScalaPack_TestResults.txt'. diff --git a/SRC/get_aocl_scalapack_version.c b/SRC/get_aocl_scalapack_version.c index 5252cb69..9e01a769 100644 --- a/SRC/get_aocl_scalapack_version.c +++ b/SRC/get_aocl_scalapack_version.c @@ -22,9 +22,9 @@ void get_aocl_scalapack_version_( version ) #endif { /* -* version = "AOCL-ScaLAPACK 3.1.2, Supports Netlib ScaLAPACK 2.1.0" +* version = "AOCL-ScaLAPACK 3.2, Supports Netlib ScaLAPACK 2.1.0" */ - strcpy(version, "AOCL-ScaLAPACK 3.1.2, supports ScaLAPACK 2.1.0"); + strcpy(version, "AOCL-ScaLAPACK 3.2, supports ScaLAPACK 2.1.0"); return; }