From c0394af4d84ebb84b7d2b95283ad65ffd84e0d45 Mon Sep 17 00:00:00 2001 From: Nai-Yuan Chiang Date: Thu, 6 May 2021 12:07:09 -0400 Subject: [PATCH] Fix row_max_abs_value() in RAJA implementation (#225) * fix function row_max_abs_value * use latest spack env on ascent * add user option to control if MDS_ex4 has empty sparse row in constraint Jacobian * add new unit test according to MDS_ex4 with empty sparse row in constraint Jacobian Co-authored-by: Nai-Yuan Chiang Co-authored-by: Chiang --- CMakeLists.txt | 22 +-- scripts/ascentVariables.sh | 26 ++-- src/Drivers/CMakeLists.txt | 8 +- src/Drivers/hpc_multisolves.cpp | 2 +- .../{nlpMDSForm_ex4.hpp => nlpMDS_ex4.hpp} | 127 +++++++++++------- src/Drivers/nlpMDS_ex4_driver.cpp | 40 ++++-- src/Drivers/nlpMDS_ex4_raja_driver.cpp | 35 +++-- ...SForm_raja_ex4.cpp => nlpMDS_raja_ex4.cpp} | 2 +- ...SForm_raja_ex4.hpp => nlpMDS_raja_ex4.hpp} | 0 src/LinAlg/hiopMatrixRajaDense.cpp | 3 + src/LinAlg/hiopMatrixRajaSparseTriplet.cpp | 5 +- 11 files changed, 171 insertions(+), 99 deletions(-) rename src/Drivers/{nlpMDSForm_ex4.hpp => nlpMDS_ex4.hpp} (83%) rename src/Drivers/{nlpMDSForm_raja_ex4.cpp => nlpMDS_raja_ex4.cpp} (99%) rename src/Drivers/{nlpMDSForm_raja_ex4.hpp => nlpMDS_raja_ex4.hpp} (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7376cc61e..ab4b30b86 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -407,6 +407,9 @@ if (HIOP_WITH_MAKETEST) add_test(NAME NlpMixedDenseSparse4_2 COMMAND ${RUNCMD} bash -c "$ 400 100 1 -selfcheck \ | ${STRIP_TABLE_CMD} \ | tee ${PROJECT_BINARY_DIR}/mds4_2.out") + + add_test(NAME NlpMixedDenseSparse4_3 COMMAND ${RUNCMD} "$" "400" "100" "0" "-empty_sp_row" "-selfcheck") + if(HIOP_USE_RAJA) add_test(NAME NlpMixedDenseSparseRaja4_1 COMMAND ${RUNCMD} bash -c "$ 400 100 0 -selfcheck \ | ${STRIP_TABLE_CMD} \ @@ -414,7 +417,8 @@ if (HIOP_WITH_MAKETEST) add_test(NAME NlpMixedDenseSparseRaja4_2 COMMAND ${RUNCMD} bash -c "$ 400 100 1 -selfcheck \ | ${STRIP_TABLE_CMD} \ | tee ${PROJECT_BINARY_DIR}/mds4_raja_2.out") - + add_test(NAME NlpMixedDenseSparseRaja4_3 COMMAND ${RUNCMD} bash -c "$ 400 100 0 -empty_sp_row -selfcheck") + if(HIOP_DEEPCHECKS) foreach(iter 1 2) add_test( @@ -428,14 +432,14 @@ if (HIOP_WITH_MAKETEST) echo 'Output tables have a different number of iterations!' exit 1 fi") - add_test( - NAME "CompareExample4_ElementWise_${iter}" - COMMAND bash -c "\ - join ${PROJECT_BINARY_DIR}/mds4_${iter}.out ${PROJECT_BINARY_DIR}/mds4_raja_${iter}.out \ - | ${PROJECT_SOURCE_DIR}/tests/testEx4CompareIterations.awk") - endforeach() - endif(HIOP_DEEPCHECKS) - endif() + add_test( + NAME "CompareExample4_ElementWise_${iter}" + COMMAND bash -c "\ + join ${PROJECT_BINARY_DIR}/mds4_${iter}.out ${PROJECT_BINARY_DIR}/mds4_raja_${iter}.out \ + | ${PROJECT_SOURCE_DIR}/tests/testEx4CompareIterations.awk") + endforeach() + endif(HIOP_DEEPCHECKS) + endif() add_test(NAME NlpMixedDenseSparse5_1 COMMAND ${RUNCMD} "$" "400" "100" "-selfcheck") diff --git a/scripts/ascentVariables.sh b/scripts/ascentVariables.sh index 36d7a64f0..0a3f5e9a6 100644 --- a/scripts/ascentVariables.sh +++ b/scripts/ascentVariables.sh @@ -3,28 +3,24 @@ if [ ! -v BUILDDIR ]; then echo BUILDDIR is not set! Your paths may be misconfigured. fi +export MY_CLUSTER=ascent +export PROJ_DIR=/gpfs/wolf/proj-shared/csc359 +source $PROJ_DIR/src/spack/share/spack/setup-env.sh module purge -module load cuda/11 -module use /gpfs/wolf/proj-shared/csc359/ascent/Modulefiles/Core +module load cuda/10.2.89 +module use $PROJ_DIR/$MY_CLUSTER/Modulefiles/Core module load exasgd-base module load gcc-ext/7.4.0 module load spectrum-mpi-ext module load openblas -module use /gpfs/wolf/proj-shared/csc359/ascent/spack-modulefiles/ -module load magma-2.5.4-gcc-7.4.0-vjotnd3 -module load metis -module load mpfr -module load suitesparse module load cmake/3.18.2 -module load raja -module load umpire -module load valgrind/3.14.0 -export MY_RAJA_DIR=$RAJA_ROOT -export MY_UMPIRE_DIR=$UMPIRE_ROOT -export MY_METIS_DIR=$OLCF_METIS_ROOT -export MY_HIOP_MAGMA_DIR=$MAGMA_ROOT -export MY_UMFPACK_DIR=$SUITESPARSE_ROOT + +ls $PROJ_DIR/src/spack/var/spack/environments/* + +spack env activate exago-v1-0-0-hiop-v0-4-1 + export MY_NVCC_ARCH="sm_70" +export EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -DHIOP_NVCC_ARCH=$MY_NVCC_ARCH" if [[ ! -f $BUILDDIR/nvblas.conf ]]; then cat > $BUILDDIR/nvblas.conf < /* Problem test for the linear algebra of Mixed Dense-Sparse NLPs + * if 'empty_sp_row' is set to true: + * min sum 0.5 {x_i*(x_{i}-1) : i=1,...,ns} + 0.5 y'*Qd*y + 0.5 s^T s + * s.t. x+s + Md y = 0, i=1,...,ns + * [-2 ] [ x_1 + e^T s] [e^T] [ 2 ] + * [-inf] <= [ ] + [e^T] y <= [ 2 ] + * [-2 ] [ x_3 ] [e^T] [inf] + * x <= 3 + * s>=0 + * -4 <=y_1 <=4, the rest of y are free + * otherwise: * min sum 0.5 {x_i*(x_{i}-1) : i=1,...,ns} + 0.5 y'*Qd*y + 0.5 s^T s * s.t. x+s + Md y = 0, i=1,...,ns * [-2 ] [ x_1 + e^T s] [e^T] [ 2 ] @@ -43,21 +53,20 @@ class Ex4 : public hiop::hiopInterfaceMDS { public: - Ex4(int ns_) - : Ex4(ns_, ns_) + Ex4(int ns_, bool empty_sp_row = false) + : Ex4(ns_, ns_, empty_sp_row) { } - Ex4(int ns_, int nd_) - : ns(ns_), sol_x_(NULL), sol_zl_(NULL), sol_zu_(NULL), sol_lambda_(NULL) + Ex4(int ns_, int nd_, bool empty_sp_row = false) + : ns(ns_), sol_x_(NULL), sol_zl_(NULL), sol_zu_(NULL), sol_lambda_(NULL), empty_sp_row_(empty_sp_row) { if(ns<0) { ns = 0; } else { if(4*(ns/4) != ns) { - ns = 4*((4+ns)/4); - printf("[warning] number (%d) of sparse vars is not a multiple ->was altered to %d\n", - ns_, ns); + ns = 4*((4+ns)/4); + printf("[warning] number (%d) of sparse vars is not a multiple ->was altered to %d\n", ns_, ns); } } @@ -154,7 +163,11 @@ class Ex4 : public hiop::hiopInterfaceMDS nx_sparse = 2*ns; nx_dense = nd; nnz_sparse_Jace = 2*ns; - nnz_sparse_Jaci = (ns==0 || !haveIneq) ? 0 : 3+ns; + if(empty_sp_row_) { + nnz_sparse_Jaci = (ns==0 || !haveIneq) ? 0 : 2+ns; + } else { + nnz_sparse_Jaci = (ns==0 || !haveIneq) ? 0 : 3+ns; + } nnz_sparse_Hess_Lagr_SS = 2*ns; nnz_sparse_Hess_Lagr_SD = 0.; return true; @@ -196,25 +209,28 @@ class Ex4 : public hiop::hiopInterfaceMDS for(int irow=0; irow0 ) { + if(((con_idx-ns==1 && !empty_sp_row_) || con_idx-ns==2) && ns>0) { //w.r.t x_2 or x_3 iJacS[nnzit] = con_idx-ns; jJacS[nnzit] = con_idx-ns; @@ -322,7 +338,7 @@ class Ex4 : public hiop::hiopInterfaceMDS nnzit++; } } else { - if( (con_idx-ns==1 || con_idx-ns==2) && ns>0) { + if(((con_idx-ns==1 && !empty_sp_row_) || con_idx-ns==2) && ns>0) { //w.r.t x_2 or x_3 MJacS[nnzit] = 1.; nnzit++; @@ -470,18 +486,21 @@ class Ex4 : public hiop::hiopInterfaceMDS double* sol_zl_; double* sol_zu_; double* sol_lambda_; + + /* indicate if problem has empty row in constraint Jacobian */ + bool empty_sp_row_; }; class Ex4OneCallCons : public Ex4 { public: - Ex4OneCallCons(int ns_in) - : Ex4(ns_in) + Ex4OneCallCons(int ns_in, bool empty_sp_row = false) + : Ex4(ns_in, empty_sp_row) { } - Ex4OneCallCons(int ns_in, int nd_in) - : Ex4(ns_in, nd_in) + Ex4OneCallCons(int ns_in, int nd_in, bool empty_sp_row = false) + : Ex4(ns_in, nd_in, empty_sp_row) { } @@ -506,23 +525,27 @@ class Ex4OneCallCons : public Ex4 for(int con_idx=0; con_idx implementation of hiop::hiopInterfaceMDS Ex4* my_nlp; if(one_call_cons) { - my_nlp = new Ex4OneCallCons(n_sp, n_de); + my_nlp = new Ex4OneCallCons(n_sp, n_de, has_empty_sp_row); } else { - my_nlp = new Ex4(n_sp, n_de); + my_nlp = new Ex4(n_sp, n_de, has_empty_sp_row); } hiopNlpMDS nlp(*my_nlp); @@ -129,7 +142,14 @@ int main(int argc, char **argv) status = solver.run(); obj_value = solver.getObjective(); - if(status<0) { + + if(selfCheck && has_empty_sp_row) { + if(fabs(obj_value-(-4.9994888159755632e+01))>1e-6) { + printf("selfcheck: objective mismatch for Ex4 MDS problem with 400 sparse variables and 100 " + "dense variables did. BTW, obj=%18.12e was returned by HiOp.\n", obj_value); + return -1; + } + } else if(status<0) { if(rank==0) printf("solver returned negative solve status: %d (with objective is %18.12e)\n", status, obj_value); return -1; diff --git a/src/Drivers/nlpMDS_ex4_raja_driver.cpp b/src/Drivers/nlpMDS_ex4_raja_driver.cpp index b5f0671fe..1bb6e79c1 100644 --- a/src/Drivers/nlpMDS_ex4_raja_driver.cpp +++ b/src/Drivers/nlpMDS_ex4_raja_driver.cpp @@ -1,4 +1,4 @@ -#include "nlpMDSForm_raja_ex4.hpp" +#include "nlpMDS_raja_ex4.hpp" #include "hiopNlpFormulation.hpp" #include "hiopAlgFilterIPM.hpp" @@ -20,9 +20,11 @@ static bool parse_arguments(int argc, char **argv, bool& self_check, long long& n_sp, long long& n_de, - bool& one_call_cons) + bool& one_call_cons, + bool& empty_sp_row) { self_check=false; + empty_sp_row = false; n_sp = 1000; n_de = 1000; one_call_cons = false; @@ -31,10 +33,19 @@ static bool parse_arguments(int argc, char **argv, //no arguments return true; break; + case 6: // 5 arguments + { + if(std::string(argv[5]) == "-selfcheck") + self_check=true; + } case 5: // 4 arguments { - if(std::string(argv[4]) == "-selfcheck") - self_check=true; + if(std::string(argv[4]) == "-selfcheck") { + self_check=true; + } + if(std::string(argv[4]) == "-empty_sp_row") { + empty_sp_row=true; + } } case 4: // 3 arguments { @@ -55,7 +66,7 @@ static bool parse_arguments(int argc, char **argv, return false; //5 or more arguments } - if(self_check && n_sp!=400 && n_de!=100) + if(self_check && (n_sp!=400 || n_de!=100) ) return false; return true; @@ -66,10 +77,11 @@ static void usage(const char* exeName) printf("HiOp driver %s that solves a synthetic problem of variable size in the " "mixed dense-sparse formulation.\n", exeName); printf("Usage: \n"); - printf(" '$ %s sp_vars_size de_vars_size eq_ineq_combined_nlp -selfcheck'\n", exeName); + printf(" '$ %s sp_vars_size de_vars_size eq_ineq_combined_nlp -empty_sp_row -selfcheck'\n", exeName); printf("Arguments, all integers, excepting string '-selfcheck'\n"); printf(" 'sp_vars_size': # of sparse variables [default 400, optional]\n"); printf(" 'de_vars_size': # of dense variables [default 100, optional]\n"); + printf(" '-empty_sp_row': set an empty row in sparser inequality Jacobian. [optional]\n"); printf(" '-selfcheck': compares the optimal objective with sp_vars_size being 400 and " "de_vars_size being 100 (these two exact values must be passed as arguments). [optional]\n"); printf(" 'eq_ineq_combined_nlp': 0 or 1, specifying whether the NLP formulation with split " @@ -101,8 +113,9 @@ int main(int argc, char **argv) std::string mem_space = "um"; bool selfCheck, one_call_cons; + bool has_empty_sp_row; long long n_sp, n_de; - if(!parse_arguments(argc, argv, selfCheck, n_sp, n_de, one_call_cons)) { + if(!parse_arguments(argc, argv, selfCheck, n_sp, n_de, one_call_cons, has_empty_sp_row)) { usage(argv[0]); return 1; } @@ -140,7 +153,13 @@ int main(int argc, char **argv) status = solver.run(); obj_value = solver.getObjective(); - if(status<0) { + if(selfCheck && has_empty_sp_row) { + if(fabs(obj_value-(-4.9994888159755632e+01))>1e-6) { + printf("selfcheck: objective mismatch for Ex4 MDS problem with 400 sparse variables and 100 " + "dense variables did. BTW, obj=%18.12e was returned by HiOp.\n", obj_value); + return -1; + } + } else if(status<0) { if(rank==0) printf("solver returned negative solve status: %d (with objective is %18.12e)\n", status, obj_value); return -1; diff --git a/src/Drivers/nlpMDSForm_raja_ex4.cpp b/src/Drivers/nlpMDS_raja_ex4.cpp similarity index 99% rename from src/Drivers/nlpMDSForm_raja_ex4.cpp rename to src/Drivers/nlpMDS_raja_ex4.cpp index 9b5aaacfb..ef2e4e150 100644 --- a/src/Drivers/nlpMDSForm_raja_ex4.cpp +++ b/src/Drivers/nlpMDS_raja_ex4.cpp @@ -1,4 +1,4 @@ -#include "nlpMDSForm_raja_ex4.hpp" +#include "nlpMDS_raja_ex4.hpp" #include #include diff --git a/src/Drivers/nlpMDSForm_raja_ex4.hpp b/src/Drivers/nlpMDS_raja_ex4.hpp similarity index 100% rename from src/Drivers/nlpMDSForm_raja_ex4.hpp rename to src/Drivers/nlpMDS_raja_ex4.hpp diff --git a/src/LinAlg/hiopMatrixRajaDense.cpp b/src/LinAlg/hiopMatrixRajaDense.cpp index 7faed28e9..7b46f6fc8 100644 --- a/src/LinAlg/hiopMatrixRajaDense.cpp +++ b/src/LinAlg/hiopMatrixRajaDense.cpp @@ -1385,6 +1385,9 @@ void hiopMatrixRajaDense::row_max_abs_value(hiopVector &ret_vec) { assert(ret_vec.get_size() == m()); ret_vec.setToZero(); + if(0 == m_local_) { + return; + } auto& vec = dynamic_cast(ret_vec); double* vd = vec.local_data(); diff --git a/src/LinAlg/hiopMatrixRajaSparseTriplet.cpp b/src/LinAlg/hiopMatrixRajaSparseTriplet.cpp index a064d5230..1e5862a06 100644 --- a/src/LinAlg/hiopMatrixRajaSparseTriplet.cpp +++ b/src/LinAlg/hiopMatrixRajaSparseTriplet.cpp @@ -492,7 +492,10 @@ void hiopMatrixRajaSparseTriplet::row_max_abs_value(hiopVector& ret_vec) #endif assert(ret_vec.get_size() == nrows_); ret_vec.setToZero(); - + if(0 == nrows_) { + return; + } + auto& vec = dynamic_cast(ret_vec); double* vd = vec.local_data();