From 0d4599a066db0edaa8bcd84dfd7c26b12242707d Mon Sep 17 00:00:00 2001 From: Marek Pecha Date: Wed, 18 Jan 2023 15:23:19 +0100 Subject: [PATCH] Implementations of stopping criteria (#20) New stopping criteria: duality gap and maximal dual violation --- include/permon/private/svmimpl.h | 2 + include/permonsvm.h | 4 + src/svm/impls/binary/binary.c | 215 +++++++++++++++++++- src/svm/impls/binary/binaryimpl.h | 3 + src/svm/interface/svm.c | 54 ++++- src/tutorials/ex4.c | 72 +++++++ src/tutorials/makefile | 6 +- src/tutorials/output/ex4_dual_violation.out | 24 +++ src/tutorials/output/ex4_duality_gap.out | 24 +++ 9 files changed, 399 insertions(+), 5 deletions(-) create mode 100644 src/tutorials/ex4.c create mode 100644 src/tutorials/output/ex4_dual_violation.out create mode 100644 src/tutorials/output/ex4_duality_gap.out diff --git a/include/permon/private/svmimpl.h b/include/permon/private/svmimpl.h index 130068c2..f060e4d6 100644 --- a/include/permon/private/svmimpl.h +++ b/include/permon/private/svmimpl.h @@ -12,6 +12,7 @@ struct _SVMOps { PetscErrorCode (*destroy)(SVM); PetscErrorCode (*setfromoptions)(PetscOptionItems *,SVM); PetscErrorCode (*setup)(SVM); + PetscErrorCode (*convergedsetup)(SVM); PetscErrorCode (*train)(SVM); PetscErrorCode (*posttrain)(SVM); PetscErrorCode (*reconstructhyperplane)(SVM); @@ -66,5 +67,6 @@ struct _p_SVM { FLLOP_EXTERN PetscLogEvent SVM_LoadDataset; FLLOP_EXTERN PetscLogEvent SVM_LoadGramian; + #endif diff --git a/include/permonsvm.h b/include/permonsvm.h index 0de6168f..c9c58e52 100644 --- a/include/permonsvm.h +++ b/include/permonsvm.h @@ -196,6 +196,10 @@ FLLOP_EXTERN PetscErrorCode SVMSetAutoPostTrain(SVM,PetscBool); FLLOP_EXTERN PetscErrorCode SVMPredict(SVM,Mat,Vec *); FLLOP_EXTERN PetscErrorCode SVMTest(SVM); +FLLOP_EXTERN PetscErrorCode SVMConvergedSetUp(SVM); +FLLOP_EXTERN PetscErrorCode SVMDefaultConvergedCreate(SVM, void **); +FLLOP_EXTERN PetscErrorCode SVMDefaultConvergedDestroy(void *); + FLLOP_EXTERN PetscErrorCode SVMComputeModelScores(SVM,Vec,Vec); FLLOP_EXTERN PetscErrorCode SVMGetModelScore(SVM,ModelScore,PetscReal *); FLLOP_EXTERN PetscErrorCode SVMComputeHingeLoss(SVM svm); diff --git a/src/svm/impls/binary/binary.c b/src/svm/impls/binary/binary.c index 64a839ab..792611f2 100644 --- a/src/svm/impls/binary/binary.c +++ b/src/svm/impls/binary/binary.c @@ -2,6 +2,7 @@ #include "binaryimpl.h" const char *const SVMLossTypes[]={"L1","L2","SVMLossType","SVM_",0}; +const char *const SVMConvergedTypes[] = {"default", "duality_gap", "dual_violation", "SVMConvergedType", "SVM_", 0}; typedef struct { SVM svm_inner; @@ -477,7 +478,6 @@ PetscErrorCode SVMGetQPS_Binary(SVM svm,QPS *qps) PetscCall(SVMCreateQPS_Binary_Private(svm,&qps_inner)); svm_binary->qps = qps_inner; } - *qps = svm_binary->qps; PetscFunctionReturn(0); } @@ -904,11 +904,15 @@ PetscErrorCode SVMSetUp_Binary(SVM svm) } } + /* Set stopping criteria */ + PetscCall(SVMConvergedSetUp(svm)); + /* Set QPS */ if (svm->setfromoptionscalled) { PetscCall(QPTFromOptions(qp)); PetscCall(QPSSetFromOptions(qps)); } + PetscCall(QPSSetUp(qps)); /* Create work vectors */ @@ -1597,6 +1601,214 @@ PetscErrorCode SVMTest_Binary(SVM svm) PetscFunctionReturn(0); } + +#undef __FUNCT__ +#define __FUNCT__ "SVMComputePGmaxPGmin_Binary_Private" +PetscErrorCode SVMComputePGminPGmax_Binary_Private(SVM svm,PetscReal *PG_min,PetscReal *PG_max) +{ + // https://www.jmlr.org/papers/volume8/loosli07a/loosli07a.pdf + SVM_Binary *svm_binary = (SVM_Binary *) svm->data; + + Vec x,g,sg; + Vec y; + + Vec lb,ub; + + IS is_yp,is_ym; + IS is_xgl,is_xlu,is_int; + + PetscReal min_1,min_2,max_1,max_2; + PetscInt rstart,rend; + + QPS qps; + QP qp; + + PetscFunctionBegin; + PetscValidHeaderSpecific(svm,SVM_CLASSID,1); + + PetscCall(SVMGetQPS(svm,&qps)); + PetscCall(QPSGetQP(qps,&qp)); + + PetscCall(QPGetSolutionVector(qp,&x)); // this function returns borrowed reference + PetscCall(QPGetBox(qp,NULL,&lb,&ub)); // this function returns borrowed reference + + y = svm_binary->y_inner; // labels remapped to +1 and -1 + g = qps->work[0]; // get gradient + + PetscCall(VecWhichGreaterThan(y,lb,&is_yp)); // lb = zeros, then is_yp contains indices of samples related to class +1 + PetscCall(VecGetOwnershipRange(y,&rstart,&rend)); + PetscCall(ISComplement(is_yp,rstart,rend,&is_ym)); // is_y, contains indices of samples related to class -1 + + PetscCall(VecWhichGreaterThan(x,lb,&is_xgl)); // get index set for x < C or x < Cp, x < Cm + PetscCall(VecWhichLessThan(x,ub,&is_xlu)); // get index set for x > 0 + + /* Compute PG_max */ + PetscCall(ISIntersect(is_yp,is_xlu,&is_int)); // get index set for x < C & y = +1 + PetscCall(VecGetSubVector(g,is_int,&sg)); // get sub gradient sg = g(x < C & y = +1) + PetscCall(VecMin(sg,NULL,&max_1)); + max_1 *= -1.0; // max_1 = max(-sg) = -min(sg) = -min(g(x < C & y = 1)) + PetscCall(VecRestoreSubVector(g,is_int,&sg)); + PetscCall(ISDestroy(&is_int)); + + PetscCall(ISIntersect(is_ym,is_xgl,&is_int)); // get index set for x > 0 & y == -1 + PetscCall(VecGetSubVector(g,is_int,&sg)); // get sub gradient sg = g(x > 0 & y == -1) + PetscCall(VecMax(sg,NULL,&max_2)); // max_2 = max(sg) = max(g(x > 0 & y = -1)) + PetscCall(VecRestoreSubVector(g,is_int,&sg)); + PetscCall(ISDestroy(&is_int)); + + *PG_max = PetscMax(max_1,max_2); + + /* Compute PG_min */ + PetscCall(ISIntersect(is_ym,is_xlu,&is_int)); // get index set x < C & y = -1 + PetscCall(VecGetSubVector(g,is_int,&sg)); // min_1 = min(sg) = min(g(x < C & y = -1)) + PetscCall(VecMin(sg,NULL,&min_1)); + PetscCall(VecRestoreSubVector(g,is_int,&sg)); + PetscCall(ISDestroy(&is_int)); + + PetscCall(ISIntersect(is_yp,is_xgl,&is_int)); // get index set x > 0 & y = +1 + PetscCall(VecGetSubVector(g,is_int,&sg)); + PetscCall(VecMax(sg,NULL,&min_2)); + min_2 *= -1.0; + PetscCall(VecRestoreSubVector(g,is_int,&sg)); + PetscCall(ISDestroy(&is_int)); + + *PG_min = PetscMin(min_1,min_2); + + PetscCall(ISDestroy(&is_yp)); + PetscCall(ISDestroy(&is_ym)); + PetscCall(ISDestroy(&is_xgl)); + PetscCall(ISDestroy(&is_xlu)); + PetscFunctionReturn(0); +} + +#undef __FUNCT__ +#define __FUNCT__ "SVMConvergedMaximalDualViolation_Binary" +PetscErrorCode SVMConvergedMaximalDualViolation_Binary(QPS qps,KSPConvergedReason *reason) +{ + SVM svm; + + PetscInt max_it; + PetscReal atol; + + PetscReal PGmin,PGmax,v=0.; + PetscInt it; + + PetscFunctionBegin; + PetscValidHeaderSpecific(qps,QPS_CLASSID,1); + + PetscCall(QPSGetConvergenceContext(qps,(void*)&svm)); + PetscCall(QPSGetIterationNumber(qps,&it)); + PetscCall(QPSGetTolerances(qps,NULL,&atol,NULL,&max_it)); + + *reason = KSP_CONVERGED_ITERATING; + if (it == 0) PetscFunctionReturn(0); + + PetscCall(SVMComputePGminPGmax_Binary_Private(svm,&PGmin,&PGmax)); + v = PGmax - PGmin; + + if (it > max_it) { + *reason = KSP_DIVERGED_ITS; + PetscCall(PetscInfo(qps,"QP solver is diverging (iteration count reached the maximum). Current dual violation %14.12e at iteration %" PetscInt_FMT "\n",(double) v,it)); + PetscFunctionReturn(0); + } + + if ((v <= atol) && PetscAbsReal(PGmax) <= atol && PetscAbsReal(PGmin) <= atol) { + *reason = KSP_CONVERGED_ATOL; + PetscCall(PetscInfo(qps,"QP solver has converged. Dual violation %14.12e at iteration %" PetscInt_FMT "\n",(double) v,it)); + } + + PetscFunctionReturn(0); +} + +#undef __FUNCT__ +#define __FUNCT__ "SVMConvergedDualityGap_Binary" +PetscErrorCode SVMConvergedDualityGap_Binary(QPS qps,KSPConvergedReason *reason) +{ + SVM_Binary *svm_binary; + SVM svm; + + PetscInt max_it; + PetscReal rtol; + + PetscReal D,P; + + PetscReal gap; + PetscInt it; + + PetscFunctionBegin; + PetscValidHeaderSpecific(qps,QPS_CLASSID,1); + + PetscCall(QPSGetConvergenceContext(qps,(void*)&svm)); + PetscCall(QPSGetIterationNumber(qps,&it)); + PetscCall(QPSGetTolerances(qps,&rtol,NULL,NULL,&max_it)); + + svm_binary = (SVM_Binary *) svm->data; + + *reason = KSP_CONVERGED_ITERATING; + if (it == 0) PetscFunctionReturn(0); + + // compute duality gap + PetscCall(SVMReconstructHyperplane(svm)); + PetscCall(SVMComputeObjFuncValues_Binary_Private(svm)); + + P = svm_binary->primalObj; + D = svm_binary->dualObj; + + gap = PetscAbsReal(P - D); + + if (it > max_it) { + *reason = KSP_DIVERGED_ITS; + PetscCall(PetscInfo(qps,"QP solver is diverging (iteration count reached the maximum). Current duality gap %14.12e at iteration %" PetscInt_FMT "\n",(double) gap,it)); + PetscFunctionReturn(0); + } + + if (gap <= rtol * P) { + *reason = KSP_CONVERGED_RTOL; + PetscCall(PetscInfo(qps,"QP solver has converged. Duality gap %14.12e at iteration %" PetscInt_FMT "\n",(double) gap,it)); + } + + PetscFunctionReturn(0); +} + +#undef __FUNCT__ +#define __FUNCT__ "SVMConvergedSetUp_Binary" +PetscErrorCode SVMConvergedSetUp_Binary(SVM svm) +{ + SVMConvergedType type_stop_criteria; + void *ctx; + + QPS qps; + PetscInt svm_mod; + + PetscFunctionBegin; + PetscValidHeaderSpecific(svm,SVM_CLASSID,1); + + type_stop_criteria = SVM_CONVERGED_DEFAULT; + + PetscCall(PetscOptionsGetEnum(NULL,((PetscObject) svm)->prefix,"-svm_binary_convergence_test",SVMConvergedTypes,(PetscEnum*)&type_stop_criteria,NULL)); + if (type_stop_criteria == SVM_CONVERGED_DEFAULT) PetscFunctionReturn(0); + + PetscCall(SVMGetMod(svm,&svm_mod)); + PetscCall(SVMGetQPS(svm,&qps)); + + switch (type_stop_criteria) { + // convergence test based on maximal dual violation + case SVM_CONVERGED_MAXIMAL_DUAL_VIOLATION: + PetscCall(SVMDefaultConvergedCreate(svm,&ctx)); + PetscCall(QPSSetConvergenceTest(qps,SVMConvergedMaximalDualViolation_Binary,ctx,SVMDefaultConvergedDestroy)); + break; + // convergence test based on duality gap + case SVM_CONVERGED_DUALITY_GAP: + PetscCall(SVMDefaultConvergedCreate(svm,&ctx)); + PetscCall(QPSSetConvergenceTest(qps,SVMConvergedDualityGap_Binary,ctx,SVMDefaultConvergedDestroy)); + break; + default: + break; + } + + PetscFunctionReturn(0); +} + #undef __FUNCT__ #define __FUNCT__ "SVMGetModelScore_Binary" PetscErrorCode SVMGetModelScore_Binary(SVM svm,ModelScore score_type,PetscReal *s) @@ -1930,6 +2142,7 @@ PetscErrorCode SVMCreate_Binary(SVM svm) svm->ops->reset = SVMReset_Binary; svm->ops->destroy = SVMDestroy_Binary; svm->ops->setfromoptions = SVMSetFromOptions_Binary; + svm->ops->convergedsetup = SVMConvergedSetUp_Binary; svm->ops->train = SVMTrain_Binary; svm->ops->posttrain = SVMPostTrain_Binary; svm->ops->reconstructhyperplane = SVMReconstructHyperplane_Binary; diff --git a/src/svm/impls/binary/binaryimpl.h b/src/svm/impls/binary/binaryimpl.h index ccae7e1a..b6b26f14 100644 --- a/src/svm/impls/binary/binaryimpl.h +++ b/src/svm/impls/binary/binaryimpl.h @@ -40,6 +40,9 @@ typedef struct { PetscReal primalObj,dualObj; } SVM_Binary; +typedef enum {SVM_CONVERGED_DEFAULT,SVM_CONVERGED_DUALITY_GAP,SVM_CONVERGED_MAXIMAL_DUAL_VIOLATION} SVMConvergedType; +FLLOP_EXTERN const char *const SVMConvergedTypes[]; + FLLOP_EXTERN PetscErrorCode SVMCrossValidation_Binary(SVM,PetscReal [],PetscInt,PetscReal []); FLLOP_EXTERN PetscErrorCode SVMKFoldCrossValidation_Binary(SVM,PetscReal [],PetscInt,PetscReal []); FLLOP_EXTERN PetscErrorCode SVMStratifiedKFoldCrossValidation_Binary(SVM,PetscReal [],PetscInt,PetscReal []); diff --git a/src/svm/interface/svm.c b/src/svm/interface/svm.c index e473acd6..fe954402 100644 --- a/src/svm/interface/svm.c +++ b/src/svm/interface/svm.c @@ -153,15 +153,27 @@ PetscErrorCode SVMDestroyDefault(SVM svm) @*/ PetscErrorCode SVMDestroy(SVM *svm) { + QPS qps; + void *cctx; PetscFunctionBegin; if (!*svm) PetscFunctionReturn(0); PetscValidHeaderSpecific(*svm,SVM_CLASSID,1); - if (--((PetscObject) (*svm))->refct > 0) { + --((PetscObject)(*svm))->refct; + if (((PetscObject)(*svm))->refct == 1) { + PetscCall(SVMGetQPS(*svm,&qps)); + PetscCall(QPSGetConvergenceContext(qps,&cctx)); + if (*svm != cctx) { + *svm = 0; + PetscFunctionReturn(0); + } + } else if (((PetscObject)(*svm))->refct > 1) { *svm = 0; PetscFunctionReturn(0); } + if (((PetscObject)(*svm))->refct < 0) PetscFunctionReturn(0); + ((PetscObject)(*svm))->refct = 0; PetscCall(SVMReset(*svm)); if ((*svm)->ops->destroy) { @@ -2067,6 +2079,46 @@ PetscErrorCode SVMTest(SVM svm) PetscFunctionReturnI(0); } +#undef __FUNCT__ +#define __FUNCT__ "SVMConvergedSetUp" +/*@ + +@*/ +PetscErrorCode SVMConvergedSetUp(SVM svm) +{ + + PetscFunctionBegin; + PetscValidHeaderSpecific(svm,SVM_CLASSID,1); + // call specific implementation of setting convergence test + if (svm->ops->convergedsetup) PetscCall(svm->ops->convergedsetup(svm)); + PetscFunctionReturn(0); +} + +#undef __FUNCT__ +#define __FUNCT__ "SVMDefaultConvergedCreate" +/*@ + +@*/ +PetscErrorCode SVMDefaultConvergedCreate(SVM svm, void **ctx) +{ + PetscFunctionBegin; + PetscValidHeaderSpecific(svm,SVM_CLASSID,1); + *ctx = svm; + PetscFunctionReturn(0); +} + +#undef __FUNCT__ +#define __FUNCT__ "SVMDefaultConvergedDestroy" +/*@ + +@*/ +PetscErrorCode SVMDefaultConvergedDestroy(void *ctx) +{ + PetscFunctionBegin; + PetscCall(SVMDestroy((SVM*)&ctx)); + PetscFunctionReturn(0); +} + #undef __FUNCT__ #define __FUNCT__ "SVMGetModelScore" /*@ diff --git a/src/tutorials/ex4.c b/src/tutorials/ex4.c new file mode 100644 index 00000000..c35b6602 --- /dev/null +++ b/src/tutorials/ex4.c @@ -0,0 +1,72 @@ + +static char help[] = "Trains binary L2-loss SVM classification model that training process was stopped using terminating condition\ +based on duality gap.\n\ +Input parameters:\n\ + -f : training dataset file\n\ + -f_test : test dataset file\n\ + -svm_binary_convergence: duality_gap\n\ +"; + +#include + +int main(int argc,char **argv) +{ + SVM svm; + + char training_file[PETSC_MAX_PATH_LEN] = "data/heart_scale.bin"; + char test_file[PETSC_MAX_PATH_LEN] = "data/heart_scale.t.bin"; + + PetscViewer viewer; + + PetscCall(PermonInitialize(&argc,&argv,(char *)0,help)); + + PetscCall(PetscOptionsGetString(NULL,NULL,"-f_training",training_file,sizeof(training_file),NULL)); + PetscCall(PetscOptionsGetString(NULL,NULL,"-f_test",test_file,sizeof(test_file),NULL)); + + /* Create SVM object */ + PetscCall(SVMCreate(PETSC_COMM_WORLD,&svm)); + PetscCall(SVMSetType(svm,SVM_BINARY)); + PetscCall(SVMSetFromOptions(svm)); + + /* Load training dataset */ + PetscCall(PetscViewerBinaryOpen(PETSC_COMM_WORLD,training_file,FILE_MODE_READ,&viewer)); + PetscCall(SVMLoadTrainingDataset(svm,viewer)); + PetscCall(PetscViewerDestroy(&viewer)); + + /* Load test dataset */ + PetscCall(PetscViewerBinaryOpen(PETSC_COMM_WORLD,test_file,FILE_MODE_READ,&viewer)); + PetscCall(SVMLoadTestDataset(svm,viewer)); + PetscCall(PetscViewerDestroy(&viewer)); + + /* Train classification model */ + PetscCall(SVMTrain(svm)); + /* Test performance of SVM model */ + PetscCall(SVMTest(svm)); + + /* Free memory */ + PetscCall(SVMDestroy(&svm)); + PetscCall(PermonFinalize()); + + return 0; +} + +/*TEST + + test: + suffix: duality_gap + filter: grep -v MPI + args: -qps_view_convergence -svm_view -svm_view_score -svm_binary_convergence_test duality_gap + args: -svm_loss_type L2 + args: -f_training $PERMON_SVM_DIR/src/tutorials/data/heart_scale.bin + args: -f_test $PERMON_SVM_DIR/src/tutorials/data/heart_scale.t.bin + output_file: output/ex4_duality_gap.out + + test: + suffix: dual_violation + filter: grep -v MPI + args: -qps_view_convergence -svm_view -svm_view_score -svm_binary_convergence_test dual_violation + args: -svm_loss_type L1 -svm_binary_mod 2 -qps_atol 1e-2 + args: -f_training $PERMON_SVM_DIR/src/tutorials/data/heart_scale.bin + args: -f_test $PERMON_SVM_DIR/src/tutorials/data/heart_scale.t.bin + output_file: output/ex4_dual_violation.out +TEST*/ diff --git a/src/tutorials/makefile b/src/tutorials/makefile index 81fcfaae..2fcf96df 100644 --- a/src/tutorials/makefile +++ b/src/tutorials/makefile @@ -4,8 +4,8 @@ CFLAGS = -I. FFLAGS = CPPFLAGS = FPPFLAGS = -TUTORIALS = exbinfile.c ex1.c ex2.c ex3.c -EXAMPLESC = exbinfile.c ex1.c ex2.c ex3.c +TUTORIALS = exbinfile.c ex1.c ex2.c ex3.c ex4.c +EXAMPLESC = exbinfile.c ex1.c ex2.c ex3.c ex4.c EXAMPLECH = EXAMPLESF = EXAMPLESFH = @@ -13,7 +13,7 @@ LOCDIR = src/examples MANSEC = DOCS = DIRS = -CLEANFILES = exbinfile ex1 ex2 ex3 +CLEANFILES = exbinfile ex1 ex2 ex3 ex4 include ${PERMON_SVM_DIR}/lib/permonsvm/conf/permonsvm_base include ${PERMON_SVM_DIR}/lib/permonsvm/conf/permonsvm_test diff --git a/src/tutorials/output/ex4_dual_violation.out b/src/tutorials/output/ex4_dual_violation.out new file mode 100644 index 00000000..10bf9a67 --- /dev/null +++ b/src/tutorials/output/ex4_dual_violation.out @@ -0,0 +1,24 @@ + type: mpgp + last QPSSolve CONVERGED due to CONVERGED_ATOL, KSPReason=3, required 122 iterations + all 1 QPSSolves from last QPSReset/QPSResetStatistics have required 122 iterations + tolerances: rtol=1.0e-01, abstol=1.0e-02, dtol=1.0e+100, maxits=10000 + mpgp specific: + from the last QPSReset: + number of Hessian multiplications 200 + number of CG steps 44 + number of expansion steps 77 + number of proportioning steps 1 + type: binary + model parameters: + ||w||=2.1926 bias=0.0000 margin=0.9121 NSV=12 + L1 hinge loss: + sum(xi_i)=60.3532 + objective functions: + primalObj=62.7570 dualObj=62.7426 gap=0.0144 + type: binary + model performance score with training parameters C=1.000, mod=2, loss=L1: + Confusion matrix: + TP = 27 FP = 9 + FN = 8 TN = 46 + accuracy=81.11% precision=75.00% sensitivity=77.14% + F1=0.76 MCC=0.60 AUC_ROC=0.80 G1=0.61 diff --git a/src/tutorials/output/ex4_duality_gap.out b/src/tutorials/output/ex4_duality_gap.out new file mode 100644 index 00000000..8aae20e1 --- /dev/null +++ b/src/tutorials/output/ex4_duality_gap.out @@ -0,0 +1,24 @@ + type: mpgp + last QPSSolve CONVERGED due to CONVERGED_RTOL, KSPReason=2, required 19 iterations + all 1 QPSSolves from last QPSReset/QPSResetStatistics have required 19 iterations + tolerances: rtol=1.0e-01, abstol=1.0e-50, dtol=1.0e+100, maxits=10000 + mpgp specific: + from the last QPSReset: + number of Hessian multiplications 39 + number of CG steps 0 + number of expansion steps 19 + number of proportioning steps 0 + type: binary + model parameters: + ||w||=1.9495 bias=0.0000 margin=1.0259 NSV=125 + L2 hinge loss: + sum(xi_i^2)=78.9355 + objective functions: + primalObj=41.3680 dualObj=38.2264 gap=3.1415 + type: binary + model performance score with training parameters C=1.000, mod=2, loss=L2: + Confusion matrix: + TP = 24 FP = 12 + FN = 8 TN = 46 + accuracy=77.78% precision=66.67% sensitivity=75.00% + F1=0.71 MCC=0.53 AUC_ROC=0.77 G1=0.54