Merge pull request #327 from FluidityProject/fix-petsc314-mg

Fix "mg" preconditioner with petsc 3.14

Author: stephankramer

aclocal.m4

@@ -439,9 +439,9 @@ AC_COMPUTE_INT(PETSC_VERSION_MAJOR, "PETSC_VERSION_MAJOR", [#include "petscversi
 AC_COMPUTE_INT(PETSC_VERSION_MINOR, "PETSC_VERSION_MINOR", [#include "petscversion.h"], 
   [AC_MSG_ERROR([Unknown petsc minor version])])
 AC_MSG_NOTICE([Detected PETSc version "$PETSC_VERSION_MAJOR"."$PETSC_VERSION_MINOR"])
-# if major<3 or minor<4
-if test "0$PETSC_VERSION_MAJOR" -lt 3 -o "0$PETSC_VERSION_MINOR" -lt 4; then
-  AC_MSG_ERROR([Fluidity needs PETSc version >=3.4])
+# if major<3 or minor<8
+if test "0$PETSC_VERSION_MAJOR" -lt 3 -o "0$PETSC_VERSION_MINOR" -lt 8; then
+  AC_MSG_ERROR([Fluidity needs PETSc version >=3.8])
 fi
 
 AC_LANG(Fortran)
@@ -451,35 +451,27 @@ ac_ext=F90
 # may need some extra flags for testing that we don't want to use in the end
 SAVE_FCFLAGS="$FCFLAGS"
 
-if test "$enable_petsc_fortran_modules" != "no" ; then
-  # now try if the petsc fortran modules work:
-  AC_LINK_IFELSE(
-          [AC_LANG_PROGRAM([],[[
-                          use petsc
-                          integer :: ierr
-                          print*, "hello petsc"
-                          call PetscInitialize(PETSC_NULL_CHARACTER, ierr)
-                          ]])],
-          [
-              AC_MSG_NOTICE([PETSc modules are working.])
-              AC_DEFINE(HAVE_PETSC_MODULES,1,[Define if you have petsc fortran modules.] )
-              FCFLAGS="$FCFLAGS -DHAVE_PETSC_MODULES"
-          ],
-          [
-              AC_MSG_NOTICE([PETSc modules don't work, using headers instead.])
-          ])
-elif test "0$PETSC_VERSION_MINOR" -ge 8; then
-  AC_MSG_ERROR([PETSc v3.8 and newer requires petsc fortran modules. Do not use the --disable-petsc-fortran-modules option])
-fi
+# now try if the petsc fortran modules work:
+AC_LINK_IFELSE(
+        [AC_LANG_PROGRAM([],[[
+                        use petsc
+                        integer :: ierr
+                        print*, "hello petsc"
+                        call PetscInitialize(PETSC_NULL_CHARACTER, ierr)
+                        ]])],
+        [
+            AC_MSG_NOTICE([PETSc modules are working.])
+        ],
+        [
+            AC_MSG_ERROR([PETSc fortran modules cannot be used. Either it the petsc*.mod files cannot be found, or petsc has been built using a different fortran compiler.])
+        ])
 
 # now try a more realistic program, it's a stripped down
 # petsc tutorial - using the headers in the same way as we do in the code
 AC_LINK_IFELSE(
 [AC_LANG_SOURCE([
 program test_petsc
-#ifdef HAVE_PETSC_MODULES
   use petsc
-#endif
 implicit none
 #include "petsc_legacy.h"
       double precision  norm

assemble/Full_Projection.F90

@@ -31,9 +31,7 @@ module Full_Projection
     use global_parameters
     use elements
     use spud
-#ifdef HAVE_PETSC_MODULES
     use petsc
-#endif
     use parallel_tools
     use data_structures
     use sparse_tools
@@ -359,11 +357,7 @@ subroutine petsc_solve_setup_full_projection(y,A,b,ksp,petsc_numbering_p,name,so
       else
          ! workaround bug in petsc 3.8: missing CHKFORTRANNULLOBJECT, so PETSC_NULL_MAT isn't translated to C null
          ! this however seems to do the trick:
-#if PETSC_VERSION_MINOR>=8
          S%v = 0
-#else
-         S = PETSC_NULL_OBJECT
-#endif
          call MatCreateSchurComplement(inner_M%M,inner_M%M,G,G_t_comp,S,A,ierr)
       end if
 

assemble/tests/test_matrix_free.F90

@@ -236,9 +236,7 @@ subroutine petsc_solve_matrix_free(x, matrix, rhs)
   use petsc_tools
   use matrix_free_solvers
 
-#ifdef HAVE_PETSC_MODULES
   use petsc 
-#endif
   implicit none
 #include "petsc_legacy.h"
   type(scalar_field), intent(inout) :: x

assemble/tests/test_pressure_solve.F90

@@ -15,9 +15,7 @@ subroutine test_pressure_solve
     use boundary_conditions
     use global_parameters, only: OPTION_PATH_LEN
     use free_surface_module
-#ifdef HAVE_PETSC_MODULES
-  use petsc 
-#endif
+    use petsc
     implicit none
 #include "petsc_legacy.h"
 

configure

@@ -785,7 +785,6 @@ with_femdem
 with_psmile
 with_netcdf
 with_exodusii
-enable_petsc_fortran_modules
 with_x
 enable_sam
 enable_dp
@@ -1463,12 +1462,6 @@ Optional Features:
   --enable-shared         Compile objects with -fPIC to enable the 'make
                           shared' target.
   --enable-verbose        turns on super verbosity
-  --disable-petsc-fortran-modules
-                          By default fluidity tries to use fortran modules
-                          provided with petsc. Use
-                          --disable-petsc-fortran-modules if these are known
-                          to be not functional (e.g. compiled with different
-                          compiler version)
   --enable-sam            use sam rather than zoltan for adaptive load
                           rebalancing
   --enable-dp=flag        compile with 64 bit floating point numbers (default)
@@ -6001,7 +5994,7 @@ then
     if test -z "$PYTHON_VERSION"; then
       # if PYTHON_VERSION is set we use python$PYTHON_VERSION as the python interpreter
       # if not, if there is no `python` in the path, or there is a `python` but its version is <3
-      # then use `python3` (by setting PETSC_VERSION=3).
+      # then use `python3` (by setting PYTHON_VERSION=3).
       # If there *is* a `python` of version 3, we use that
       # Extract the first word of "python", so it can be a program name with args.
 set dummy python; ac_word=$2
@@ -11791,11 +11784,6 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu
 
 
 
-fi
-
-# Check whether --enable-petsc-fortran-modules was given.
-if test "${enable_petsc_fortran_modules+set}" = set; then :
-  enableval=$enable_petsc_fortran_modules;
 fi
 
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for X" >&5
@@ -12724,9 +12712,9 @@ fi
 
 { $as_echo "$as_me:${as_lineno-$LINENO}: Detected PETSc version \"$PETSC_VERSION_MAJOR\".\"$PETSC_VERSION_MINOR\"" >&5
 $as_echo "$as_me: Detected PETSc version \"$PETSC_VERSION_MAJOR\".\"$PETSC_VERSION_MINOR\"" >&6;}
-# if major<3 or minor<4
-if test "0$PETSC_VERSION_MAJOR" -lt 3 -o "0$PETSC_VERSION_MINOR" -lt 4; then
-  as_fn_error $? "Fluidity needs PETSc version >=3.4" "$LINENO" 5
+# if major<3 or minor<8
+if test "0$PETSC_VERSION_MAJOR" -lt 3 -o "0$PETSC_VERSION_MINOR" -lt 8; then
+  as_fn_error $? "Fluidity needs PETSc version >=3.8" "$LINENO" 5
 fi
 
 ac_ext=${ac_fc_srcext-f}
@@ -12740,47 +12728,36 @@ ac_ext=F90
 # may need some extra flags for testing that we don't want to use in the end
 SAVE_FCFLAGS="$FCFLAGS"
 
-if test "$enable_petsc_fortran_modules" != "no" ; then
-  # now try if the petsc fortran modules work:
-  cat > conftest.$ac_ext <<_ACEOF
+# now try if the petsc fortran modules work:
+cat > conftest.$ac_ext <<_ACEOF
       program main
 
-                          use petsc
-                          integer :: ierr
-                          print*, "hello petsc"
-                          call PetscInitialize(PETSC_NULL_CHARACTER, ierr)
+                        use petsc
+                        integer :: ierr
+                        print*, "hello petsc"
+                        call PetscInitialize(PETSC_NULL_CHARACTER, ierr)
 
       end
 _ACEOF
 if ac_fn_fc_try_link "$LINENO"; then :
 
-              { $as_echo "$as_me:${as_lineno-$LINENO}: PETSc modules are working." >&5
+            { $as_echo "$as_me:${as_lineno-$LINENO}: PETSc modules are working." >&5
 $as_echo "$as_me: PETSc modules are working." >&6;}
 
-$as_echo "#define HAVE_PETSC_MODULES 1" >>confdefs.h
-
-              FCFLAGS="$FCFLAGS -DHAVE_PETSC_MODULES"
-
 else
 
-              { $as_echo "$as_me:${as_lineno-$LINENO}: PETSc modules don't work, using headers instead." >&5
-$as_echo "$as_me: PETSc modules don't work, using headers instead." >&6;}
+            as_fn_error $? "PETSc fortran modules cannot be used. Either it the petsc*.mod files cannot be found, or petsc has been built using a different fortran compiler." "$LINENO" 5
 
 fi
 rm -f core conftest.err conftest.$ac_objext \
     conftest$ac_exeext conftest.$ac_ext
-elif test "0$PETSC_VERSION_MINOR" -ge 8; then
-  as_fn_error $? "PETSc v3.8 and newer requires petsc fortran modules. Do not use the --disable-petsc-fortran-modules option" "$LINENO" 5
-fi
 
 # now try a more realistic program, it's a stripped down
 # petsc tutorial - using the headers in the same way as we do in the code
 cat > conftest.$ac_ext <<_ACEOF
 
 program test_petsc
-#ifdef HAVE_PETSC_MODULES
   use petsc
-#endif
 implicit none
 #include "petsc_legacy.h"
       double precision  norm

configure.in

@@ -193,7 +193,7 @@ then
     if test -z "$PYTHON_VERSION"; then
       # if PYTHON_VERSION is set we use python$PYTHON_VERSION as the python interpreter
       # if not, if there is no `python` in the path, or there is a `python` but its version is <3
-      # then use `python3` (by setting PETSC_VERSION=3).
+      # then use `python3` (by setting PYTHON_VERSION=3).
       # If there *is* a `python` of version 3, we use that
       AC_CHECK_PROG(have_bare_python, python, "yes")
       if test "x$have_bare_python" = "xyes"; then
@@ -828,9 +828,6 @@ if test "$with_exodusii" = "yes" ; then
                                            ])
 fi
 AC_SUBST(HAVE_EXODUSII)
-AC_ARG_ENABLE(petsc-fortran-modules,
-    [AC_HELP_STRING([--disable-petsc-fortran-modules],
-    [By default fluidity tries to use fortran modules provided with petsc. Use --disable-petsc-fortran-modules if these are known to be not functional (e.g. compiled with different compiler version)])])
 ACX_PETSc([AC_MSG_NOTICE(["PETSc found, enabling scalable solver"])], [AC_MSG_ERROR(["PETSc not found"])])
 ACX_hypre([AC_MSG_NOTICE(["hypre found, enabling hypre preconditioners"])], [AC_MSG_WARN(["hypre not found"])])
 CPPFLAGS="$CPPFLAGS -DHAVE_PETSC"

femtools/ISCopyIndices.cpp

@@ -1,25 +1,18 @@
 #include "confdefs.h"
 #include "petsc.h"
-#if PETSC_VERSION_MINOR==0
-#include "petscfix.h"
-#include "petscis.h"
-#endif
 
 #include <string.h>
 
 extern "C" {
   void iscopyindices_(IS *is, PetscInt *iarray,PetscErrorCode *ierr);
   void petscobjectreference_(PetscObject obj, int *__ierr );
+  void pcmgsetlevels_nocomms_(PC *pc, PetscInt *levels, int *ierr);
 }
 
 // our own version of IsGetIndices, that just does a copy
 // to avoid silly fortran interface issues
 void iscopyindices_(IS *is, PetscInt *iarray,PetscErrorCode *ierr){
-#if PETSC_VERSION_MAJOR==3
   const PetscInt *iptr;
-#else
-  PetscInt *iptr;
-#endif
   PetscInt n;
 
   *ierr = ISGetIndices(*is, &iptr); if (*ierr) return;
@@ -58,3 +51,14 @@ void petscobjectreference_(PetscObject obj, int *__ierr ){
 }
 
 
+// This is a woraround bug in https://gitlab.com/petsc/petsc/-/issues/868
+// In 3.14 not only is it impossible to "fake" a NULL argument using PETSC_NULL_KSP
+// as we'd done previously, even if you do provide a comms filled with MPI_COMM_WORLD
+// that because MPI_COMM_WORLD==0 it is still mistaken for a NULL argument and hits the
+// bug in the petsc fortran wrapper for this routine
+// So instead we just write our own wrapper without the comms argument:
+void pcmgsetlevels_nocomms_(PC *pc, PetscInt *levels, int *ierr){
+  *ierr = PCMGSetLevels(*pc, *levels, NULL);
+}
+
+

femtools/Multigrid.F90

@@ -6,9 +6,7 @@ module multigrid
 use spud
 use futils
 use parallel_tools
-#ifdef HAVE_PETSC_MODULES
   use petsc
-#endif
 use Sparse_tools
 use Petsc_Tools
 use sparse_tools_petsc
@@ -412,8 +410,7 @@ subroutine SetupSmoothedAggregation(prec, matrix, ierror, &
         end do
         deallocate(matrices, prolongators, contexts)
         ! Need to set n/o levels (to 1) otherwise PCDestroy will fail:
-        ! See note below about PETSC_NULL_KSP argument
-        call PCMGSetLevels(prec, 1, PETSC_NULL_KSP, ierr)
+        call PCMGSetLevels_nocomms(prec, 1, ierr)
         ierror=1
         return
       end if
@@ -435,10 +432,9 @@ subroutine SetupSmoothedAggregation(prec, matrix, ierror, &
       nolevels=i
     end if
 
-    ! NOTE: in petsc v3.8 it's unclear what the legal null argument should be for MPI_Comm *comms
-    ! it does not accept any scalar null object (e.g. PETSC_NULL_INTEGER) - luckily there's no
-    ! explicit interface so we can pass this instead which does get correctly translated to a null argument
-    call PCMGSetLevels(prec, nolevels, PETSC_NULL_KSP, ierr)
+    ! NOTE: we use a wrapper around PCMGLEVELS defined in femtools/ISCopyIndices.cpp
+    ! to deal with bug in https://gitlab.com/petsc/petsc/-/issues/868
+    call PCMGSetLevels_nocomms(prec, nolevels, ierr)
 
     if (lno_top_smoothing) then
       top_level=nolevels-2