Fix "mg" preconditioner with petsc 3.14

This is dealing with bug https://gitlab.com/petsc/petsc/-/issues/868
Which is fixed for 3.15. Unfortunately no workaround exists in fortran,
because even if we do provide the optional argument it still gets
recognized for NULL and hits the bug. So instead write our own wrapper
in C.

This also drop support for petsc version <3.8 Note that Bionic official
packages are 3.7 - but we have always supplied our own 3.8 package for
it. Groovy and Hirsute are now on 3.14 Dropping support for <3.8
simplifies/removes a lot of the legacy workarounds. We no longer support
linking with petsc without fortran modules (we never did for versions
>=3.8).

Author: stephankramer

aclocal.m4

@@ -439,9 +439,9 @@ AC_COMPUTE_INT(PETSC_VERSION_MAJOR, "PETSC_VERSION_MAJOR", [#include "petscversi
 AC_COMPUTE_INT(PETSC_VERSION_MINOR, "PETSC_VERSION_MINOR", [#include "petscversion.h"], 
   [AC_MSG_ERROR([Unknown petsc minor version])])
 AC_MSG_NOTICE([Detected PETSc version "$PETSC_VERSION_MAJOR"."$PETSC_VERSION_MINOR"])
-# if major<3 or minor<4
-if test "0$PETSC_VERSION_MAJOR" -lt 3 -o "0$PETSC_VERSION_MINOR" -lt 4; then
-  AC_MSG_ERROR([Fluidity needs PETSc version >=3.4])
+# if major<3 or minor<8
+if test "0$PETSC_VERSION_MAJOR" -lt 3 -o "0$PETSC_VERSION_MINOR" -lt 8; then
+  AC_MSG_ERROR([Fluidity needs PETSc version >=3.8])
 fi
 
 AC_LANG(Fortran)
@@ -451,35 +451,27 @@ ac_ext=F90
 # may need some extra flags for testing that we don't want to use in the end
 SAVE_FCFLAGS="$FCFLAGS"
 
-if test "$enable_petsc_fortran_modules" != "no" ; then
-  # now try if the petsc fortran modules work:
-  AC_LINK_IFELSE(
-          [AC_LANG_PROGRAM([],[[
-                          use petsc
-                          integer :: ierr
-                          print*, "hello petsc"
-                          call PetscInitialize(PETSC_NULL_CHARACTER, ierr)
-                          ]])],
-          [
-              AC_MSG_NOTICE([PETSc modules are working.])
-              AC_DEFINE(HAVE_PETSC_MODULES,1,[Define if you have petsc fortran modules.] )
-              FCFLAGS="$FCFLAGS -DHAVE_PETSC_MODULES"
-          ],
-          [
-              AC_MSG_NOTICE([PETSc modules don't work, using headers instead.])
-          ])
-elif test "0$PETSC_VERSION_MINOR" -ge 8; then
-  AC_MSG_ERROR([PETSc v3.8 and newer requires petsc fortran modules. Do not use the --disable-petsc-fortran-modules option])
-fi
+# now try if the petsc fortran modules work:
+AC_LINK_IFELSE(
+        [AC_LANG_PROGRAM([],[[
+                        use petsc
+                        integer :: ierr
+                        print*, "hello petsc"
+                        call PetscInitialize(PETSC_NULL_CHARACTER, ierr)
+                        ]])],
+        [
+            AC_MSG_NOTICE([PETSc modules are working.])
+        ],
+        [
+            AC_MSG_ERROR([PETSc fortran modules cannot be used. Either it the petsc*.mod files cannot be found, or petsc has been built using a different fortran compiler.])
+        ])
 
 # now try a more realistic program, it's a stripped down
 # petsc tutorial - using the headers in the same way as we do in the code
 AC_LINK_IFELSE(
 [AC_LANG_SOURCE([
 program test_petsc
-#ifdef HAVE_PETSC_MODULES
   use petsc
-#endif
 implicit none
 #include "petsc_legacy.h"
       double precision  norm

assemble/Full_Projection.F90

@@ -31,9 +31,7 @@ module Full_Projection
     use global_parameters
     use elements
     use spud
-#ifdef HAVE_PETSC_MODULES
     use petsc
-#endif
     use parallel_tools
     use data_structures
     use sparse_tools
@@ -359,11 +357,7 @@ subroutine petsc_solve_setup_full_projection(y,A,b,ksp,petsc_numbering_p,name,so
       else
          ! workaround bug in petsc 3.8: missing CHKFORTRANNULLOBJECT, so PETSC_NULL_MAT isn't translated to C null
          ! this however seems to do the trick:
-#if PETSC_VERSION_MINOR>=8
          S%v = 0
-#else
-         S = PETSC_NULL_OBJECT
-#endif
          call MatCreateSchurComplement(inner_M%M,inner_M%M,G,G_t_comp,S,A,ierr)
       end if
 

assemble/tests/test_matrix_free.F90

@@ -236,9 +236,7 @@ subroutine petsc_solve_matrix_free(x, matrix, rhs)
   use petsc_tools
   use matrix_free_solvers
 
-#ifdef HAVE_PETSC_MODULES
   use petsc 
-#endif
   implicit none
 #include "petsc_legacy.h"
   type(scalar_field), intent(inout) :: x

assemble/tests/test_pressure_solve.F90

@@ -15,9 +15,7 @@ subroutine test_pressure_solve
     use boundary_conditions
     use global_parameters, only: OPTION_PATH_LEN
     use free_surface_module
-#ifdef HAVE_PETSC_MODULES
-  use petsc 
-#endif
+    use petsc
     implicit none
 #include "petsc_legacy.h"
 

configure

@@ -785,7 +785,6 @@ with_femdem
 with_psmile
 with_netcdf
 with_exodusii
-enable_petsc_fortran_modules
 with_x
 enable_sam
 enable_dp
@@ -1463,12 +1462,6 @@ Optional Features:
   --enable-shared         Compile objects with -fPIC to enable the 'make
                           shared' target.
   --enable-verbose        turns on super verbosity
-  --disable-petsc-fortran-modules
-                          By default fluidity tries to use fortran modules
-                          provided with petsc. Use
-                          --disable-petsc-fortran-modules if these are known
-                          to be not functional (e.g. compiled with different
-                          compiler version)
   --enable-sam            use sam rather than zoltan for adaptive load
                           rebalancing
   --enable-dp=flag        compile with 64 bit floating point numbers (default)
@@ -6001,7 +5994,7 @@ then
     if test -z "$PYTHON_VERSION"; then
       # if PYTHON_VERSION is set we use python$PYTHON_VERSION as the python interpreter
       # if not, if there is no `python` in the path, or there is a `python` but its version is <3
-      # then use `python3` (by setting PETSC_VERSION=3).
+      # then use `python3` (by setting PYTHON_VERSION=3).
       # If there *is* a `python` of version 3, we use that
       # Extract the first word of "python", so it can be a program name with args.
 set dummy python; ac_word=$2
@@ -11791,11 +11784,6 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu
 
 
 
-fi
-
-# Check whether --enable-petsc-fortran-modules was given.
-if test "${enable_petsc_fortran_modules+set}" = set; then :
-  enableval=$enable_petsc_fortran_modules;
 fi
 
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for X" >&5
@@ -12724,9 +12712,9 @@ fi
 
 { $as_echo "$as_me:${as_lineno-$LINENO}: Detected PETSc version \"$PETSC_VERSION_MAJOR\".\"$PETSC_VERSION_MINOR\"" >&5
 $as_echo "$as_me: Detected PETSc version \"$PETSC_VERSION_MAJOR\".\"$PETSC_VERSION_MINOR\"" >&6;}
-# if major<3 or minor<4
-if test "0$PETSC_VERSION_MAJOR" -lt 3 -o "0$PETSC_VERSION_MINOR" -lt 4; then
-  as_fn_error $? "Fluidity needs PETSc version >=3.4" "$LINENO" 5
+# if major<3 or minor<8
+if test "0$PETSC_VERSION_MAJOR" -lt 3 -o "0$PETSC_VERSION_MINOR" -lt 8; then
+  as_fn_error $? "Fluidity needs PETSc version >=3.8" "$LINENO" 5
 fi
 
 ac_ext=${ac_fc_srcext-f}
@@ -12740,47 +12728,36 @@ ac_ext=F90
 # may need some extra flags for testing that we don't want to use in the end
 SAVE_FCFLAGS="$FCFLAGS"
 
-if test "$enable_petsc_fortran_modules" != "no" ; then
-  # now try if the petsc fortran modules work:
-  cat > conftest.$ac_ext <<_ACEOF
+# now try if the petsc fortran modules work:
+cat > conftest.$ac_ext <<_ACEOF
       program main
 
-                          use petsc
-                          integer :: ierr
-                          print*, "hello petsc"
-                          call PetscInitialize(PETSC_NULL_CHARACTER, ierr)
+                        use petsc
+                        integer :: ierr
+                        print*, "hello petsc"
+                        call PetscInitialize(PETSC_NULL_CHARACTER, ierr)
 
       end
 _ACEOF
 if ac_fn_fc_try_link "$LINENO"; then :
 
-              { $as_echo "$as_me:${as_lineno-$LINENO}: PETSc modules are working." >&5
+            { $as_echo "$as_me:${as_lineno-$LINENO}: PETSc modules are working." >&5
 $as_echo "$as_me: PETSc modules are working." >&6;}
 
-$as_echo "#define HAVE_PETSC_MODULES 1" >>confdefs.h
-
-              FCFLAGS="$FCFLAGS -DHAVE_PETSC_MODULES"
-
 else
 
-              { $as_echo "$as_me:${as_lineno-$LINENO}: PETSc modules don't work, using headers instead." >&5
-$as_echo "$as_me: PETSc modules don't work, using headers instead." >&6;}
+            as_fn_error $? "PETSc fortran modules cannot be used. Either it the petsc*.mod files cannot be found, or petsc has been built using a different fortran compiler." "$LINENO" 5
 
 fi
 rm -f core conftest.err conftest.$ac_objext \
     conftest$ac_exeext conftest.$ac_ext
-elif test "0$PETSC_VERSION_MINOR" -ge 8; then
-  as_fn_error $? "PETSc v3.8 and newer requires petsc fortran modules. Do not use the --disable-petsc-fortran-modules option" "$LINENO" 5
-fi
 
 # now try a more realistic program, it's a stripped down
 # petsc tutorial - using the headers in the same way as we do in the code
 cat > conftest.$ac_ext <<_ACEOF
 
 program test_petsc
-#ifdef HAVE_PETSC_MODULES
   use petsc
-#endif
 implicit none
 #include "petsc_legacy.h"
       double precision  norm

configure.in

@@ -193,7 +193,7 @@ then
     if test -z "$PYTHON_VERSION"; then
       # if PYTHON_VERSION is set we use python$PYTHON_VERSION as the python interpreter
       # if not, if there is no `python` in the path, or there is a `python` but its version is <3
-      # then use `python3` (by setting PETSC_VERSION=3).
+      # then use `python3` (by setting PYTHON_VERSION=3).
       # If there *is* a `python` of version 3, we use that
       AC_CHECK_PROG(have_bare_python, python, "yes")
       if test "x$have_bare_python" = "xyes"; then
@@ -828,9 +828,6 @@ if test "$with_exodusii" = "yes" ; then
                                            ])
 fi
 AC_SUBST(HAVE_EXODUSII)
-AC_ARG_ENABLE(petsc-fortran-modules,
-    [AC_HELP_STRING([--disable-petsc-fortran-modules],
-    [By default fluidity tries to use fortran modules provided with petsc. Use --disable-petsc-fortran-modules if these are known to be not functional (e.g. compiled with different compiler version)])])
 ACX_PETSc([AC_MSG_NOTICE(["PETSc found, enabling scalable solver"])], [AC_MSG_ERROR(["PETSc not found"])])
 ACX_hypre([AC_MSG_NOTICE(["hypre found, enabling hypre preconditioners"])], [AC_MSG_WARN(["hypre not found"])])
 CPPFLAGS="$CPPFLAGS -DHAVE_PETSC"

femtools/ISCopyIndices.cpp

@@ -1,25 +1,18 @@
 #include "confdefs.h"
 #include "petsc.h"
-#if PETSC_VERSION_MINOR==0
-#include "petscfix.h"
-#include "petscis.h"
-#endif
 
 #include <string.h>
 
 extern "C" {
   void iscopyindices_(IS *is, PetscInt *iarray,PetscErrorCode *ierr);
   void petscobjectreference_(PetscObject obj, int *__ierr );
+  void pcmgsetlevels_nocomms_(PC *pc, PetscInt *levels, int *ierr);
 }
 
 // our own version of IsGetIndices, that just does a copy
 // to avoid silly fortran interface issues
 void iscopyindices_(IS *is, PetscInt *iarray,PetscErrorCode *ierr){
-#if PETSC_VERSION_MAJOR==3
   const PetscInt *iptr;
-#else
-  PetscInt *iptr;
-#endif
   PetscInt n;
 
   *ierr = ISGetIndices(*is, &iptr); if (*ierr) return;
@@ -58,3 +51,14 @@ void petscobjectreference_(PetscObject obj, int *__ierr ){
 }
 
 
+// This is a woraround bug in https://gitlab.com/petsc/petsc/-/issues/868
+// In 3.14 not only is it impossible to "fake" a NULL argument using PETSC_NULL_KSP
+// as we'd done previously, even if you do provide a comms filled with MPI_COMM_WORLD
+// that because MPI_COMM_WORLD==0 it is still mistaken for a NULL argument and hits the
+// bug in the petsc fortran wrapper for this routine
+// So instead we just write our own wrapper without the comms argument:
+void pcmgsetlevels_nocomms_(PC *pc, PetscInt *levels, int *ierr){
+  *ierr = PCMGSetLevels(*pc, *levels, NULL);
+}
+
+

femtools/Multigrid.F90

@@ -6,9 +6,7 @@ module multigrid
 use spud
 use futils
 use parallel_tools
-#ifdef HAVE_PETSC_MODULES
   use petsc
-#endif
 use Sparse_tools
 use Petsc_Tools
 use sparse_tools_petsc
@@ -412,8 +410,7 @@ subroutine SetupSmoothedAggregation(prec, matrix, ierror, &
         end do
         deallocate(matrices, prolongators, contexts)
         ! Need to set n/o levels (to 1) otherwise PCDestroy will fail:
-        ! See note below about PETSC_NULL_KSP argument
-        call PCMGSetLevels(prec, 1, PETSC_NULL_KSP, ierr)
+        call PCMGSetLevels_nocomms(prec, 1, ierr)
         ierror=1
         return
       end if
@@ -435,10 +432,9 @@ subroutine SetupSmoothedAggregation(prec, matrix, ierror, &
       nolevels=i
     end if
 
-    ! NOTE: in petsc v3.8 it's unclear what the legal null argument should be for MPI_Comm *comms
-    ! it does not accept any scalar null object (e.g. PETSC_NULL_INTEGER) - luckily there's no
-    ! explicit interface so we can pass this instead which does get correctly translated to a null argument
-    call PCMGSetLevels(prec, nolevels, PETSC_NULL_KSP, ierr)
+    ! NOTE: we use a wrapper around PCMGLEVELS defined in femtools/ISCopyIndices.cpp
+    ! to deal with bug in https://gitlab.com/petsc/petsc/-/issues/868
+    call PCMGSetLevels_nocomms(prec, nolevels, ierr)
 
     if (lno_top_smoothing) then
       top_level=nolevels-2