Merge pull request #668 from andrsd/petsc-3.21

Updates for PETSc 3.21

.github/workflows/build.yml

@@ -26,6 +26,10 @@ jobs:
             python-version: "3.9"
             petsc: 3.20.6
             build: Release
+          - os: ubuntu-22.04
+            python-version: "3.9"
+            petsc: 3.21.6
+            build: Release
     runs-on: ${{ matrix.os }}
     steps:
       - name: Set up miniconda

README.md

@@ -7,7 +7,7 @@ godzilla
 [![github-pages](https://github.com/andrsd/godzilla/actions/workflows/gh-pages.yml/badge.svg)](https://github.com/andrsd/godzilla/actions/workflows/gh-pages.yml)
 [![codecov](https://codecov.io/gh/andrsd/godzilla/branch/main/graph/badge.svg?token=7KL45W9Z4G)](https://codecov.io/gh/andrsd/godzilla)
 [![License](http://img.shields.io/:license-mit-blue.svg)](https://andrsd.mit-license.org/)
-[![PETSc](https://img.shields.io/badge/PETSc-3.20-red)](https://petsc.org/)
+[![PETSc](https://img.shields.io/badge/PETSc-3.20,%203.21-red)](https://petsc.org/)
 [![Scc Count Badge](https://sloc.xyz/github/andrsd/godzilla/)](https://github.com/andrsd/godzilla/)
 
 

examples/advect-eqn/test/CMakeLists.txt

@@ -1,5 +1,7 @@
 include(TesterExodusII)
 
+if(${PETSC_VERSION} VERSION_LESS "3.21.0")
+
 add_test_exodiff(
     NAME   advect
     BIN    ${PROJECT_NAME}
@@ -8,3 +10,5 @@ add_test_exodiff(
     GOLD   ${PROJECT_SOURCE_DIR}/test/gold/1d.exo
     ARGS   -Floor 1e-16
 )
+
+endif()

include/godzilla/Mesh.h

@@ -93,6 +93,9 @@ class Mesh {
     /// @param c coordinate vector
     void set_coordinates_local(const Vector & c);
 
+    /// If a mesh is periodic, create local coordinates for cells having periodic faces
+    void localize_coordinates() const;
+
     /// Sets up the data structures inside the `DM` object
     void set_up();
 

include/godzilla/Partitioner.h

@@ -33,6 +33,15 @@ class Partitioner {
                    const Section & target_section,
                    Section & partSection,
                    IndexSet & partition);
+    void partition(Int n_parts,
+                   Int n_vertices,
+                   Int start[],
+                   Int adjacency[],
+                   const Section & vertex_section,
+                   const Section & edge_section,
+                   const Section & target_section,
+                   Section & partSection,
+                   IndexSet & partition);
 
     void view(PetscViewer viewer = PETSC_VIEWER_STDOUT_WORLD) const;
 

include/godzilla/Problem.h

@@ -228,6 +228,17 @@ class Problem : public Object, public PrintInterface {
     /// @return The global indices for the subproblem
     IndexSet create_section_subis(const std::vector<Int> & fields) const;
 
+    /// Returns an IndexSet containing a Section that encapsulates a subproblem defined by
+    /// a subset of the fields and components in a Section in the problem.
+    ///
+    /// @param fields The field numbers of the selected fields
+    /// @param n_comps The number of components from each field to incorporate into the subproblem
+    /// @param comps The component numbers of the selected fields
+    /// @return The global indices for the subproblem
+    IndexSet create_section_subis(const std::vector<Int> & fields,
+                                  const std::vector<Int> & n_comps,
+                                  const std::vector<Int> & comps) const;
+
 protected:
     /// Set vector/matrix types
     virtual void set_up_types();

src/DGProblemInterface.cpp

@@ -49,6 +49,8 @@ DGProblemInterface::init()
 
         PETSC_CHECK(DMCopyDisc(dm, cdm));
         PETSC_CHECK(DMGetCoarseDM(cdm, &cdm));
+        if (cdm)
+            PETSC_CHECK(DMLocalizeCoordinates(cdm));
     }
 }
 
@@ -395,6 +397,7 @@ DGProblemInterface::create()
     CALL_STACK_MSG();
     set_up_fields();
     DiscreteProblemInterface::create();
+    get_unstr_mesh()->localize_coordinates();
 }
 
 void

src/FEProblemInterface.cpp

@@ -89,6 +89,7 @@ FEProblemInterface::create()
     this->asmbl = new AssemblyData(dim);
     set_up_fields();
     DiscreteProblemInterface::create();
+    get_unstr_mesh()->localize_coordinates();
 }
 
 void
@@ -112,6 +113,8 @@ FEProblemInterface::init()
 
         PETSC_CHECK(DMCopyDisc(dm, cdm));
         PETSC_CHECK(DMGetCoarseDM(cdm, &cdm));
+        if (cdm)
+            PETSC_CHECK(DMLocalizeCoordinates(cdm));
     }
 
     set_up_assembly_data_aux();

src/FVProblemInterface.cpp

@@ -76,6 +76,8 @@ FVProblemInterface::init()
 
         PETSC_CHECK(DMCopyDisc(dm, cdm));
         PETSC_CHECK(DMGetCoarseDM(cdm, &cdm));
+        if (cdm)
+            PETSC_CHECK(DMLocalizeCoordinates(cdm));
     }
 }
 
@@ -364,6 +366,12 @@ FVProblemInterface::create()
     set_up_fields();
     DiscreteProblemInterface::create();
     get_unstr_mesh()->construct_ghost_cells();
+    get_unstr_mesh()->localize_coordinates();
+
+#if PETSC_VERSION_GE(3, 21, 0)
+    if (get_unstr_mesh()->get_dimension() == 1)
+        throw Exception("FV in 1D is not possible due to a bug in PETSc. Use PETSc 3.20 instead.");
+#endif
 }
 
 void

src/Mesh.cpp

@@ -132,6 +132,13 @@ Mesh::set_coordinates_local(const Vector & c)
     PETSC_CHECK(DMSetCoordinatesLocal(this->dm, c));
 }
 
+void
+Mesh::localize_coordinates() const
+{
+    CALL_STACK_MSG();
+    PETSC_CHECK(DMLocalizeCoordinates(this->dm));
+}
+
 void
 Mesh::set_up()
 {

src/Partitioner.cpp

@@ -4,6 +4,7 @@
 #include "godzilla/Partitioner.h"
 #include "godzilla/CallStack.h"
 #include "godzilla/Error.h"
+#include "godzilla/Exception.h"
 
 using namespace godzilla;
 
@@ -87,15 +88,56 @@ Partitioner::partition(Int n_parts,
                        IndexSet & partition)
 {
     CALL_STACK_MSG();
+#if PETSC_VERSION_GE(3, 21, 0)
     PETSC_CHECK(PetscPartitionerPartition(this->part,
                                           n_parts,
                                           n_vertices,
                                           start,
                                           adjacency,
                                           vertex_section,
+                                          NULL,
                                           target_section,
                                           part_section,
                                           partition));
+#else
+    PETSC_CHECK(PetscPartitionerPartition(this->part,
+                                          n_parts,
+                                          n_vertices,
+                                          start,
+                                          adjacency,
+                                          vertex_section,
+                                          target_section,
+                                          part_section,
+                                          partition));
+#endif
+}
+
+void
+Partitioner::partition(Int n_parts,
+                       Int n_vertices,
+                       Int start[],
+                       Int adjacency[],
+                       const Section & vertex_section,
+                       const Section & edge_section,
+                       const Section & target_section,
+                       Section & part_section,
+                       IndexSet & partition)
+{
+    CALL_STACK_MSG();
+#if PETSC_VERSION_GE(3, 21, 0)
+    PETSC_CHECK(PetscPartitionerPartition(this->part,
+                                          n_parts,
+                                          n_vertices,
+                                          start,
+                                          adjacency,
+                                          vertex_section,
+                                          edge_section,
+                                          target_section,
+                                          part_section,
+                                          partition));
+#else
+    throw Exception("PETSc 3.21+ is needed for Partitioner::partition with edge_section parameter");
+#endif
 }
 
 Partitioner::operator PetscPartitioner() const

src/Problem.cpp

@@ -437,19 +437,37 @@ Problem::create_section_subis(const std::vector<Int> & fields) const
 {
     CALL_STACK_MSG();
 #if PETSC_VERSION_GE(3, 21, 0)
+    IS is;
+    PETSC_CHECK(
+        DMCreateSectionSubDM(get_dm(), fields.size(), fields.data(), NULL, NULL, &is, NULL));
+    return IndexSet(is);
+#else
+    IS is;
+    PETSC_CHECK(DMCreateSectionSubDM(get_dm(), fields.size(), fields.data(), &is, NULL));
+    return IndexSet(is);
+#endif
+}
+
+IndexSet
+Problem::create_section_subis(const std::vector<Int> & fields,
+                              const std::vector<Int> & n_comps,
+                              const std::vector<Int> & comps) const
+{
+    CALL_STACK_MSG();
+#if PETSC_VERSION_GE(3, 21, 0)
+    assert(fields.size() == n_comps.size());
     IS is;
     PETSC_CHECK(DMCreateSectionSubDM(get_dm(),
                                      fields.size(),
                                      fields.data(),
-                                     PETSC_DECIDE,
-                                     nullptr,
+                                     n_comps.data(),
+                                     comps.data(),
                                      &is,
-                                     nullptr));
+                                     NULL));
     return IndexSet(is);
 #else
-    IS is;
-    PETSC_CHECK(DMCreateSectionSubDM(get_dm(), fields.size(), fields.data(), &is, nullptr));
-    return IndexSet(is);
+    throw Exception(
+        "PETSc 3.21+ is needed for Problem::create_section_subis with component support");
 #endif
 }
 

test/src/ExplicitFVLinearProblem_test.cpp

@@ -1,6 +1,7 @@
 #include "gmock/gmock.h"
 #include "godzilla/CallStack.h"
 #include "godzilla/LineMesh.h"
+#include "godzilla/RectangleMesh.h"
 #include "godzilla/NaturalRiemannBC.h"
 #include "godzilla/ExplicitFVLinearProblem.h"
 #include "godzilla/Parameters.h"
@@ -112,10 +113,19 @@ TEST(ExplicitFVLinearProblemTest, api)
 {
     TestApp app;
 
+#if PETSC_VERSION_GE(3, 21, 0)
+    // PETSc 3.21.0+ has a bug in forming the mass matrix in 1D, se we use 2D mesh in this test
+    Parameters mesh_pars = RectangleMesh::parameters();
+    mesh_pars.set<App *>("_app") = &app;
+    mesh_pars.set<Int>("nx") = 2;
+    mesh_pars.set<Int>("ny") = 1;
+    RectangleMesh mesh(mesh_pars);
+#else
     Parameters mesh_pars = LineMesh::parameters();
     mesh_pars.set<App *>("_app") = &app;
     mesh_pars.set<Int>("nx") = 2;
     LineMesh mesh(mesh_pars);
+#endif
 
     Parameters prob_pars = TestExplicitFVLinearProblem::parameters();
     prob_pars.set<App *>("_app") = &app;
@@ -201,10 +211,19 @@ TEST(ExplicitFVLinearProblemTest, fields)
 {
     TestApp app;
 
+#if PETSC_VERSION_GE(3, 21, 0)
+    // PETSc 3.21.0+ has a bug in forming the mass matrix in 1D, se we use 2D mesh in this test
+    Parameters mesh_pars = RectangleMesh::parameters();
+    mesh_pars.set<App *>("_app") = &app;
+    mesh_pars.set<Int>("nx") = 2;
+    mesh_pars.set<Int>("ny") = 1;
+    RectangleMesh mesh(mesh_pars);
+#else
     Parameters mesh_pars = LineMesh::parameters();
     mesh_pars.set<App *>("_app") = &app;
     mesh_pars.set<Int>("nx") = 2;
     LineMesh mesh(mesh_pars);
+#endif
 
     Parameters prob_pars = TestExplicitFVLinearProblem::parameters();
     prob_pars.set<App *>("_app") = &app;
@@ -244,10 +263,20 @@ TEST(ExplicitFVLinearProblemTest, test_mass_matrix)
 {
     TestApp app;
 
+#if PETSC_VERSION_GE(3, 21, 0)
+    // PETSc 3.21.0+ has a bug in forming the mass matrix in 1D, se we use 2D mesh in this test
+    Parameters mesh_pars = RectangleMesh::parameters();
+    mesh_pars.set<App *>("_app") = &app;
+    mesh_pars.set<Int>("nx") = 3;
+    mesh_pars.set<Int>("ny") = 1;
+    mesh_pars.set<Real>("ymax") = 3.;
+    RectangleMesh mesh(mesh_pars);
+#else
     Parameters mesh_pars = LineMesh::parameters();
     mesh_pars.set<App *>("_app") = &app;
     mesh_pars.set<Int>("nx") = 3;
     LineMesh mesh(mesh_pars);
+#endif
 
     Parameters prob_pars = TestExplicitFVLinearProblem::parameters();
     prob_pars.set<App *>("_app") = &app;
@@ -276,6 +305,9 @@ TEST(ExplicitFVLinearProblemTest, solve)
 {
     TestApp app;
 
+#if PETSC_VERSION_GE(3, 21, 0)
+    // PETSc 3.21.0+ has a bug in forming the mass matrix in 1D, se we use 2D mesh in this test
+#else
     Parameters mesh_pars = LineMesh::parameters();
     mesh_pars.set<App *>("_app") = &app;
     mesh_pars.set<Int>("nx") = 2;
@@ -330,16 +362,26 @@ TEST(ExplicitFVLinearProblemTest, solve)
     EXPECT_NEAR(lx[2], 0., 1e-15);
     EXPECT_NEAR(lx[3], 0., 1e-15);
     loc_sln.restore_array_read(lx);
+#endif
 }
 
 TEST(ExplicitFVLinearProblemTest, set_schemes)
 {
     TestApp app;
 
+#if PETSC_VERSION_GE(3, 21, 0)
+    // PETSc 3.21.0+ has a bug in forming the mass matrix in 1D, se we use 2D mesh in this test
+    Parameters mesh_pars = RectangleMesh::parameters();
+    mesh_pars.set<App *>("_app") = &app;
+    mesh_pars.set<Int>("nx") = 2;
+    mesh_pars.set<Int>("ny") = 1;
+    RectangleMesh mesh(mesh_pars);
+#else
     Parameters mesh_pars = LineMesh::parameters();
     mesh_pars.set<App *>("_app") = &app;
     mesh_pars.set<Int>("nx") = 2;
     LineMesh mesh(mesh_pars);
+#endif
 
     Parameters prob_pars = TestExplicitFVLinearProblem::parameters();
     prob_pars.set<App *>("_app") = &app;
@@ -373,10 +415,19 @@ TEST(ExplicitFVLinearProblemTest, wrong_schemes)
 
     TestApp app;
 
+#if PETSC_VERSION_GE(3, 21, 0)
+    // PETSc 3.21.0+ has a bug in forming the mass matrix in 1D, se we use 2D mesh in this test
+    Parameters mesh_pars = RectangleMesh::parameters();
+    mesh_pars.set<App *>("_app") = &app;
+    mesh_pars.set<Int>("nx") = 2;
+    mesh_pars.set<Int>("ny") = 1;
+    RectangleMesh mesh(mesh_pars);
+#else
     Parameters mesh_pars = LineMesh::parameters();
     mesh_pars.set<App *>("_app") = &app;
     mesh_pars.set<Int>("nx") = 2;
     LineMesh mesh(mesh_pars);
+#endif
 
     Parameters prob_pars = TestExplicitFVLinearProblem::parameters();
     prob_pars.set<App *>("_app") = &app;