Update nektar in NESO to support implicit HW (#240)

* Update NESO-Spack, nektar submodules to incorporate nektar changes for implicit HW.

* Use nektar@master

* Tweaks to make SimpleSOL/H3LAPD solvers, tests work with nektar5.5.

* Workaround/pyr mappings (#241)

* Replaced non-linear mesh with linear mesh. Added workaround for inconsistent mappings in Nektar++

* added test that particle is contained within [-1,1]^3 for mapping tests

* replaced mesh for a linear one in projection order and advection integration tests

* fixed 2D halo test issue based on tolerances

* Fix SimpleSOL.2Drot45 integration test.

---------

Co-authored-by: Will Saunders <[email protected]>

include/nektar_interface/particle_mesh_interface.hpp

@@ -251,7 +251,10 @@ inline void bounding_box_claim(int element_id, T element,
     const INT index_global = cell_volume.first;
     const double volume = cell_volume.second;
     const double ratio = volume * inverse_cell_volume;
-    const int weight = 1000000.0 * ratio;
+    int weight = 1000000.0 * ratio;
+    if ((volume > 0) && (weight == 0)) {
+      weight++;
+    }
     local_claim.claim(index_global, weight, ratio);
     mh_geom_map[index_global].push_back(element_id);
   }

solvers/H3LAPD/ParticleSystems/NeutralParticleSystem.hpp

@@ -250,7 +250,8 @@ class NeutralParticleSystem {
     std::vector<int> components = {0};
     m_field_project->project(syms, components);
     if (m_low_order_project) {
-      FieldUtils::Interpolator interpolator{};
+      FieldUtils::Interpolator<std::vector<MultiRegions::ExpListSharedPtr>>
+          interpolator{};
       std::vector<MultiRegions::ExpListSharedPtr> in_exp = {
           m_fields["ne_src_interp"]};
       std::vector<MultiRegions::ExpListSharedPtr> out_exp = {

spack.yaml

@@ -5,23 +5,23 @@
 spack:
   # add package specs to the `specs` list
   specs:
-  - neso%oneapi ^nektar%oneapi ^intel-oneapi-mpi ^intel-oneapi-mkl ^py-numpy%intel
-    ^boost%intel ^py-cython%oneapi ^dpcpp ^scotch@6
-  - neso%gcc ^openblas ^hipsycl ^scotch@6
+    - neso%oneapi ^nektar%oneapi ^intel-oneapi-mpi ^intel-oneapi-mkl ^py-numpy%intel
+      ^boost%intel ^py-cython%oneapi ^dpcpp ^scotch@6
+    - neso%gcc ^openblas ^hipsycl ^scotch@6
   view:
     gcc-hipsycl:
       root: views/gcc-hipsycl
-      select: ['%gcc']
+      select: ["%gcc"]
       link_type: symlink
     oneapi-dpcpp:
       root: views/oneapi-dpcpp
-      select: ['%oneapi', '%intel']
+      select: ["%oneapi", "%intel"]
       link_type: symlink
   concretizer:
     unify: when_possible
   repos:
-  - ./NESO-Spack
-  - $spack/var/spack/repos/builtin
+    - ./NESO-Spack
+    - $spack/var/spack/repos/builtin
   packages:
     all:
       providers:
@@ -31,6 +31,6 @@ spack:
       path: .
       spec: neso@working
     nektar:
-      spec: nektar@5.3.0-2022-09-03
+      spec: nektar@master
     neso-particles:
       spec: neso-particles@working

test/integration/nektar_interface/test_function_projection_order_3d.cpp

@@ -179,8 +179,8 @@ static inline void projection_wrapper_order_3d(std::string condtions_file_s,
   const int Nsample = 4;
 
   std::vector<int> Nparticles;
-  Nparticles.push_back(400000);
-  Nparticles.push_back(1600000);
+  Nparticles.push_back(200000);
+  Nparticles.push_back(800000);
 
   std::map<int, std::vector<double>> R_errors;
 
@@ -209,21 +209,21 @@ static inline void projection_wrapper_order_3d(std::string condtions_file_s,
 
 TEST(ParticleFunctionProjectionOrder3D, DisContFieldHex) {
   projection_wrapper_order_3d<MultiRegions::DisContField>(
-      "reference_hex_cube/conditions_nummodes_2.xml",
-      "reference_hex_cube/hex_cube_0.3_perturbed.xml");
+      "reference_hex_cube/conditions_nummodes_4.xml",
+      "reference_hex_cube/hex_cube_0.5.xml");
 }
 TEST(ParticleFunctionProjectionOrder3D, DisContFieldPrismTet) {
   projection_wrapper_order_3d<MultiRegions::DisContField>(
       "reference_prism_tet_cube/conditions_nummodes_4.xml",
-      "reference_prism_tet_cube/prism_tet_cube_0.5_perturbed.xml");
+      "reference_prism_tet_cube/prism_tet_cube_0.5.xml");
 }
 TEST(ParticleFunctionProjectionOrder3D, ContFieldHex) {
   projection_wrapper_order_3d<MultiRegions::DisContField>(
-      "reference_hex_cube/conditions_cg_nummodes_2.xml",
-      "reference_hex_cube/hex_cube_0.3_perturbed.xml");
+      "reference_hex_cube/conditions_cg_nummodes_4.xml",
+      "reference_hex_cube/hex_cube_0.5.xml");
 }
 TEST(ParticleFunctionProjectionOrder3D, ContFieldPrismTet) {
   projection_wrapper_order_3d<MultiRegions::DisContField>(
       "reference_prism_tet_cube/conditions_cg_nummodes_4.xml",
-      "reference_prism_tet_cube/prism_tet_cube_0.5_perturbed.xml");
+      "reference_prism_tet_cube/prism_tet_cube_0.5.xml");
 }

test/integration/nektar_interface/test_particle_advection.cpp

@@ -153,26 +153,34 @@ TEST(ParticleGeometryInterface, Advection2D) {
   auto lambda_check_owning_cell = [&] {
     Array<OneD, NekDouble> global_coord(3);
     Array<OneD, NekDouble> local_coord(3);
+    Array<OneD, NekDouble> eta(3);
     for (int cellx = 0; cellx < cell_count; cellx++) {
 
       auto positions = A->position_dat->cell_dat.get_cell(cellx);
       auto cell_ids = A->cell_id_dat->cell_dat.get_cell(cellx);
+      auto reference_positions =
+          A->get_cell(Sym<REAL>("NESO_REFERENCE_POSITIONS"), cellx);
 
       for (int rowx = 0; rowx < cell_ids->nrow; rowx++) {
 
         const int cell_neso = (*cell_ids)[0][rowx];
         ASSERT_EQ(cell_neso, cellx);
         const int cell_nektar = cell_id_translation.map_to_nektar[cell_neso];
 
-        global_coord[0] = (*positions)[0][rowx];
-        global_coord[1] = (*positions)[1][rowx];
-
-        NekDouble dist;
         auto geom = graph->GetGeometry2D(cell_nektar);
-        auto is_contained =
-            geom->ContainsPoint(global_coord, local_coord, tol, dist);
-
-        ASSERT_TRUE(is_contained);
+        local_coord[0] = reference_positions->at(rowx, 0);
+        local_coord[1] = reference_positions->at(rowx, 1);
+        global_coord[0] = geom->GetCoord(0, local_coord);
+        global_coord[1] = geom->GetCoord(1, local_coord);
+
+        geom->GetXmap()->LocCoordToLocCollapsed(local_coord, eta);
+        // check the global coordinate matches the one on the particle
+        for (int dimx = 0; dimx < ndim; dimx++) {
+          const double err_abs =
+              ABS(positions->at(rowx, dimx) - global_coord[dimx]);
+          ASSERT_TRUE(err_abs <= tol);
+          ASSERT_TRUE(std::fabs((double)eta[dimx]) < (1.0 + tol));
+        }
       }
     }
   };
@@ -347,27 +355,36 @@ TEST_P(ParticleAdvection3D, Advection3D) {
   auto lambda_check_owning_cell = [&] {
     Array<OneD, NekDouble> global_coord(3);
     Array<OneD, NekDouble> local_coord(3);
+    Array<OneD, NekDouble> eta(3);
     for (int cellx = 0; cellx < cell_count; cellx++) {
 
       auto positions = A->position_dat->cell_dat.get_cell(cellx);
       auto cell_ids = A->cell_id_dat->cell_dat.get_cell(cellx);
+      auto reference_positions =
+          A->get_cell(Sym<REAL>("NESO_REFERENCE_POSITIONS"), cellx);
 
       for (int rowx = 0; rowx < cell_ids->nrow; rowx++) {
 
         const int cell_neso = (*cell_ids)[0][rowx];
         ASSERT_EQ(cell_neso, cellx);
         const int cell_nektar = cell_id_translation.map_to_nektar[cell_neso];
-
-        global_coord[0] = (*positions)[0][rowx];
-        global_coord[1] = (*positions)[1][rowx];
-        global_coord[2] = (*positions)[2][rowx];
-
-        NekDouble dist;
         auto geom = geoms_3d[cell_nektar];
-        auto is_contained =
-            geom->ContainsPoint(global_coord, local_coord, tol, dist);
-
-        ASSERT_TRUE(is_contained || (dist < (tol * 10.0)));
+        local_coord[0] = reference_positions->at(rowx, 0);
+        local_coord[1] = reference_positions->at(rowx, 1);
+        local_coord[2] = reference_positions->at(rowx, 2);
+        global_coord[0] = geom->GetCoord(0, local_coord);
+        global_coord[1] = geom->GetCoord(1, local_coord);
+        global_coord[2] = geom->GetCoord(2, local_coord);
+
+        geom->GetXmap()->LocCoordToLocCollapsed(local_coord, eta);
+
+        // check the global coordinate matches the one on the particle
+        for (int dimx = 0; dimx < ndim; dimx++) {
+          const double err_abs =
+              ABS(positions->at(rowx, dimx) - global_coord[dimx]);
+          ASSERT_TRUE(err_abs <= tol);
+          ASSERT_TRUE(std::fabs((double)eta[dimx]) < (1.0 + tol));
+        }
       }
     }
   };
@@ -402,14 +419,14 @@ TEST_P(ParticleAdvection3D, Advection3D) {
 
 INSTANTIATE_TEST_SUITE_P(
     MultipleMeshes, ParticleAdvection3D,
-    testing::Values(
-        std::tuple<std::string, std::string, double>(
-            "reference_all_types_cube/conditions.xml",
-            "reference_all_types_cube/mixed_ref_cube_0.5_perturbed.xml",
-            1.0e-4 // The non-linear exit tolerance in Nektar is like (err_x *
-                   // err_x
-                   // + err_y * err_y) < 1.0e-8
-            ),
-        std::tuple<std::string, std::string, double>(
-            "reference_all_types_cube/conditions.xml",
-            "reference_all_types_cube/mixed_ref_cube_0.5.xml", 1.0e-10)));
+    testing::Values(std::tuple<std::string, std::string, double>(
+                        "reference_all_types_cube/conditions.xml",
+                        "reference_all_types_cube/linear_non_regular_0.5.xml",
+                        1.0e-4 // The non-linear exit tolerance in Nektar is
+                               // like (err_x * err_x
+                               // + err_y * err_y) < 1.0e-8
+                        ),
+                    std::tuple<std::string, std::string, double>(
+                        "reference_all_types_cube/conditions.xml",
+                        "reference_all_types_cube/mixed_ref_cube_0.5.xml",
+                        1.0e-10)));

test/integration/solvers/SimpleSOL/test_SimpleSOL.hpp

@@ -61,11 +61,19 @@ class SimpleSOLTest : public NektarSolverTest {
   }
 
   std::string get_interp_str(double theta) {
-    // Assume fixed s_max = 110, n_pts = 1101 (to match analytic data)
+    // Fix s_max = 110, n_pts = 1101 to match analytic data
+    int constexpr n_pts = 1101;
+    double constexpr s_max = 110.0;
+    /* Move interp line away from the boundary by a small amount, otherwise the
+     * first point evaluates to zero
+     */
+    double epsilon = 1e-12;
+    double x_min = 0.0;
+    double y_min = 0.0 + epsilon;
+    double x_max = s_max * cos(theta) - epsilon;
+    double y_max = s_max * sin(theta);
     std::stringstream ss;
-    double x_max = 110.0 * cos(theta);
-    double y_max = 110.0 * sin(theta);
-    ss << "1101,0,0," << x_max << "," << y_max;
+    ss << n_pts << "," << x_min << "," << y_min << "," << x_max << "," << y_max;
     return ss.str();
   }