diff --git a/samples/lb_circular_couette.py b/samples/lb_circular_couette.py
index 0d915237ec..81a621b2ce 100644
--- a/samples/lb_circular_couette.py
+++ b/samples/lb_circular_couette.py
@@ -57,7 +57,7 @@
cyl_center = agrid * (grid_size // 2 + 0.5) * [1, 1, 0]
cylinder_in = espressomd.shapes.Cylinder(
center=cyl_center, axis=[0, 0, 1], length=3 * system.box_l[2],
- radius=8.1 * agrid, direction=1)
+ radius=8.6 * agrid, direction=1)
cylinder_out = espressomd.shapes.Cylinder(
center=cyl_center, axis=[0, 0, 1], length=3 * system.box_l[2],
radius=14.5 * agrid, direction=-1)
diff --git a/src/script_interface/walberla/LBFluid.cpp b/src/script_interface/walberla/LBFluid.cpp
index 44c5bbcd69..b66f954ef4 100644
--- a/src/script_interface/walberla/LBFluid.cpp
+++ b/src/script_interface/walberla/LBFluid.cpp
@@ -108,7 +108,6 @@ Variant LBFluid::do_call_method(std::string const &name,
}
if (name == "clear_boundaries") {
m_instance->clear_boundaries();
- m_instance->ghost_communication();
::System::get_system().on_lb_boundary_conditions_change();
return {};
}
@@ -269,8 +268,8 @@ void LBFluid::load_checkpoint(std::string const &filename, int mode) {
};
auto const on_success = [&lb_obj]() {
- lb_obj.reallocate_ubb_field();
lb_obj.ghost_communication();
+ lb_obj.reallocate_ubb_field();
};
load_checkpoint_common(*context(), "LB", filename, mode, read_metadata,
diff --git a/src/script_interface/walberla/LBFluidNode.cpp b/src/script_interface/walberla/LBFluidNode.cpp
index af9ff8b678..f30e26a395 100644
--- a/src/script_interface/walberla/LBFluidNode.cpp
+++ b/src/script_interface/walberla/LBFluidNode.cpp
@@ -53,6 +53,7 @@ Variant LBFluidNode::do_call_method(std::string const &name,
m_lb_fluid->set_node_velocity_at_boundary(m_index, u);
}
m_lb_fluid->ghost_communication();
+ m_lb_fluid->reallocate_ubb_field();
return {};
}
if (name == "get_velocity_at_boundary") {
diff --git a/src/script_interface/walberla/LBFluidSlice.cpp b/src/script_interface/walberla/LBFluidSlice.cpp
index 1aee5f49fd..cb6bc905b1 100644
--- a/src/script_interface/walberla/LBFluidSlice.cpp
+++ b/src/script_interface/walberla/LBFluidSlice.cpp
@@ -99,8 +99,10 @@ Variant LBFluidSlice::do_call_method(std::string const &name,
1. / m_conv_velocity);
}
if (name == "set_velocity_at_boundary") {
- return call(&LatticeModel::set_slice_velocity_at_boundary, {1},
- m_conv_velocity);
+ auto const retval = call(&LatticeModel::set_slice_velocity_at_boundary, {1},
+ m_conv_velocity);
+ m_lb_fluid->reallocate_ubb_field();
+ return retval;
}
if (name == "get_pressure_tensor") {
return call(&LatticeModel::get_slice_pressure_tensor, {3, 3},
diff --git a/src/walberla_bridge/src/BoundaryPackInfo.hpp b/src/walberla_bridge/src/BoundaryPackInfo.hpp
new file mode 100644
index 0000000000..3055b3ebe0
--- /dev/null
+++ b/src/walberla_bridge/src/BoundaryPackInfo.hpp
@@ -0,0 +1,147 @@
+/*
+ * Copyright (C) 2024 The ESPResSo project
+ *
+ * This file is part of ESPResSo.
+ *
+ * ESPResSo is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * ESPResSo is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see .
+ */
+
+#pragma once
+
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include
+#include
+#include
+
+namespace walberla {
+namespace field {
+namespace communication {
+
+template
+class BoundaryPackInfo : public PackInfo {
+protected:
+ using PackInfo::bdId_;
+
+public:
+ using PackInfo::PackInfo;
+ using PackInfo::numberOfGhostLayersToCommunicate;
+
+ ~BoundaryPackInfo() override = default;
+
+ void setup_boundary_handle(std::shared_ptr lattice,
+ std::shared_ptr boundary) {
+ m_lattice = std::move(lattice);
+ m_boundary = std::move(boundary);
+ }
+
+ bool constantDataExchange() const override { return false; }
+ bool threadsafeReceiving() const override { return true; }
+
+ void communicateLocal(IBlock const *sender, IBlock *receiver,
+ stencil::Direction dir) override {
+ mpi::SendBuffer sBuffer;
+ packDataImpl(sender, dir, sBuffer);
+ mpi::RecvBuffer rBuffer(sBuffer);
+ unpackData(receiver, stencil::inverseDir[dir], rBuffer);
+ }
+
+ void unpackData(IBlock *receiver, stencil::Direction dir,
+ mpi::RecvBuffer &buffer) override {
+
+ auto *flag_field = receiver->getData(bdId_);
+ WALBERLA_ASSERT_NOT_NULLPTR(flag_field);
+ WALBERLA_ASSERT_NOT_NULLPTR(m_boundary);
+ WALBERLA_ASSERT_NOT_NULLPTR(m_lattice);
+
+ auto const boundary_flag = flag_field->getFlag(Boundary_flag);
+ auto const gl = numberOfGhostLayersToCommunicate(flag_field);
+ auto const begin = [gl, dir](auto const *flag_field) {
+ return flag_field->beginGhostLayerOnly(gl, dir);
+ };
+
+#ifndef NDEBUG
+ uint_t xSize, ySize, zSize, bSize;
+ buffer >> xSize >> ySize >> zSize >> bSize;
+ uint_t buf_size{0u};
+ for (auto it = begin(flag_field); it != flag_field->end(); ++it) {
+ if (isFlagSet(it, boundary_flag)) {
+ ++buf_size;
+ }
+ }
+ WALBERLA_ASSERT_EQUAL(xSize, flag_field->xSize());
+ WALBERLA_ASSERT_EQUAL(ySize, flag_field->ySize());
+ WALBERLA_ASSERT_EQUAL(zSize, flag_field->zSize());
+ WALBERLA_ASSERT_EQUAL(bSize, buf_size);
+#endif
+
+ auto const offset = std::get<0>(m_lattice->get_local_grid_range());
+ typename Boundary_T::value_type value;
+ for (auto it = begin(flag_field); it != flag_field->end(); ++it) {
+ if (isFlagSet(it, boundary_flag)) {
+ auto const node = offset + Utils::Vector3i{{it.x(), it.y(), it.z()}};
+ buffer >> value;
+ m_boundary->unpack_node(node, value);
+ }
+ }
+ }
+
+protected:
+ void packDataImpl(IBlock const *sender, stencil::Direction dir,
+ mpi::SendBuffer &buffer) const override {
+
+ auto const *flag_field = sender->getData(bdId_);
+ WALBERLA_ASSERT_NOT_NULLPTR(flag_field);
+ WALBERLA_ASSERT_NOT_NULLPTR(m_boundary);
+ WALBERLA_ASSERT_NOT_NULLPTR(m_lattice);
+
+ auto const boundary_flag = flag_field->getFlag(Boundary_flag);
+ auto const gl = numberOfGhostLayersToCommunicate(flag_field);
+ auto const begin = [gl, dir](auto const *flag_field) {
+ return flag_field->beginSliceBeforeGhostLayer(dir, gl);
+ };
+
+#ifndef NDEBUG
+ uint_t buf_size{0u};
+ for (auto it = begin(flag_field); it != flag_field->end(); ++it) {
+ if (isFlagSet(it, boundary_flag)) {
+ ++buf_size;
+ }
+ }
+ buffer << flag_field->xSize() << flag_field->ySize() << flag_field->zSize()
+ << buf_size;
+#endif
+
+ auto const offset = std::get<0>(m_lattice->get_local_grid_range());
+ for (auto it = begin(flag_field); it != flag_field->end(); ++it) {
+ if (isFlagSet(it, boundary_flag)) {
+ auto const node = offset + Utils::Vector3i{{it.x(), it.y(), it.z()}};
+ buffer << m_boundary->get_node_value_at_boundary(node);
+ }
+ }
+ }
+
+private:
+ std::shared_ptr m_lattice;
+ std::shared_ptr m_boundary;
+};
+
+} // namespace communication
+} // namespace field
+} // namespace walberla
diff --git a/src/walberla_bridge/src/lattice_boltzmann/LBWalberlaImpl.hpp b/src/walberla_bridge/src/lattice_boltzmann/LBWalberlaImpl.hpp
index 0e144abb47..ecf7e57a64 100644
--- a/src/walberla_bridge/src/lattice_boltzmann/LBWalberlaImpl.hpp
+++ b/src/walberla_bridge/src/lattice_boltzmann/LBWalberlaImpl.hpp
@@ -45,6 +45,7 @@
#include
#include "../BoundaryHandling.hpp"
+#include "../BoundaryPackInfo.hpp"
#include "InterpolateAndShiftAtBoundary.hpp"
#include "ResetForce.hpp"
#include "lb_kernels.hpp"
@@ -235,8 +236,9 @@ class LBWalberlaImpl : public LBWalberlaBase {
typename FieldTrait::template PackInfo;
// communicators
- std::shared_ptr m_full_communication;
- std::shared_ptr m_pdf_streaming_communication;
+ std::shared_ptr m_boundary_communicator;
+ std::shared_ptr m_pdf_full_communicator;
+ std::shared_ptr m_pdf_streaming_communicator;
// ResetForce sweep + external force handling
std::shared_ptr> m_reset_force;
@@ -350,27 +352,33 @@ class LBWalberlaImpl : public LBWalberlaBase {
reset_boundary_handling();
// Set up the communication and register fields
- m_pdf_streaming_communication =
+ m_pdf_streaming_communicator =
std::make_shared(blocks);
- m_pdf_streaming_communication->addPackInfo(
+ m_pdf_streaming_communicator->addPackInfo(
std::make_shared>(m_pdf_field_id, n_ghost_layers));
- m_pdf_streaming_communication->addPackInfo(
+ m_pdf_streaming_communicator->addPackInfo(
std::make_shared>(m_last_applied_force_field_id,
n_ghost_layers));
- m_pdf_streaming_communication->addPackInfo(
- std::make_shared>(
- m_flag_field_id, n_ghost_layers));
- m_full_communication = std::make_shared(blocks);
- m_full_communication->addPackInfo(
+ m_pdf_full_communicator = std::make_shared(blocks);
+ m_pdf_full_communicator->addPackInfo(
std::make_shared>(m_pdf_field_id, n_ghost_layers));
- m_full_communication->addPackInfo(std::make_shared>(
- m_last_applied_force_field_id, n_ghost_layers));
- m_full_communication->addPackInfo(std::make_shared>(
- m_velocity_field_id, n_ghost_layers));
- m_full_communication->addPackInfo(
+ m_pdf_full_communicator->addPackInfo(
+ std::make_shared>(m_last_applied_force_field_id,
+ n_ghost_layers));
+ m_pdf_full_communicator->addPackInfo(
+ std::make_shared>(m_velocity_field_id,
+ n_ghost_layers));
+
+ m_boundary_communicator = std::make_shared(blocks);
+ m_boundary_communicator->addPackInfo(
std::make_shared>(
m_flag_field_id, n_ghost_layers));
+ auto boundary_packinfo = std::make_shared<
+ field::communication::BoundaryPackInfo>(
+ m_flag_field_id, n_ghost_layers);
+ boundary_packinfo->setup_boundary_handle(m_lattice, m_boundary);
+ m_boundary_communicator->addPackInfo(boundary_packinfo);
// Instantiate the sweep responsible for force double buffering and
// external forces
@@ -439,13 +447,13 @@ class LBWalberlaImpl : public LBWalberlaBase {
integrate_reset_force(blocks);
// LB collide
integrate_collide(blocks);
- m_pdf_streaming_communication->communicate();
+ m_pdf_streaming_communicator->communicate();
// Handle boundaries
integrate_boundaries(blocks);
// LB stream
integrate_stream(blocks);
// Refresh ghost layers
- m_full_communication->communicate();
+ ghost_communication_pdfs();
}
void integrate_pull_scheme() {
@@ -458,7 +466,7 @@ class LBWalberlaImpl : public LBWalberlaBase {
// LB collide
integrate_collide(blocks);
// Refresh ghost layers
- ghost_communication();
+ ghost_communication_pdfs();
}
protected:
@@ -474,7 +482,6 @@ class LBWalberlaImpl : public LBWalberlaBase {
public:
void integrate() override {
- reallocate_ubb_field();
if (has_lees_edwards_bc()) {
integrate_pull_scheme();
} else {
@@ -485,7 +492,16 @@ class LBWalberlaImpl : public LBWalberlaBase {
}
void ghost_communication() override {
- m_full_communication->communicate();
+ ghost_communication_boundary();
+ ghost_communication_pdfs();
+ }
+
+ void ghost_communication_boundary() {
+ m_boundary_communicator->communicate();
+ }
+
+ void ghost_communication_pdfs() {
+ m_pdf_full_communicator->communicate();
if (has_lees_edwards_bc()) {
auto const &blocks = get_lattice().get_blocks();
apply_lees_edwards_pdf_interpolation(blocks);
@@ -1097,7 +1113,10 @@ class LBWalberlaImpl : public LBWalberlaBase {
void reallocate_ubb_field() override { m_boundary->boundary_update(); }
- void clear_boundaries() override { reset_boundary_handling(); }
+ void clear_boundaries() override {
+ reset_boundary_handling();
+ ghost_communication();
+ }
void
update_boundary_from_shape(std::vector const &raster_flat,
@@ -1105,6 +1124,8 @@ class LBWalberlaImpl : public LBWalberlaBase {
auto const grid_size = get_lattice().get_grid_dimensions();
auto const data = fill_3D_vector_array(data_flat, grid_size);
set_boundary_from_grid(*m_boundary, get_lattice(), raster_flat, data);
+ ghost_communication();
+ reallocate_ubb_field();
}
// Pressure tensor
diff --git a/src/walberla_bridge/tests/LBWalberlaImpl_unit_tests.cpp b/src/walberla_bridge/tests/LBWalberlaImpl_unit_tests.cpp
index 516d9ab242..94e90de862 100644
--- a/src/walberla_bridge/tests/LBWalberlaImpl_unit_tests.cpp
+++ b/src/walberla_bridge/tests/LBWalberlaImpl_unit_tests.cpp
@@ -202,6 +202,7 @@ BOOST_DATA_TEST_CASE(update_boundary_from_shape, bdata::make(all_lbs()),
std::vector vel_flat(vel_3d.data(),
vel_3d.data() + vel_3d.num_elements());
lb->update_boundary_from_shape(raster_flat, vel_flat);
+ lb->ghost_communication();
}
for (auto const &node : nodes) {
diff --git a/testsuite/python/CMakeLists.txt b/testsuite/python/CMakeLists.txt
index ee457e27fc..0389f71ad7 100644
--- a/testsuite/python/CMakeLists.txt
+++ b/testsuite/python/CMakeLists.txt
@@ -334,6 +334,7 @@ python_test(FILE thole.py MAX_NUM_PROC 4)
python_test(FILE lb_slice.py MAX_NUM_PROC 2)
python_test(FILE lb_boundary_velocity.py MAX_NUM_PROC 1)
# python_test(FILE lb_boundary_volume_force.py MAX_NUM_PROC 2) # TODO
+python_test(FILE lb_boundary_ghost_layer.py MAX_NUM_PROC 2)
python_test(FILE lb_circular_couette.py MAX_NUM_PROC 2 GPU_SLOTS 1)
python_test(FILE lb_poiseuille.py MAX_NUM_PROC 4 GPU_SLOTS 1)
python_test(FILE lb_poiseuille_cylinder.py MAX_NUM_PROC 2 GPU_SLOTS 1)
diff --git a/testsuite/python/lb_boundary_ghost_layer.py b/testsuite/python/lb_boundary_ghost_layer.py
new file mode 100644
index 0000000000..0c226d4e03
--- /dev/null
+++ b/testsuite/python/lb_boundary_ghost_layer.py
@@ -0,0 +1,103 @@
+#
+# Copyright (C) 2024 The ESPResSo project
+#
+# This file is part of ESPResSo.
+#
+# ESPResSo is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ESPResSo is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+
+import unittest as ut
+import unittest_decorators as utx
+import numpy as np
+import scipy.optimize
+
+import espressomd.lb
+import espressomd.shapes
+
+AGRID = 0.5
+KINEMATIC_VISC = 2.7
+DENS = 1.7
+TIME_STEP = 0.07
+LB_PARAMS = {"agrid": AGRID, "tau": TIME_STEP, "density": DENS,
+ "kinematic_viscosity": KINEMATIC_VISC}
+
+
+class TestCommon:
+
+ system = espressomd.System(box_l=[16.0, 1.0, 1.0])
+ system.time_step = TIME_STEP
+ system.cell_system.skin = 0.4 * AGRID
+ n_nodes = system.cell_system.get_state()["n_nodes"]
+
+ def setUp(self):
+ self.lbf = self.lb_class(**LB_PARAMS, **self.lb_params)
+ self.system.lb = self.lbf
+ self.ubb = espressomd.lb.VelocityBounceBack([0., 0., 1e-5])
+
+ def tearDown(self):
+ self.system.lb = None
+
+ def get_profile(self):
+ xdata = np.arange(1, self.lbf.shape[0])
+ ydata = []
+ for x in xdata:
+ ydata.append(np.mean(self.lbf[x, :, :].velocity[:, :, 2]))
+ return xdata, np.array(ydata)
+
+ def check_profile(self):
+ def quadratic(x, a, b, c):
+ return a * x**2 + b * x + c
+
+ self.system.integrator.run(40)
+ xdata, ydata = self.get_profile()
+ popt_ref = (4e-8, -1e-6, 1e-5)
+ popt, _ = scipy.optimize.curve_fit(
+ quadratic, xdata, ydata, p0=popt_ref)
+ rtol = 0.3 if self.lbf.single_precision else 0.1
+ np.testing.assert_allclose(popt, popt_ref, rtol=0.5, atol=0.)
+ np.testing.assert_allclose(ydata, quadratic(xdata, *popt),
+ rtol=rtol, atol=0.)
+
+ def test_node_setter(self):
+ for i in (0, 1):
+ for j in (0, 1):
+ self.lbf[0, i, j].boundary = self.ubb
+ self.check_profile()
+
+ def test_slice_setter(self):
+ self.lbf[0, :, :].boundary = self.ubb
+ self.check_profile()
+
+ def test_shape_setter(self):
+ shape = espressomd.shapes.Wall(normal=[1, 0, 0], dist=AGRID)
+ self.lbf.add_boundary_from_shape(shape, velocity=self.ubb.velocity)
+ self.check_profile()
+
+
+@utx.skipIfMissingFeatures(["WALBERLA"])
+@ut.skipIf(TestCommon.n_nodes != 2, "only runs for 2 MPI ranks")
+class LBPoiseuilleWalberlaSinglePrecision(TestCommon, ut.TestCase):
+ lb_class = espressomd.lb.LBFluidWalberla
+ lb_params = {"single_precision": True}
+
+
+@utx.skipIfMissingFeatures(["WALBERLA"])
+@ut.skipIf(TestCommon.n_nodes != 2, "only runs for 2 MPI ranks")
+class LBPoiseuilleWalberlaDoublePrecision(TestCommon, ut.TestCase):
+ lb_class = espressomd.lb.LBFluidWalberla
+ lb_params = {"single_precision": False}
+
+
+if __name__ == "__main__":
+ ut.main()
diff --git a/testsuite/python/lb_circular_couette.py b/testsuite/python/lb_circular_couette.py
index 71f5836671..e27e52d6cf 100644
--- a/testsuite/python/lb_circular_couette.py
+++ b/testsuite/python/lb_circular_couette.py
@@ -143,8 +143,8 @@ def test_taylor_couette_flow(self):
a_ref, b_ref = taylor_couette(slip_vel, 0.0, cyl1.radius, cyl2.radius)
v_phi_ref = a_ref * r + b_ref / r
v_phi_drift = np.mean(v_phi) - np.mean(v_phi_ref)
- np.testing.assert_allclose(v_phi_drift, 0., atol=1.2e-4)
- np.testing.assert_allclose(v_phi - v_phi_drift, v_phi_ref, atol=1e-4)
+ np.testing.assert_allclose(v_phi_drift, 0., atol=4e-4)
+ np.testing.assert_allclose(v_phi - v_phi_drift, v_phi_ref, atol=4e-4)
@utx.skipIfMissingFeatures(["WALBERLA"])
diff --git a/testsuite/scripts/samples/test_lb_circular_couette.py b/testsuite/scripts/samples/test_lb_circular_couette.py
index c91ca03242..5111ee9032 100644
--- a/testsuite/scripts/samples/test_lb_circular_couette.py
+++ b/testsuite/scripts/samples/test_lb_circular_couette.py
@@ -52,19 +52,18 @@ def test_taylor_couette_flow(self):
np.testing.assert_allclose(v_phi[:7], 0., atol=1e-7)
# check azimuthal velocity in the linear regime
- self.assertGreater(v_phi[7], v_phi[6])
self.assertGreater(v_phi[8], v_phi[7])
self.assertGreater(v_phi[9], v_phi[8])
# check azimuthal velocity in the Couette regime
- xdata = sample.profile_r[9:]
- ydata = v_phi[9:]
+ xdata = sample.profile_r[9:-1]
+ ydata = v_phi[9:-1]
a_ref, b_ref = taylor_couette(
sample.velocity_magnitude, 0.0, sample.cylinder_in.radius,
sample.cylinder_out.radius, sample.agrid)
(a_sim, b_sim), _ = scipy.optimize.curve_fit(
lambda x, a, b: a * x + b / x, xdata, ydata)
- np.testing.assert_allclose([a_sim, b_sim], [a_ref, b_ref], atol=1e-3)
+ np.testing.assert_allclose([a_sim, b_sim], [a_ref, b_ref], rtol=0.05)
if __name__ == "__main__":