[WIP] Automatic python bindings with cppyy

CMakeLists.txt

@@ -3,14 +3,14 @@ cmake_minimum_required(VERSION 3.21)
 project(
   NESO
   VERSION 0.0.1
-  LANGUAGES CXX)
+  LANGUAGES C CXX)
 
 set(CMAKE_CXX_STANDARD 17)
 
 set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH})
 
 option(ENABLE_NESO_TESTS
-       "Build unit tests for this project and register with ctest" ON)
+       "Build unit tests for this project and register with ctest" OFF)
 
 # Various sanitizers, including coverage and address sanitizer
 include(cmake/Sanitizers.cmake)
@@ -112,7 +112,7 @@ set(HEADER_FILES
 
 # Create library
 set(LIBRARY_NAME nesolib)
-add_library(${LIBRARY_NAME} ${LIB_SRC_FILES} ${HEADER_FILES})
+add_library(${LIBRARY_NAME} SHARED ${LIB_SRC_FILES} ${HEADER_FILES})
 enable_sanitizers(${LIBRARY_NAME})
 target_include_directories(
   ${LIBRARY_NAME} PUBLIC $<INSTALL_INTERFACE:include>
@@ -157,6 +157,53 @@ if(ENABLE_NESO_TESTS)
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/test)
 endif()
 
+# ##############################################################################
+# Add cppyy python bindings
+# ##############################################################################
+
+# Get include files
+file(GLOB LIB_HPPS ${CMAKE_CURRENT_SOURCE_DIR}/include/*.hpp
+)# ${CMAKE_CURRENT_SOURCE_DIR}/include/nektar_interface/*.hpp)
+
+set(CMAKE_MODULE_PATH ${CPPYY_MODULE_PATH} ${CMAKE_MODULE_PATH})
+set(Cppyy_DIR ${CPPYY_MODULE_PATH})
+find_package(Cppyy)
+
+set(HIPSYCL_INC
+    /home/jparker/spack/opt/spack/linux-centos7-haswell/gcc-11.2.0/hipsycl-0.9.2-nfvpn6i7t3b25zbzgw3lpx35yuo3khbq/include/
+)
+
+message(${LIB_HPPS})
+
+# Specification at https://cppyy.readthedocs.io/en/latest/cmake_interface.html
+cppyy_add_bindings(
+  "PyNESO"
+  "0.0.1"
+  "Joseph Parker"
+  "[email protected]"
+  LANGUAGE_STANDARD
+  "17"
+  H_FILES
+  ${LIB_HPPS}
+  GENERATE_OPTIONS
+  -D${SYCL_FLAG}
+  -I
+  ${HIPSYCL_INC}
+  -DBINDING_BUILD=on
+  -pthread
+  -std=c++1z
+  -m64
+  -I/home/jparker/code/NESO/venv_3_9_12/lib/python3.9/site-packages/cppyy_backend/include
+  INCLUDE_DIRS
+  ${CMAKE_SOURCE_DIR}/include
+  ${MPI_CXX_INCLUDE_PATH}
+  $<INSTALL_INTERFACE:include>
+  ${HIPSYCL_INC}
+  /home/jparker/code/NESO/include
+  /home/jparker/spack/opt/spack/linux-centos7-haswell/gcc-11.2.0/fftw-3.3.10-u6j4gsbk2z3s6iosn32aw3crgsitva5v/include
+  LINK_LIBRARIES
+  ${BUILD_TYPE_LINK_FLAGS})
+
 # ##############################################################################
 # Configure installation
 # ##############################################################################

README_cppyy.md

@@ -0,0 +1,187 @@
+Cppyy is a library for automatically generating python bindings for C++ code. This allows us to call any function from NESO in python code.
+As an example, the whole of NESO main is replicated in `cppyy_run.py`.
+
+Cppyy requires llvm / clang to build. I've had problems setting up the build
+environment automatically, but this set up works for me.
+
+## Activate the spack environment
+
+```
+git submodule update --init
+. activate
+```
+
+## Set up the Python environment
+
+Install the python dependencies in a virtual environment by doing
+
+```
+python3 -m venv venv_cppyy
+. venv_cppyy/bin/activate
+pip3 install -r requirements
+```
+
+This will install the dependencies:
+
+* cppyy==2.4.2
+* libclang
+* clang-format
+* setuptools
+* wheel
+
+# Install the spack environment
+
+Set up the `spack.yaml` to the supported toolchain:
+
+```
+spack:
+  specs:
+  - neso%[email protected] ^openblas ^hipsycl ^scotch@6 ^[email protected] ^[email protected]%[email protected]
+  - [email protected]%[email protected] ^[email protected]%[email protected]
+  - [email protected]%[email protected] +python +clang
+  - [email protected]%[email protected]
+  - [email protected]
+```
+
+These versions are pinned to what works for me - other versions may work too.
+
+Install the environment with
+
+```
+spack concretize -f
+spack install
+```
+
+This command might fail if the clang compiler is not installed. 
+In that case, try replacing the first line with
+
+```
+spack:
+  specs:
+  - neso%[email protected] ^openblas ^hipsycl ^scotch@6 ^[email protected] ^[email protected]%[email protected]
+```
+
+do `spack concretize -f; spack install`, and then register the clang compiler with spack by doing
+
+```
+spack load [email protected]%[email protected]
+spack compiler find
+```
+
+then reinstating the first line and doing `spack concretize -f; spack install` again.
+
+This will probably fail too... 
+
+# Building manually
+
+To build manually, do 
+
+```
+spack build-env neso%clang cmake . -B build_new -DCPPYY_MODULE_PATH=venv_cppyy/lib/python3.9/site-packages/cppyy_backend/cmake/
+spack build-env neso%clang cmake --build build_new -j 1 -v
+```
+
+This will fail with errors that look like:
+```
+In file included from input_line_3:2:
+In file included from /home/jparker/spack/opt/spack/linux-centos7-haswell/gcc-12.2.0/gcc-11.2.0-54m6goknrjplkcc6tvobxnijkssbx4qg/lib/gcc/x86_64-pc-linux-gnu/11.2.0/../../../../include/c++/11.2.0/string:40:
+In file included from /home/jparker/spack/opt/spack/linux-centos7-haswell/gcc-12.2.0/gcc-11.2.0-54m6goknrjplkcc6tvobxnijkssbx4qg/lib/gcc/x86_64-pc-linux-gnu/11.2.0/../../../../include/c++/11.2.0/bits/char_traits.h:699:
+/home/jparker/spack/opt/spack/linux-centos7-haswell/gcc-12.2.0/gcc-11.2.0-54m6goknrjplkcc6tvobxnijkssbx4qg/lib/gcc/x86_64-pc-linux-gnu/11.2.0/../../../../include/c++/11.2.0/cstdint:47:11: error: no member named 'int8_t' in the global namespace
+  using ::int8_t;
+```
+
+Copy the last line from the verbose build and execute on the command line.
+For me, this is:
+```
+cd /home/jparker/code/NESO/build_new/PyNESO && /home/jparker/code/NESO/venv_3_9_12/bin/rootcling -f /home/jparker/code/NESO/build_new/PyNESO.cpp -s PyNESO -rmf /home/jparker/code/NESO/build_new/PyNESO/PyNESO.rootmap -rml libPyNESOCppyy.so -cxxflags='-DNESO_HIPSYCL -I /home/jparker/spack/opt/spack/linux-centos7-haswell/gcc-11.2.0/hipsycl-0.9.2-nfvpn6i7t3b25zbzgw3lpx35yuo3khbq/include/ -DBINDING_BUILD=on -pthread -std=c++1z -m64 -I/home/jparker/code/NESO/venv_3_9_12/lib/python3.9/site-packages/cppyy_backend/include -std=c++17 -I/home/jparker/code/NESO -I/home/jparker/code/NESO/include -I -I/home/jparker/spack/opt/spack/linux-centos7-haswell/gcc-11.2.0/hipsycl-0.9.2-nfvpn6i7t3b25zbzgw3lpx35yuo3khbq/include/ -I/home/jparker/code/NESO/include -I/home/jparker/spack/opt/spack/linux-centos7-haswell/gcc-11.2.0/fftw-3.3.10-u6j4gsbk2z3s6iosn32aw3crgsitva5v/include' /home/jparker/code/NESO/include/custom_types.hpp /home/jparker/code/NESO/include/diagnostics.hpp /home/jparker/code/NESO/include/fft_fftw.hpp /home/jparker/code/NESO/include/fft_mkl.hpp /home/jparker/code/NESO/include/fft_wrappers.hpp /home/jparker/code/NESO/include/mesh.hpp /home/jparker/code/NESO/include/plasma.hpp /home/jparker/code/NESO/include/simulation.hpp /home/jparker/code/NESO/include/species.hpp /home/jparker/code/NESO/include/velocity.hpp /home/jparker/code/NESO/build_new/linkdef.h
+```
+
+This works. Then do
+
+```
+cd ../..
+spack build-env neso%clang cmake --build build_new -j 1 -v
+```
+
+This fails with error like:
+
+```
+ERROR: Unknown container kind CursorKind.OMP_PARALLEL_MASKED_DIRECTIVE
+Traceback (most recent call last):
+  File "/home/jparker/.local/lib/python3.9/site-packages/cppyy_backend/_cppyy_generator.py", line 737, in main
+    json.dump(mapping, f, indent=1, sort_keys=True)
+  File "/home/jparker/spack/opt/spack/linux-centos7-haswell/gcc-11.2.0/python-3.9.12-zoa7syazdqtlpe7bhp735modrmwwz27r/lib/python3.9/json/__init__.py", line 179, in dump
+    for chunk in iterable:
+  File "/home/jparker/spack/opt/spack/linux-centos7-haswell/gcc-11.2.0/python-3.9.12-zoa7syazdqtlpe7bhp735modrmwwz27r/lib/python3.9/json/encoder.py", line 429, in _iterencode
+    yield from _iterencode_list(o, _current_indent_level)
+  File "/home/jparker/spack/opt/spack/linux-centos7-haswell/gcc-11.2.0/python-3.9.12-zoa7syazdqtlpe7bhp735modrmwwz27r/lib/python3.9/json/encoder.py", line 325, in _iterencode_list
+    yield from chunks
+  File "/home/jparker/spack/opt/spack/linux-centos7-haswell/gcc-11.2.0/python-3.9.12-zoa7syazdqtlpe7bhp735modrmwwz27r/lib/python3.9/json/encoder.py", line 405, in _iterencode_dict
+    yield from chunks
+  File "/home/jparker/spack/opt/spack/linux-centos7-haswell/gcc-11.2.0/python-3.9.12-zoa7syazdqtlpe7bhp735modrmwwz27r/lib/python3.9/json/encoder.py", line 438, in _iterencode
+    o = _default(o)
+  File "/home/jparker/spack/opt/spack/linux-centos7-haswell/gcc-11.2.0/python-3.9.12-zoa7syazdqtlpe7bhp735modrmwwz27r/lib/python3.9/json/encoder.py", line 179, in default
+    raise TypeError(f'Object of type {o.__class__.__name__} '
+TypeError: Object of type CursorKind is not JSON serializable
+
+gmake[2]: *** [PyNESO/PyNESO.map] Error 1
+gmake[2]: *** Deleting file `PyNESO/PyNESO.map'
+gmake[2]: Leaving directory `/home/jparker/code/NESO/build_new'
+gmake[1]: *** [CMakeFiles/PyNESOCppyy.dir/all] Error 2
+gmake[1]: Leaving directory `/home/jparker/code/NESO/build_new'
+gmake: *** [all] Error 2
+```
+
+Again execute the last displayed command, for me:
+
+```
+cd /home/jparker/code/NESO/build_new/PyNESO && python3 /home/jparker/code/NESO/venv_3_9_12/bin/cppyy-generator --libclang /home/jparker/spack/opt/spack/linux-centos7-haswell/gcc-11.2.0/llvm-12.0.1-oomlmdehchcg455mfixdplvmluxogicz/lib/libclang.so --flags "\-DNESO_HIPSYCL;\-I;/home/jparker/spack/opt/spack/linux-centos7-haswell/gcc-11.2.0/hipsycl-0.9.2-nfvpn6i7t3b25zbzgw3lpx35yuo3khbq/include/;\-DBINDING_BUILD=on;\-pthread;\-std=c++1z;\-m64;\-I/home/jparker/code/NESO/venv_3_9_12/lib/python3.9/site-packages/cppyy_backend/include;\-std=c++17;\-I/home/jparker/code/NESO;\-I/home/jparker/code/NESO/include;\-I;\-I/home/jparker/spack/opt/spack/linux-centos7-haswell/gcc-11.2.0/hipsycl-0.9.2-nfvpn6i7t3b25zbzgw3lpx35yuo3khbq/include/;\-I/home/jparker/code/NESO/include;\-I/home/jparker/spack/opt/spack/linux-centos7-haswell/gcc-11.2.0/fftw-3.3.10-u6j4gsbk2z3s6iosn32aw3crgsitva5v/include" /home/jparker/code/NESO/build_new/PyNESO/PyNESO.map /home/jparker/code/NESO/include/custom_types.hpp /home/jparker/code/NESO/include/diagnostics.hpp /home/jparker/code/NESO/include/fft_fftw.hpp /home/jparker/code/NESO/include/fft_mkl.hpp /home/jparker/code/NESO/include/fft_wrappers.hpp /home/jparker/code/NESO/include/mesh.hpp /home/jparker/code/NESO/include/plasma.hpp /home/jparker/code/NESO/include/simulation.hpp /home/jparker/code/NESO/include/species.hpp /home/jparker/code/NESO/include/velocity.hpp
+```
+
+This works. Again do
+
+```
+cd ../..
+spack build-env neso%clang cmake --build build_new -j 1 -v
+```
+
+This fails with
+
+```
+[ 72%] Generating dist/PyNESO-0.0.1-py3-none-linux_x86_64.whl
+python3 setup.py bdist_wheel
+Traceback (most recent call last):
+  File "/home/jparker/code/NESO/build_new/setup.py", line 7, in <module>
+    import setuptools
+ModuleNotFoundError: No module named 'setuptools'
+gmake[2]: *** [dist/PyNESO-0.0.1-py3-none-linux_x86_64.whl] Error 1
+gmake[2]: Leaving directory `/home/jparker/code/NESO/build_new'
+gmake[1]: *** [CMakeFiles/wheel.dir/all] Error 2
+gmake[1]: Leaving directory `/home/jparker/code/NESO/build_new'
+gmake: *** [all] Error 2
+```
+
+Now do
+
+```
+cd build_new
+python3 setup.py bdist_wheel
+cd ..
+spack build-env neso%clang cmake --build build_new -j 1 -v
+```
+
+which now should finish successfully.
+
+## Executing the code
+
+To run the code, do
+
+```
+python3 cppyy_run.py
+```
+
+This will execute the python mock up of NESO's main, and should finish with the same final values:
+
+```
+0.882496 0.75386 0.128635
+```

cmake/FindSYCL.cmake

@@ -28,7 +28,7 @@ foreach(candidate version IN ZIP_LISTS candidates versions)
 
   if(${candidate}_FOUND)
     set(SYCL_FOUND TRUE)
-    set(SYCL_IMPLEMENTATION ${candidate})
+    set(SYCL_IMPLEMENTATION ${candidate} ${version})
     break()
   endif()
 endforeach()

cppyy_run.py

@@ -0,0 +1,84 @@
+import cppyy
+
+# import numpy as np
+# from tqdm import tqdm
+
+cppyy.load_library("build_new/libnesolib.so")
+cppyy.load_library("build_new/build/lib/PyNESO/libPyNESOCppyy.so")
+from cppyy.gbl import Mesh, Species, Plasma, Diagnostics, FFT, evolve
+from cppyy.gbl import sycl
+from cppyy.gbl.std import vector
+
+print(dir(cppyy.gbl))
+
+##############
+# Parameters #
+##############
+# Mesh
+nintervals = 128  # int
+dt = 0.05  # double
+nt = 40  # int
+
+# Ions
+kinetic_i = False  # Whether the species are treated as kinetic
+T_i = 2.0  # Temperature
+q_i = -1.0  # Charge
+m_i = 1836.2  # Mass
+n_i = 1  # Number of particles
+
+# Electrons
+kinetic_e = True  # Whether the species are treated as kinetic
+T_e = 2.0  # Temperature
+q_e = 1.0  # Charge
+m_e = 1.0  # Mass
+n_e = 12800  # Number of particles
+
+# Q = sycl::queue{sycl::default_selector{}, asyncHandler};
+Q = sycl.queue()  # {sycl.default_selector{}, asyncHandler};
+
+mesh = Mesh(nintervals, dt, nt)
+ions = Species(mesh, kinetic_i, T_i, q_i, m_i, n_i)
+electrons = Species(mesh, kinetic_e, T_e, q_e, m_e, n_e)
+
+species_list = vector[Species]()
+species_list.push_back(ions)
+species_list.push_back(electrons)
+plasma = Plasma(species_list)
+
+diagnostics = Diagnostics()
+fft = FFT(Q, mesh.nintervals)
+
+mesh.set_initial_field(Q, mesh, plasma, fft)
+evolve(Q, mesh, plasma, fft, diagnostics)
+
+###for it in tqdm(np.arange(1,int(mesh.nt+1))):
+###
+###    mesh.it = int(it)
+###    #print("it %d\n", mesh.it);
+###
+###    plasma.assemble_rhs(opt, AI, mesh, fft);
+###    timestepper.time_advance(opt, AI, mesh, plasma, fft);
+###
+###    mesh.t += mesh.dt;
+###    diagnostics.store_time(mesh);
+###
+###    plasma.solve_for_electrostatic_potential(AI, mesh);
+###    plasma.get_electric_field_from_electrostatic_potential(AI, mesh);
+###
+###    diagnostics.compute_total_energy(AI, mesh, plasma);
+###    fileio.write_time_slice(mesh, plasma, diagnostics);
+###
+####print("solving_vlasov_poisson: "+str(opt.solving_vlasov_poisson()))
+####print("solving_drift_kinetics: "+str(opt.solving_drift_kinetics()))
+####print("solving_gyro_or_drift_kinetics: "+str(opt.solving_gyro_or_drift_kinetics()))
+####print("exlicit_euler: "+str(opt.explicit_euler()))
+####print("timestepper::expected_order: "+str(timestepper.expected_order))
+###
+####print(np.log(diagnostics.total_energy))
+####print(diagnostics.total_energy)
+####
+####import matplotlib.pyplot as plt
+####plt.semilogy(diagnostics.total_energy)
+####plt.savefig("out.pdf")
+####plt.clf()
+###

include/fft_mkl.hpp

@@ -1,6 +1,7 @@
 #ifndef NEPTUNE_FFTMKL_H
 #define NEPTUNE_FFTMKL_H
 
+#ifdef NESO_INTEL_MKL_FFT
 #include "custom_types.hpp"
 #include "oneapi/mkl/dfti.hpp"
 #include <CL/sycl.hpp>
@@ -48,4 +49,5 @@ class FFT {
   mutable bool init_plan;
 };
 
+#endif
 #endif // NEPTUNE_FFT_MKL_H

include/mesh.hpp

@@ -17,7 +17,8 @@ class Mesh;
 
 class Mesh {
 public:
-  Mesh(int nintervals = 10, double dt = 0.1, int nt = 1000);
+  Mesh();
+  Mesh(int nintervals, double dt = 0.1, int nt = 1000);
   // time
   double t;
   // time step

include/species.hpp

@@ -14,6 +14,8 @@ class Species;
 
 class Species {
 public:
+  Species(); // unused but required by cppyy python bindings
+
   Species(const Mesh &mesh, bool kinetic = true, double T = 1.0, double q = 1,
           double m = 1, int n = 10);
   // Whether this species is treated kinetically (true) or adiabatically (false)