From 949e24511e1b1f7443ff37ee2800f1ed76051fd7 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 27 Oct 2023 09:37:27 -0700 Subject: [PATCH] Remove distutils Distutils is gone in Python 3.12+. --- .../dependencies/clang-san-openmpi.sh | 3 +- .github/workflows/dependencies/clang-tidy.sh | 3 +- .github/workflows/dependencies/gcc-openmpi.sh | 3 +- .github/workflows/dependencies/gcc.sh | 3 +- .github/workflows/macos.yml | 3 +- .github/workflows/windows.yml | 6 +- README.md | 187 +----------------- docs/source/developers/testing.rst | 3 +- docs/source/install/cmake.rst | 3 +- pyproject.toml | 3 +- setup.py | 18 +- 11 files changed, 30 insertions(+), 205 deletions(-) diff --git a/.github/workflows/dependencies/clang-san-openmpi.sh b/.github/workflows/dependencies/clang-san-openmpi.sh index 9b7261223..df5b8293f 100755 --- a/.github/workflows/dependencies/clang-san-openmpi.sh +++ b/.github/workflows/dependencies/clang-san-openmpi.sh @@ -26,7 +26,8 @@ sudo apt-get install -y \ python3-pip \ wget -python3 -m pip install -U pip setuptools wheel +python3 -m pip install -U pip +python3 -m pip install -U build packaging setuptools wheel python3 -m pip install -U cmake pytest python3 -m pip install -U -r requirements_mpi.txt python3 -m pip install -U -r examples/requirements.txt diff --git a/.github/workflows/dependencies/clang-tidy.sh b/.github/workflows/dependencies/clang-tidy.sh index 4b8bb76a2..50cd73db0 100755 --- a/.github/workflows/dependencies/clang-tidy.sh +++ b/.github/workflows/dependencies/clang-tidy.sh @@ -25,7 +25,8 @@ sudo apt-get install -y \ python3-pip \ wget -python3 -m pip install -U pip setuptools wheel +python3 -m pip install -U pip +python3 -m pip install -U build packaging setuptools wheel python3 -m pip install -U cmake pytest python3 -m pip install -U -r requirements_mpi.txt python3 -m pip install -U -r examples/requirements.txt diff --git a/.github/workflows/dependencies/gcc-openmpi.sh b/.github/workflows/dependencies/gcc-openmpi.sh index 53c1bb4d0..bf921df33 100755 --- a/.github/workflows/dependencies/gcc-openmpi.sh +++ b/.github/workflows/dependencies/gcc-openmpi.sh @@ -22,7 +22,8 @@ sudo apt-get install -y \ python3-pip \ wget -python3 -m pip install -U pip setuptools wheel +python3 -m pip install -U pip +python3 -m pip install -U build packaging setuptools wheel python3 -m pip install -U cmake pytest python3 -m pip install -U -r requirements_mpi.txt python3 -m pip install -U -r examples/requirements.txt diff --git a/.github/workflows/dependencies/gcc.sh b/.github/workflows/dependencies/gcc.sh index 504bfc608..a4c94482e 100755 --- a/.github/workflows/dependencies/gcc.sh +++ b/.github/workflows/dependencies/gcc.sh @@ -21,7 +21,8 @@ sudo apt-get install -y \ python3-pip \ wget -python3 -m pip install -U pip setuptools wheel +python3 -m pip install -U pip +python3 -m pip install -U build packaging setuptools wheel python3 -m pip install -U cmake pytest python3 -m pip install -U -r examples/requirements.txt diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 886a0321b..42dd3c654 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -51,7 +51,8 @@ jobs: brew install ninja brew install open-mpi brew install pkg-config - python3 -m pip install --upgrade pip setuptools wheel pytest + python3 -m pip install --upgrade pip + python3 -m pip install --upgrade build packaging setuptools wheel pytest python3 -m pip install --upgrade pytest python3 -m pip install --upgrade -r requirements_mpi.txt python3 -m pip install --upgrade -r examples/requirements.txt diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 7175cee1d..396dc6eea 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -52,7 +52,8 @@ jobs: $env:HDF5_DIR = "C:/Program Files/HDF_Group/HDF5/1.12.2/cmake/" $env:HDF5_USE_STATIC_LIBRARIES = "ON" - python3 -m pip install -U pip setuptools wheel pytest + python3 -m pip install -U pip + python3 -m pip install -U build packaging setuptools wheel pytest python3 -m pip install -U -r requirements.txt python3 -m pip install -U -r examples/requirements.txt python3 -m pip install -U openPMD-validator @@ -137,7 +138,8 @@ jobs: set "HDF5_DIR=C:/Program Files/HDF_Group/HDF5/1.12.2/cmake/" set "HDF5_USE_STATIC_LIBRARIES=ON" - python3 -m pip install -U pip setuptools wheel pytest + python3 -m pip install -U pip + python3 -m pip install -U build packaging setuptools wheel pytest python3 -m pip install -U -r requirements.txt python3 -m pip install -U -r examples/requirements.txt python3 -m pip install -U openPMD-validator diff --git a/README.md b/README.md index 056e042bb..3b1a7a0b7 100644 --- a/README.md +++ b/README.md @@ -30,189 +30,7 @@ Our workflow is described in [CONTRIBUTING.rst](CONTRIBUTING.rst). ## Developer Environment -Please prepare you local development environment as follows. -Pick *one* of the methods below: - -### Perlmutter (NERSC) - -```bash -ssh perlmutter-p1.nersc.gov -``` - -Now ``cd`` to your ImpactX source directory. - -```bash -module load cmake/3.22.0 -module load cray-hdf5-parallel/1.12.2.1 - -# CCache for faster rebuilds -export PATH=/global/common/software/spackecp/perlmutter/e4s-22.05/78535/spack/opt/spack/cray-sles15-zen3/gcc-11.2.0/ccache-4.5.1-ybl7xefvggn6hov4dsdxxnztji74tolj/bin:$PATH - -# Python -module load cray-python/3.9.13.1 -if [ -d "$HOME/sw/perlmutter/venvs/impactx" ] -then - source $HOME/sw/perlmutter/venvs/impactx/bin/activate -else - python3 -m pip install --user --upgrade pip - python3 -m pip install --user virtualenv - python3 -m venv $HOME/sw/perlmutter/venvs/impactx - source $HOME/sw/perlmutter/venvs/impactx/bin/activate - - python3 -m pip install --upgrade pip - MPICC="cc -target-accel=nvidia80 -shared" python3 -m pip install -U --no-cache-dir -v mpi4py - python3 -m pip install --upgrade pytest - python3 -m pip install --upgrade -r requirements.txt - python3 -m pip install --upgrade -r examples/requirements.txt -fi - -# GPU-aware MPI -export MPICH_GPU_SUPPORT_ENABLED=1 - -# necessary to use CUDA-Aware MPI and run a job -export CRAY_ACCEL_TARGET=nvidia80 - -# optimize CUDA compilation for A100 -export AMREX_CUDA_ARCH=8.0 - -# compiler environment hints -export CC=cc -export CXX=CC -export FC=ftn -export CUDACXX=$(which nvcc) -export CUDAHOSTCXX=CC -``` - -```bash -# work on an interactive node -salloc -N 1 --ntasks-per-node=4 -t 1:00:00 -C gpu --gpu-bind=single:1 -c 32 -G 4 --qos=interactive -A m3906_g - -# configure -cmake -S . -B build_perlmutter -DImpactX_COMPUTE=CUDA -DImpactX_PYTHON=ON - -# compile -cmake --build build_perlmutter -j 64 - -# test -ctest --test-dir build_perlmutter -E AMReX --output-on-failure - -# run -cd build_perlmutter/bin -srun ./impactx ../../examples/fodo/input_fodo.in -``` - -### Cori KNL (NERSC) - -```bash -ssh cori.nersc.gov -``` - -Now ``cd`` to your ImpactX source directory. - -```bash -module swap craype-haswell craype-mic-knl -module swap PrgEnv-intel PrgEnv-gnu -module load cmake/3.22.1 -module load cray-hdf5-parallel/1.10.5.2 -module load cray-fftw/3.3.8.10 - -# Python -module load cray-python/3.9.7.1 -if [ -d "$HOME/sw/knl/venvs/impactx" ] -then - source $HOME/sw/knl/venvs/impactx/bin/activate -else - python3 -m pip install --user --upgrade pip - python3 -m pip install --user virtualenv - python3 -m venv $HOME/sw/knl/venvs/impactx - source $HOME/sw/knl/venvs/impactx/bin/activate - - python3 -m pip install --upgrade pip - MPICC="cc -shared" python3 -m pip install -U --no-cache-dir -v mpi4py - python3 -m pip install --upgrade pytest - python3 -m pip install --upgrade -r requirements.txt - python3 -m pip install --upgrade -r examples/requirements.txt -fi - -# tune exactly for KNL sub-architecture -export CXXFLAGS="-march=knl" -export CFLAGS="-march=knl" -``` - -```bash -# configure -cmake -S . -B build_knl - -# compile -cmake --build build_knl -j 8 - -# test -srun -C knl -N 1 -t 30 -q debug ctest --test-dir build_knl --output-on-failure - -# run -cd build_knl/bin -srun -C knl -N 1 -t 30 -q debug ./impactx ../../examples/fodo/input_fodo.in -``` - -### Homebrew (macOS) - -```bash -brew update -brew install adios2 # for openPMD -brew install ccache -brew install cmake -brew install fftw -brew install git -brew install hdf5-mpi # for openPMD -brew install libomp # for OpenMP -brew install pkg-config # for fftw -brew install open-mpi -``` - -### Apt (Debian/Ubuntu) - -```bash -sudo apt update -sudo apt install build-essential ccache cmake g++ git libfftw3-mpi-dev libfftw3-dev libhdf5-openmpi-dev libopenmpi-dev pkg-config python3 python3-matplotlib python3-numpy python3-scipy -``` - -### Spack (Linux) - -```bash -spack env create impactx-dev -spack env activate impactx-dev -spack add adios2 # for openPMD -spack add ccache -spack add cmake -spack add fftw -spack add hdf5 # for openPMD -spack add mpi -spack add pkgconfig # for fftw -spack add python -spack add py-pip -spack add py-setuptools -spack add py-wheel - -# OpenMP support on macOS -[[ $OSTYPE == 'darwin'* ]] && spack add llvm-openmp - -# optional: Linux only -#spack add cuda - -spack install -python3 -m pip install matplotlib numpy openpmd-api pandas pytest scipy -``` - -In new terminals, re-activate the environment with `spack env activate impactx-dev` again. - -### Conda (Linux/macOS/Windows) - -```bash -conda create -n impactx-dev -c conda-forge adios2 ccache cmake compilers git hdf5 fftw matplotlib ninja numpy openpmd-api pandas pytest scipy -conda activate impactx-dev - -# compile with -DImpactX_MPI=OFF -``` +Please see our [developer installation section](https://impactx.readthedocs.io/en/latest/install/dependencies.html#install-dependencies) of the documentation for an easy install of our software dependencies. ## Get the Source Code @@ -303,7 +121,8 @@ Examples: In order to run our tests, you need to have a few Python packages installed: ```console -python3 -m pip install --upgrade pip setuptools wheel pytest +python3 -m pip install --upgrade pip +python3 -m pip install --upgrade build packaging setuptools wheel pytest python3 -m pip install --upgrade -r examples/requirements.txt ``` diff --git a/docs/source/developers/testing.rst b/docs/source/developers/testing.rst index ad5a0359b..fc3450f15 100644 --- a/docs/source/developers/testing.rst +++ b/docs/source/developers/testing.rst @@ -12,7 +12,8 @@ In order to run our tests, you need to have a few :ref:`Python packages installe .. code-block:: sh - python3 -m pip install -U pip setuptools wheel pytest + python3 -m pip install -U pip + python3 -m pip install -U build packaging setuptools wheel pytest python3 -m pip install -r examples/requirements.txt Run diff --git a/docs/source/install/cmake.rst b/docs/source/install/cmake.rst index af2617bf5..65ad6238d 100644 --- a/docs/source/install/cmake.rst +++ b/docs/source/install/cmake.rst @@ -35,7 +35,8 @@ ImpactX depends on popular third party software. .. code-block:: bash - python3 -m pip install -U pip setuptools wheel pytest + python3 -m pip install -U pip + python3 -m pip install -U build packaging setuptools wheel pytest python3 -m pip install -U -r examples/requirements.txt diff --git a/pyproject.toml b/pyproject.toml index d7096f82d..f53522b56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,7 @@ requires = [ "setuptools>=42", "wheel", - "cmake>=3.20.0,<4.0.0" + "cmake>=3.20.0,<4.0.0", + "packaging>=23", ] build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py index 1c8b1e365..0a87d1f44 100644 --- a/setup.py +++ b/setup.py @@ -5,9 +5,6 @@ # Authors: Axel Huebl # License: BSD-3-Clause-LBNL # -from distutils.command.build import build -from distutils.command.clean import clean -from distutils.version import LooseVersion import os import platform import re @@ -16,6 +13,7 @@ import sys from setuptools import Extension, setup +from setuptools.command.build import build from setuptools.command.build_ext import build_ext @@ -33,10 +31,8 @@ def run(self): # by default, this stays around. we want to make sure generated # files like libwarpx.(2d|3d|rz).(so|pyd) are always only the # ones we want to package and not ones from an earlier wheel's stage - c = clean(self.distribution) - c.all = True - c.finalize_options() - c.run() + if os.path.exists(self.build_base): + shutil.rmtree(self.build_base) # call superclass build.run(self) @@ -68,6 +64,8 @@ def __init__(self, name, sourcedir=""): class CMakeBuild(build_ext): def run(self): + from packaging.version import parse + try: out = subprocess.check_output(["cmake", "--version"]) except OSError: @@ -77,10 +75,8 @@ def run(self): + ", ".join(e.name for e in self.extensions) ) - cmake_version = LooseVersion( - re.search(r"version\s*([\d.]+)", out.decode()).group(1) - ) - if cmake_version < "3.20.0": + cmake_version = parse(re.search(r"version\s*([\d.]+)", out.decode()).group(1)) + if cmake_version < parse("3.20.0"): raise RuntimeError("CMake >= 3.20.0 is required") for ext in self.extensions: