Updated python recipes

- removed some older intel based python recipes to deprecated
- updated the requirements files of some python recipes
- updated recipes to be able to handle newer os base images

python/cuda-hpc-python/Dockerfile

@@ -0,0 +1 @@
+buildhpcpythoncuda.dockerfile

python/cuda-hpc-python/buildhpcpythoncuda.dockerfile

@@ -0,0 +1,82 @@
+ARG PAWSEY_BASE="3.11"
+FROM quay.io/pawsey/hpc-python:${PAWSEY_BASE}
+
+ARG CUDA_VERSION=12.3
+ARG CUDA_PKG_VERSION=12-3
+
+RUN echo "Installing CUDA" \
+    && wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.1-1_all.deb \
+    && dpkg -i cuda-keyring_1.1-1_all.deb \
+    && apt-get update \
+    && apt-get -y install cuda-toolkit-${CUDA_PKG_VERSION} \
+    && apt-get install -y nvidia-kernel-open-545 \
+    && apt-get install -y cuda-drivers-545 \
+    && echo "Done"
+
+RUN echo "Installing NVHPC SDK" \
+    && curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg \
+    && echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list \
+    && apt-get update -y \
+    && apt-get install -y nvhpc-23-11 \
+    && echo "Done"
+
+# Update to libmozjs which is at this point version 68. Upgrade to 102 
+#RUN echo "Updating libmozjs" \
+#    && sed -i "s/focal/jammy/g"  /etc/apt/sources.list \
+#    && apt-get update \ 
+#    && apt-get install -y libmozjs-102-0 \
+#    && apt-get remove -y libmozjs-68-0 \
+#    && echo "Done"
+
+ENV CUDA_VERSION 12.3
+ENV CUDA_PKG_VERSION 12-3=$CUDA_VERSION-1
+
+# # For libraries in the cuda-compat-* package: https://docs.nvidia.com/cuda/eula/index.html#attachment-a
+# RUN apt-get update && apt-get install -y --no-install-recommends \
+#     cuda-cudart-${CUDA_PKG_VERSION} \
+#     cuda-compat-${CUDA_PKG_VERSION} \
+#     && ln -s cuda-${CUDA_VERSION}} /usr/local/cuda && \
+#     rm -rf /var/lib/apt/lists/*
+
+# # Required for nvidia-docker v1
+# RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf && \
+#     echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf
+
+# ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
+# ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64
+
+# nvidia-container-runtime
+ENV NVIDIA_VISIBLE_DEVICES all
+ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
+ENV NVIDIA_REQUIRE_CUDA "cuda>=12.3" 
+#brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411 brand=tesla,driver>=418,driver<419 brand=tesla,driver>=440,driver<441"
+
+# ENV NCCL_VERSION 2.10.3
+
+
+# RUN apt-get update && apt-get install -y --no-install-recommends \
+#     cuda-libraries-$CUDA_PKG_VERSION \
+#     cuda-npp-$CUDA_PKG_VERSION \
+#     cuda-nvtx-$CUDA_PKG_VERSION \
+#     cuda-cusparse-$CUDA_PKG_VERSION \
+#     libcublas10=10.2.2.89-1 \
+#     libnccl2=$NCCL_VERSION-1+cuda10.2 \
+#     && apt-mark hold libcublas10 libnccl2 \
+#     && rm -rf /var/lib/apt/lists/*
+
+# RUN apt-get update && apt-get install -y --no-install-recommends \
+#     cuda-nvml-dev-$CUDA_PKG_VERSION \
+#     cuda-command-line-tools-$CUDA_PKG_VERSION \
+#     cuda-nvprof-$CUDA_PKG_VERSION \
+#     cuda-npp-dev-$CUDA_PKG_VERSION \
+#     cuda-libraries-dev-$CUDA_PKG_VERSION \
+#     cuda-minimal-build-$CUDA_PKG_VERSION \
+#     libcublas-dev=10.2.2.89-1 \
+#     libnccl-dev=$NCCL_VERSION-1+cuda10.2 \
+#     && apt-mark hold libcublas-dev libnccl-dev \
+#     && rm -rf /var/lib/apt/lists/*
+
+# ENV LIBRARY_PATH /usr/local/cuda/lib64/stubs
+
+##### END   NVIDIA CUDA DOCKERFILES ####
+

python/deprecated/intel-hpc-python/Dockerfile

@@ -0,0 +1 @@
+buildhpcpythonintel.dockerfile

python/intel-hpc-python/Dockerfile → python/deprecated/intel-hpc-python/buildhpcpythonintel.dockerfile

@@ -1,13 +1,11 @@
-ARG IPY_VERSION="2020.2"
+ARG IPY_VERSION="2023.1.0-0"
 FROM intelpython/intelpython3_core:${IPY_VERSION}
 
-LABEL maintainer="[email protected]"
-
+LABEL maintainer="[email protected]"
 
 # Just one extra conda variable
 ENV CONDA_PREFIX="/opt/conda"
 
-
 # Install package dependencies
 RUN apt-get update -qq --allow-releaseinfo-change \
       && apt-get -y --no-install-recommends install \
@@ -68,6 +66,9 @@ ARG MPI4PY_VERSION="3.1.3"
 
 RUN pip --no-cache-dir install --no-deps ${DEPENDENCIES} h5netcdf==${H5NETCDF_VERSION} h5py==${H5PY_VERSION} mpi4py==${MPI4PY_VERSION}
 
+# and copy the recipe into the docker recipes directory
+RUN mkdir -p /opt/docker-recipes/
+COPY buildhpcpythonintel.dockerfile /opt/docker-recipes/
 
 
 # Final

python/hpc-python-hdf5mpi/buildhpcpythonhdf5mpi.dockerfile

@@ -34,7 +34,7 @@ ARG DATE_FILE="12-2023"
 RUN echo "Install h5py " \
     && H5PY_VERSION=$(grep 'h5py' /requirements-${DATE_FILE}.txt | sed 's/==/ /g' | sed 's/ # / /'g | awk '{print $2}') \
     && H5NETCDF_VERSION=$(grep 'h5netcdf' /requirements-${DATE_FILE}.txt | sed 's/==/ /g' | sed 's/ # / /'g | awk '{print $2}') \
-    && CC="mpicc" CXX="mpic++" HDF5_MPI="ON" HDF5_DIR="/usr/local" pip3 --no-cache-dir install --no-deps --no-binary=h5py h5py==${H5PY_VERSION} h5netcdf==${H5NETCDF_VERSION}
+    && CC="mpicc" CXX="mpic++" HDF5_MPI="ON" HDF5_DIR="/usr/local" pip3 --no-cache-dir install --break-system-packages --no-deps --no-binary=h5py h5py==${H5PY_VERSION} h5netcdf==${H5NETCDF_VERSION}
 
 # and copy the recipe into the docker recipes directory
 RUN mkdir -p /opt/docker-recipes/

python/hpc-python/buildhpcpython.dockerfile

@@ -1,40 +1,45 @@
-ARG IMAGE_VERSION="3.4.3_ubuntu20.04_lustrerelease"
-FROM quay.io/pawsey/mpich-lustre-base:${IMAGE_VERSION}
+ARG MPICH_VERSION=3.4.3
+ARG OS_VERSION=ubuntu23.04
+#ARG IMAGE_VERSION="3.4.3_ubuntu23.04_lustrerelease"
+FROM quay.io/pawsey/mpich-lustre-base:${MPICH_VERSION}_${OS_VERSION}_lustrerelease
 ARG PY_VERSION=3.11
 
-LABEL org.opencontainers.image.created="2023-12"
+LABEL org.opencontainers.image.created="2024-01"
 LABEL org.opencontainers.image.authors="Pascal Jahan Elahi <[email protected]>"
 LABEL org.opencontainers.image.documentation="https://github.com/PawseySC/pawsey-containers/"
 LABEL org.opencontainers.image.source="https://github.com/PawseySC/pawsey-containers/python/hpcpython/buildhpcpython.dockerfile"
 LABEL org.opencontainers.image.vendor="Pawsey Supercomputing Research Centre"
 LABEL org.opencontainers.image.licenses="GNU GPL3.0"
 LABEL org.opencontainers.image.title="Setonix compatible python container"
 LABEL org.opencontainers.image.description="Common base image providing python on Setonix built on the lustre aware mpich"
-LABEL org.opencontainers.image.base.name="pawsey/hpcpython:${PY_VERSION}"
+LABEL org.opencontainers.image.base.name="pawsey/hpcpython:${PY_VERSION}-ubuntu23.04"
 
 # add python 
 ENV DEBIAN_FRONTEND="noninteractive"
 RUN echo "Updating python to ${PY_VERSION}" \
     && apt-get update -qq \
     && apt install -y --no-install-recommends software-properties-common \
-    && add-apt-repository ppa:deadsnakes/ppa \
-    && apt-get update -qq \
+    && if [ ${OS_VERSION} != "ubuntu23.04" ]; then add-apt-repository ppa:deadsnakes/ppa; apt-get update -qq; fi \
     && apt-get -y --no-install-recommends install \
         python${PY_VERSION}-dev python${PY_VERSION}-distutils python${PY_VERSION}-full \
     && apt-get clean all \
     && rm -r /var/lib/apt/lists/* \
+    && echo "Done"
+RUN echo "Setting default python to desired version" \
     && update-alternatives --install /usr/bin/python python /usr/bin/python${PY_VERSION} 1 \
     && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PY_VERSION} 1 \
-    && curl -sS https://bootstrap.pypa.io/get-pip.py | sudo python3 \
-    && curl -sS https://bootstrap.pypa.io/get-pip.py | python3 \
-    && python -m pip install pip --upgrade \
-    && pip install pip-tools \
+    && if [ ${OS_VERSION} != "ubuntu23.04" ]; then \
+        curl -sS https://bootstrap.pypa.io/get-pip.py | sudo python3; \
+        curl -sS https://bootstrap.pypa.io/get-pip.py | python3; \
+        python -m pip install --upgrade --break-system-packages pip; \
+       fi \
+    && pip install --break-system-packages pip-tools \
     && echo "Done "
 
 # Install Python packages
 ARG DATE_FILE="12-2023"
 ADD requirements.in requirements-${DATE_FILE}.txt /
-RUN pip3 --no-cache-dir install -r /requirements-${DATE_FILE}.txt --no-deps
+RUN pip3 --no-cache-dir install --break-system-packages -r /requirements-${DATE_FILE}.txt --no-deps
 
 # and copy the recipe into the docker recipes directory
 RUN mkdir -p /opt/docker-recipes/